placate Grim Reaper with sun4v support

178 files changed, 37469 insertions, 0 deletions

diff --git a/sys/conf/Makefile.sun4v b/sys/conf/Makefile.sun4v
new file mode 100644
index 000000000000..947ec87ace50
--- /dev/null
+++ b/sys/conf/Makefile.sun4v
@@ -0,0 +1,49 @@
+# Makefile.sparc64 -- with config changes.
+# Copyright 1990 W. Jolitz
+#	from: @(#)Makefile.i386	7.1 5/10/91
+# $FreeBSD$
+#
+# Makefile for FreeBSD
+#
+# This makefile is constructed from a machine description:
+#	config machineid
+# Most changes should be made in the machine description
+#	/sys/sparc64/conf/``machineid''
+# after which you should do
+#	 config machineid
+# Generic makefile changes should be made in
+#	/sys/conf/Makefile.sparc64
+# after which config should be rerun for all machines.
+#
+
+# Which version of config(8) is required.
+%VERSREQ=	600004
+
+STD8X16FONT?=	iso
+
+.if !defined(S)
+.if exists(./@/.)
+S=	./@
+.else
+S=	../../..
+.endif
+.endif
+.include "$S/conf/kern.pre.mk"
+
+MDOBJS=	exception.o interrupt.o
+
+%BEFORE_DEPEND
+
+%OBJS
+
+%FILES.c
+
+%FILES.s
+
+%FILES.m
+
+%CLEAN
+
+%RULES
+
+.include "$S/conf/kern.post.mk"
diff --git a/sys/conf/files.sun4v b/sys/conf/files.sun4v
new file mode 100644
index 000000000000..1af3f2fe8434
--- /dev/null
+++ b/sys/conf/files.sun4v
@@ -0,0 +1,115 @@
+# This file tells config what files go into building a kernel,
+# files marked standard are always included.
+#
+# $FreeBSD$
+#
+# The long compile-with and dependency lines are required because of
+# limitations in config: backslash-newline doesn't work in strings, and
+# dependency lines other than the first are silently ignored.
+#
+atkbdmap.h			optional	atkbd_dflt_keymap	\
+	compile-with	"/usr/sbin/kbdcontrol -L ${ATKBD_DFLT_KEYMAP} | sed -e 's/^static keymap_t.* = /static keymap_t key_map = /' -e 's/^static accentmap_t.* = /static accentmap_t accent_map = /' > atkbdmap.h"			\
+	no-obj no-implicit-rule before-depend				\
+	clean		"atkbdmap.h"
+#
+ukbdmap.h			optional	ukbd_dflt_keymap	\
+	compile-with	"/usr/sbin/kbdcontrol -L ${UKBD_DFLT_KEYMAP} | sed -e 's/^static keymap_t.* = /static keymap_t key_map = /' -e 's/^static accentmap_t.* = /static accentmap_t accent_map = /' > ukbdmap.h"			\
+	no-obj no-implicit-rule before-depend				\
+	clean		"ukbdmap.h"
+#
+#
+crypto/blowfish/bf_enc.c	optional	crypto | ipsec ipsec_esp
+crypto/des/des_enc.c		optional	crypto | ipsec ipsec_esp | \
+						netsmbcrypto
+dev/ofw/ofw_bus_if.m		standard
+dev/ofw/ofw_bus_subr.c		standard
+dev/ofw/ofw_console.c		optional	ofw_console
+dev/ofw/openfirm.c		standard
+dev/ofw/openfirmio.c		standard
+dev/ofw/openpromio.c		standard
+dev/uart/uart_cpu_sparc64.c	optional	uart
+geom/geom_bsd.c			standard
+geom/geom_bsd_enc.c		standard
+geom/geom_sunlabel.c		standard
+geom/geom_sunlabel_enc.c	standard
+kern/syscalls.c			optional	ktr
+libkern/ffs.c			standard
+libkern/ffsl.c			standard
+libkern/fls.c			standard
+libkern/flsl.c			standard
+sun4v/sun4v/autoconf.c	standard
+sun4v/sun4v/bus_machdep.c	standard
+sun4v/sun4v/clock.c		standard
+sun4v/sun4v/counter.c	standard
+sun4v/sun4v/db_disasm.c	optional	ddb
+sun4v/sun4v/db_interface.c	optional	ddb
+sun4v/sun4v/db_trace.c	optional	ddb
+sun4v/sun4v/db_hwwatch.c	optional	ddb
+sun4v/sun4v/dump_machdep.c	standard
+sun4v/sun4v/elf_machdep.c	standard
+sun4v/sun4v/exception.S	standard	no-obj
+sun4v/sun4v/eeprom.c	optional	eeprom ebus | eeprom fhc | \
+						eeprom sbus
+sun4v/sun4v/gdb_machdep.c	optional	gdb
+sun4v/sun4v/hv_pci.c	optional	pci
+sun4v/sun4v/trap_trace.S	optional	trap_tracing
+sparc64/pci/ofw_pci.c		optional	pci
+sparc64/pci/ofw_pcib.c		optional	pci
+sparc64/pci/ofw_pcib_subr.c	optional	pci
+sparc64/pci/ofw_pcibus.c	optional	pci
+sparc64/pci/ofw_pci_if.m	optional	pci
+sparc64/dtrace/dtrace_asm.s	optional	dtrace
+sparc64/dtrace/dtrace_isa.c	optional	dtrace
+sparc64/dtrace/fasttrap_isa.c	optional	dtrace
+sparc64/dtrace/fbt.c		optional	dtrace
+sparc64/dtrace/sdt.c		optional	dtrace
+sparc64/dtrace/dtrace_stub.c	standard
+
+# XXX hvcons should be optional
+sun4v/sun4v/hvcons.c	standard	
+sun4v/sun4v/hcall.S	standard
+sun4v/sun4v/hviommu.c	standard
+sun4v/sun4v/identcpu.c	standard
+sun4v/sun4v/in_cksum.c	optional	inet
+sun4v/sun4v/interrupt.S	standard	no-obj
+sun4v/sun4v/intr_machdep.c	standard
+sun4v/sun4v/locore.S	standard	no-obj
+sun4v/sun4v/machdep.c	standard
+sun4v/sun4v/mem.c		optional	mem
+sun4v/sun4v/mp_exception.S	optional	smp
+sun4v/sun4v/mp_locore.S	optional	smp
+sun4v/sun4v/mp_machdep.c	optional	smp
+sun4v/sun4v/nexus.c		standard
+sun4v/sun4v/t1_copy.S	standard
+sun4v/sun4v/ofw_bus.c	standard
+sun4v/sun4v/ofw_machdep.c	standard
+sun4v/sun4v/pmap.c		standard
+sun4v/sun4v/prof_machdep.c	optional	profiling-routine
+sun4v/sun4v/rwindow.c		standard
+sun4v/sun4v/simdisk.c		optional	simulator
+sun4v/sun4v/support.S		standard
+sun4v/sun4v/sys_machdep.c	standard
+sun4v/sun4v/swtch.S		standard
+sun4v/sun4v/tsb.c		standard
+sun4v/sun4v/tte.c		standard
+sun4v/sun4v/tte_hash.c		standard
+sun4v/sun4v/tick.c		standard
+sun4v/sun4v/trap.c		standard
+sun4v/sun4v/uio_machdep.c	standard
+sun4v/sun4v/vm_machdep.c	standard
+sun4v/sun4v/vnex.c		standard
+
+
+sun4v/mdesc/mdesc_bus_if.m	standard
+sun4v/mdesc/mdesc_scandag.c	standard
+sun4v/mdesc/mdesc_init.c	standard
+sun4v/mdesc/mdesc_init_intern.c	standard
+sun4v/mdesc/mdesc_subr.c	standard
+sun4v/mdesc/mdesc_bus_subr.c	standard
+sun4v/mdesc/mdesc_getpropstr.c	standard
+sun4v/mdesc/mdesc_getpropval.c	standard
+sun4v/mdesc/mdesc_rootnode.c	standard
+sun4v/mdesc/mdesc_findname.c	standard
+sun4v/mdesc/mdesc_nodecount.c	standard
+sun4v/mdesc/mdesc_findnodeprop.c	standard
+sun4v/mdesc/mdesc_vdevfindval.c	standard
diff --git a/sys/conf/options.sun4v b/sys/conf/options.sun4v
new file mode 100644
index 000000000000..393be4608e8f
--- /dev/null
+++ b/sys/conf/options.sun4v
@@ -0,0 +1,20 @@
+# $FreeBSD$
+
+SUN4V			opt_global.h
+
+GFB_DEBUG		opt_gfb.h
+GFB_NO_FONT_LOADING	opt_gfb.h
+GFB_NO_MODE_CHANGE	opt_gfb.h
+
+DEBUGGER_ON_POWERFAIL	opt_psycho.h
+OFW_PCI_DEBUG		opt_ofw_pci.h
+OFWCONS_POLL_HZ		opt_ofw.h
+# Debug IOMMU inserts/removes using diagnostic accesses. Very loud.
+IOMMU_DIAG		opt_iommu.h
+PMAP_STATS		opt_pmap.h
+SIMULATOR		opt_simulator.h
+
+DTRACE			opt_global.h
+
+TRAP_TRACING		opt_trap_trace.h
+TRAP_TRACE_ENTRIES	opt_trap_trace.h
diff --git a/sys/sun4v/compile/.cvsignore b/sys/sun4v/compile/.cvsignore
new file mode 100644
index 000000000000..e69de29bb2d1
--- /dev/null
+++ b/sys/sun4v/compile/.cvsignore
diff --git a/sys/sun4v/conf/DEFAULTS b/sys/sun4v/conf/DEFAULTS
new file mode 100644
index 000000000000..5695ed658457
--- /dev/null
+++ b/sys/sun4v/conf/DEFAULTS
@@ -0,0 +1,9 @@
+#
+# DEFAULTS -- Default kernel configuration file for FreeBSD/sparc64
+#
+# $FreeBSD$
+
+machine		sun4v
+
+# Pseudo devices.
+device		mem		# Memory and kernel memory devices
diff --git a/sys/sun4v/conf/GENERIC b/sys/sun4v/conf/GENERIC
new file mode 100644
index 000000000000..e5421c61cd1a
--- /dev/null
+++ b/sys/sun4v/conf/GENERIC
@@ -0,0 +1,217 @@
+#
+# GENERIC -- Generic kernel configuration file for FreeBSD/sparc64
+#
+# For more information on this file, please read the handbook section on
+# Kernel Configuration Files:
+#
+#    http://www.FreeBSD.org/doc/en_US.ISO8859-1/books/handbook/kernelconfig-config.html
+#
+# The handbook is also available locally in /usr/share/doc/handbook
+# if you've installed the doc distribution, otherwise always see the
+# FreeBSD World Wide Web server (http://www.FreeBSD.org/) for the
+# latest information.
+#
+# An exhaustive list of options and more detailed explanations of the
+# device lines is also present in the ../../conf/NOTES and NOTES files.
+# If you are in doubt as to the purpose or necessity of a line, check first
+# in NOTES.
+#
+# For hardware specific information check HARDWARE.TXT
+#
+# $FreeBSD$
+
+cpu		SUN4V
+ident		GENERIC
+
+makeoptions	DEBUG=-g		# Build kernel with gdb(1) debug symbols
+makeoptions	NO_MODULES=""		# disable modules (aicasm in buildenv)
+
+# Platforms supported
+#	At this time all platforms are supported, as-is.
+
+#options 	SCHED_ULE		# ULE scheduler
+options 	SCHED_4BSD		# 4BSD scheduler
+#options 	PREEMPTION		# Enable kernel thread preemption
+options 	INET			# InterNETworking
+options 	INET6			# IPv6 communications protocols
+options 	FFS			# Berkeley Fast Filesystem
+options 	SOFTUPDATES		# Enable FFS soft updates support
+options 	UFS_ACL			# Support for access control lists
+options 	UFS_DIRHASH		# Improve performance on big directories
+options 	MD_ROOT			# MD is a potential root device
+options 	NFSCLIENT		# Network Filesystem Client
+options 	NFSSERVER		# Network Filesystem Server
+options 	NFS_ROOT		# NFS usable as /, requires NFSCLIENT
+#options 	MSDOSFS			# MSDOS Filesystem
+options		NULLFS
+options 	CD9660			# ISO 9660 Filesystem
+options 	PROCFS			# Process filesystem (requires PSEUDOFS)
+options 	PSEUDOFS		# Pseudo-filesystem framework
+options 	GEOM_GPT		# GUID Partition Tables.
+options 	COMPAT_43		# Compatible with BSD 4.3 [KEEP THIS!]
+options 	COMPAT_43TTY		# BSD 4.3 TTY compat [KEEP THIS!]
+options 	COMPAT_FREEBSD5		# Compatible with FreeBSD5
+options 	SCSI_DELAY=5000		# Delay (in ms) before probing SCSI
+options 	KTRACE			# ktrace(1) support
+options 	SYSVSHM			# SYSV-style shared memory
+options 	SYSVMSG			# SYSV-style message queues
+options 	SYSVSEM			# SYSV-style semaphores
+options 	_KPOSIX_PRIORITY_SCHEDULING # POSIX P1003_1B real-time extensions
+options 	AHC_REG_PRETTY_PRINT	# Print register bitfields in debug
+					# output.  Adds ~128k to driver.
+options 	ADAPTIVE_GIANT		# Giant mutex is adaptive.
+
+# Debugging for use in -current
+options 	KDB			# Enable kernel debugger support.
+options		KDB_TRACE
+options 	DDB			# Support DDB.
+options 	TRAP_TRACING		# Enable trap tracing.
+options 	TRAP_TRACE_ENTRIES=256	# Trap trace buffer entries.
+#options 	GDB			# Support remote GDB.
+#options 	INVARIANTS		# Enable calls of extra sanity checking
+#options 	INVARIANT_SUPPORT	# Extra sanity checks of internal structures, required by INVARIANTS
+#options 	WITNESS			# Enable checks to detect deadlocks and cycles
+#options 	WITNESS_SKIPSPIN	# Don't run witness on spinlocks for speed
+#options		DEBUG_LOCKS
+#options		DEBUG_VFS_LOCKS
+
+# To make an SMP kernel, the next line is needed
+options 	SMP			# Symmetric MultiProcessor Kernel
+
+# Standard busses
+device		pci
+#device		isa
+
+
+# Floppy drives
+#device		fdc
+
+# ATA and ATAPI devices
+#device		ata
+#device		atadisk		# ATA disk drives
+#device		atapicd		# ATAPI CDROM drives
+#device		atapifd		# ATAPI floppy drives
+#device		atapist		# ATAPI tape drives
+#	Do NOT enable ATA_STATIC_ID -- cmd646 controller will be !ata2!,
+#	and you will not mount an ATA /.
+#options 	ATA_STATIC_ID	# Static device numbering
+
+# SCSI Controllers
+#device		ahc		# AHA2940 and onboard AIC7xxx devices
+#device		isp		# Qlogic family
+#device		ispfw		# Firmware module for Qlogic host adapters
+device		mpt		# LSI-Logic MPT-Fusion (not yet)
+#device		ncr		# NCR/Symbios Logic
+#device		sym		# NCR/Symbios Logic (newer chipsets + those of `ncr')
+#device		esp		# NCR53c9x (FEPS/FAS366)
+
+# SCSI peripherals
+device		scbus		# SCSI bus (required for SCSI)
+device		ch		# SCSI media changers
+device		da		# Direct Access (disks)
+device		sa		# Sequential Access (tape etc)
+device		cd		# CD
+device		pass		# Passthrough device (direct SCSI access)
+device		ses		# SCSI Environmental Services (and SAF-TE)
+
+# RAID controllers
+#device		amr		# AMI MegaRAID
+#device		mlx		# Mylex DAC960 family
+
+# syscons is the default console driver, resembling an SCO console
+#device		sc
+#device		splash		# Splash screen and screen saver support
+#options 	KBD_INSTALL_CDEV # install a CDEV entry in /dev
+
+#device		ofw_console	# Open Firmware console device
+
+# Builtin hardware
+#device		auxio		# auxiliary I/O device
+#device		clkbrd		# Clock Board (blinkenlight on Sun Exx00)
+device		genclock	# Generic clock interface
+#device		eeprom		# eeprom (really a front-end for the MK48Txx)
+#device		mk48txx		# Mostek MK48Txx clocks
+#device		rtc		# rtc (really a front-end for the MC146818)
+#device		mc146818	# Motorola MC146818 and compatible clocks
+
+# Serial (COM) ports
+#device		sab		# Siemens SAB82532 based serial ports
+#device		uart		# Multi-uart driver
+#device		puc		# Multi-channel uarts
+
+# Parallel port
+#device		ppc
+#device		ppbus		# Parallel port bus (required)
+#device		lpt		# Printer
+#device		plip		# TCP/IP over parallel
+#device		ppi		# Parallel port interface device
+#device		vpo		# Requires scbus and da
+ 
+# PCI Ethernet NICs.
+#device		de		# DEC/Intel DC21x4x (``Tulip'')
+device		em		# Intel PRO/1000 adapter Gigabit Ethernet Card
+#device		ixgb		# Intel PRO/10GbE Ethernet Card
+#device		le		# AMD Am7900 LANCE and Am79C9xx PCnet
+#device		txp		# 3Com 3cR990 (``Typhoon'')
+#device		vx		# 3Com 3c590, 3c595 (``Vortex'')
+
+# PCI Ethernet NICs that use the common MII bus controller code.
+# NOTE: Be sure to keep the 'device miibus' line in order to use these NICs!
+device		miibus		# MII bus support
+device		bge		# Broadcom BCM570xx Gigabit Ethernet
+
+# Pseudo devices.
+device		loop		# Network loopback
+device		random		# Entropy device
+device		ether		# Ethernet support
+device		sl		# Kernel SLIP
+device		ppp		# Kernel PPP
+device		tun		# Packet tunnel.
+device		pty		# Pseudo-ttys (telnet etc)
+device		md		# Memory "disks"
+device		gif		# IPv6 and IPv4 tunneling
+device		faith		# IPv6-to-IPv4 relaying (translation)
+
+# The `bpf' device enables the Berkeley Packet Filter.
+# Be aware of the administrative consequences of enabling this!
+# Note that 'bpf' is required for DHCP.
+device		bpf		# Berkeley packet filter
+
+# USB support
+#device		uhci		# UHCI PCI->USB interface
+#device		ohci		# OHCI PCI->USB interface
+device		usb		# USB Bus (required)
+#device		udbp		# USB Double Bulk Pipe devices
+device		ugen		# Generic
+device		uhid		# "Human Interface Devices"
+#device		ukbd		# Keyboard
+device		ulpt		# Printer
+#device		umass		# Disks/Mass storage - Requires scbus and da
+device		ums		# Mouse
+#device		urio		# Diamond Rio 500 MP3 player
+#device		uscanner	# Scanners
+# USB Ethernet, requires mii
+#device		aue		# ADMtek USB Ethernet
+#device		axe		# ASIX Electronics USB Ethernet
+#device		cdce		# Generic USB over Ethernet
+#device		cue		# CATC USB Ethernet
+#device		kue		# Kawasaki LSI USB Ethernet
+#device		rue		# RealTek RTL8150 USB Ethernet
+
+# FireWire support
+#device		firewire	# FireWire bus code
+#device		sbp		# SCSI over FireWire (Requires scbus and da)
+#device		fwe		# Ethernet over FireWire (non-standard!)
+
+#options		SIMULATOR	# compile for ISA simulator
+#options		KTR
+#options		KTR_MASK=(KTR_TRAP)
+#options			USB_DEBUG
+#options			USBVERBOSE
+#options			DEBUG_LOCKS
+#options			DEBUG_VFS_LOCKS
+
+device		ccd
+options		SPIN_PROFILING
+options		HZ=100
+#device		vnet
diff --git a/sys/sun4v/conf/GENERIC.hints b/sys/sun4v/conf/GENERIC.hints
new file mode 100644
index 000000000000..e8c0da7a1d7e
--- /dev/null
+++ b/sys/sun4v/conf/GENERIC.hints
@@ -0,0 +1 @@
+# $FreeBSD$
diff --git a/sys/sun4v/conf/MAC b/sys/sun4v/conf/MAC
new file mode 100644
index 000000000000..56af83d0a806
--- /dev/null
+++ b/sys/sun4v/conf/MAC
@@ -0,0 +1,28 @@
+# MAC -- Generic kernel configuration file for FreeBSD/sparc64 MAC
+#
+# The Mandatory Access Control, or MAC, framework allows administrators to
+# finely control system security by providing for a loadable security pol-
+# icy architecture.
+#
+# For more information see:
+#
+# http://www.FreeBSD.org/doc/en_US.ISO8859-1/books/handbook/mac.html 
+#
+# $FreeBSD$
+
+include 	GENERIC
+ident 		MAC
+
+options 	MAC
+
+#options 	MAC_BIBA		# BIBA data integrity policy
+#options 	MAC_BSDEXTENDED		# File system firewall policy
+#options 	MAC_IFOFF		# Network interface silencing policy
+#options 	MAC_LOMAC		# Low-watermark data integrity policy
+#options 	MAC_MLS			# Multi-level confidentiality policy
+#options 	MAC_NONE		# NULL policy
+#options 	MAC_PARTITION		# Process partition policy
+#options 	MAC_PORTACL		# Network port access control policy
+#options 	MAC_SEEOTHERUIDS	# UID visibility policy
+#options 	MAC_STUB		# Stub policy
+#options 	MAC_TEST		# Testing policy for the MAC framework
diff --git a/sys/sun4v/conf/Makefile b/sys/sun4v/conf/Makefile
new file mode 100644
index 000000000000..2c006e9c2c80
--- /dev/null
+++ b/sys/sun4v/conf/Makefile
@@ -0,0 +1,3 @@
+# $FreeBSD$
+
+.include "${.CURDIR}/../../conf/makeLINT.mk"
diff --git a/sys/sun4v/conf/NOTES b/sys/sun4v/conf/NOTES
new file mode 100644
index 000000000000..4cc7c45cd775
--- /dev/null
+++ b/sys/sun4v/conf/NOTES
@@ -0,0 +1,132 @@
+# $FreeBSD$
+#
+# This file contains machine dependent kernel configuration notes.  For
+# machine independent notes, look in /sys/conf/NOTES.
+
+
+#####################################################################
+# CPU OPTIONS
+
+#
+# You must specify at least one CPU (the one you intend to run on);
+# deleting the specification for CPUs you don't need to use may make
+# parts of the system run faster.
+# XXX: On the Sparc64, there is only one CPU type
+cpu		SUN4U
+
+
+#####################################################################
+# HARDWARE BUS CONFIGURATION
+
+device		ebus
+device		isa
+device		pci
+device		sbus
+device		central
+device		fhc
+
+
+#####################################################################
+# HARDWARE DEVICE CONFIGURATION
+
+#
+# Mandatory devices:
+#
+
+device		genclock	# Generic clock interface
+device		eeprom		# eeprom (really a front-end for the MK48Txx)
+device		mk48txx		# Mostek MK48Txx clocks
+device		rtc		# rtc (really a front-end for the MC146818)
+device		mc146818	# Motorola MC146818 and compatible clocks
+
+#
+# Optional devices:
+#
+
+device		auxio		# auxiliary I/O device
+device		clkbrd		# Clock Board (blinkenlight on Sun Exx00)
+device		creator		# Creator, Creator3D and Elite3D framebuffers
+device		machfb		# ATI Mach64 framebuffers
+
+device		ofw_console	# Open Firmware console device
+option 		OFWCONS_POLL_HZ=4 # 20 or more works best on Ultra2
+
+device		sab		# Siemens SAB82532 based serial ports
+
+
+#####################################################################
+# Devices we don't want to deal with
+
+nodevice	vga
+nodevice	daemon_saver
+nodevice	snake_saver
+nodevice	star_saver
+nodevice	bktr
+nodevice	fdc
+nodevice	ppc
+nodevice	snd_ad1816
+nodevice	snd_als4000
+nodevice	snd_au88x0
+nodevice	snd_cmi
+nodevice	snd_cs4281
+nodevice	snd_csa
+nodevice	snd_ds1
+nodevice	snd_emu10k1
+nodevice	snd_ess
+nodevice	snd_fm801
+nodevice	snd_gusc
+nodevice	snd_ich
+nodevice	snd_maestro
+nodevice	snd_maestro3
+nodevice	snd_mss
+nodevice	snd_neomagic
+nodevice	snd_sb16
+nodevice	snd_sb8
+nodevice	snd_sbc
+nodevice	snd_solo
+nodevice	snd_t4dwave
+nodevice	snd_via8233
+nodevice	snd_via82c686
+nodevice	snd_vibes
+nodevice	snd_uaudio
+nodevice	aha
+nodevice	bt
+nodevice	wds
+nodevice	ep
+nodevice	ex
+nodevice	sio
+
+
+#####################################################################
+# Options we don't want to deal with
+
+nooption	FDC_DEBUG
+nooption	COM_ESP
+nooption	CONSPEED
+nooption	VGA_DEBUG
+nooption	SC_RENDER_DEBUG
+nooption	SC_DEBUG_LEVEL
+nooption	PPC_DEBUG
+nooption	PPC_PROBE_CHIPSET
+nooption	SC_NO_SUSPEND_VTYSWITCH
+nooption	SC_NO_FONT_LOADING
+nooption	SC_KERNEL_CONS_REV_ATTR
+nooption	SC_KERNEL_CONS_ATTR
+nooption	SC_NORM_REV_ATTR
+nooption	SC_NORM_ATTR
+nooption	SC_DFLT_FONT
+nooption	SC_ALT_MOUSE_IMAGE
+nooption	VGA_WIDTH90
+nooption	VGA_SLOW_IOACCESS
+nooption	VGA_ALT_SEQACCESS
+nooption	PSM_RESETAFTERSUSPEND
+nooption	PSM_HOOKRESUME
+nooption	ATKBD_DFLT_KEYMAP
+nooption	EXT2FS
+
+
+#####################################################################
+# Make options we don't want to deal with
+
+nomakeoption	SC_DFLT_FONT
+nomakeoption	ATKBD_DFLT_KEYMAP
diff --git a/sys/sun4v/include/_bus.h b/sys/sun4v/include/_bus.h
new file mode 100644
index 000000000000..7cbe96f96fd6
--- /dev/null
+++ b/sys/sun4v/include/_bus.h
@@ -0,0 +1,43 @@
+/*-
+ * Copyright (c) 2005 M. Warner Losh.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions, and the following disclaimer,
+ *    without modification, immediately at the beginning of the file.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef SPARC64_INCLUDE__BUS_H
+#define SPARC64_INCLUDE__BUS_H
+
+/*
+ * Bus address and size types
+ */
+typedef	u_long		bus_space_handle_t;
+typedef int		bus_type_t;
+typedef u_long		bus_addr_t;
+typedef u_long		bus_size_t;
+typedef struct bus_space_tag *bus_space_tag_t;
+
+#endif /* SPARC64_INCLUDE__BUS_H */
diff --git a/sys/sun4v/include/_inttypes.h b/sys/sun4v/include/_inttypes.h
new file mode 100644
index 000000000000..e6b2536ff379
--- /dev/null
+++ b/sys/sun4v/include/_inttypes.h
@@ -0,0 +1,220 @@
+/*-
+ * Copyright (c) 2001 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The NetBSD Foundation
+ * by Klaus Klein.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ *    must display the following acknowledgement:
+ *        This product includes software developed by the NetBSD
+ *        Foundation, Inc. and its contributors.
+ * 4. Neither the name of The NetBSD Foundation nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ *	From: $NetBSD: int_fmtio.h,v 1.2 2001/04/26 16:25:21 kleink Exp $
+ * $FreeBSD$
+ */
+
+#ifndef _MACHINE_INTTYPES_H_
+#define _MACHINE_INTTYPES_H_
+
+/*
+ * Macros for format specifiers.
+ */
+
+/* fprintf(3) macros for signed integers. */
+
+#define	PRId8		"d"	/* int8_t */
+#define	PRId16		"d"	/* int16_t */
+#define	PRId32		"d"	/* int32_t */
+#define	PRId64		"ld"	/* int64_t */
+#define	PRIdLEAST8	"d"	/* int_least8_t */
+#define	PRIdLEAST16	"d"	/* int_least16_t */
+#define	PRIdLEAST32	"d"	/* int_least32_t */
+#define	PRIdLEAST64	"ld"	/* int_least64_t */
+#define	PRIdFAST8	"d"	/* int_fast8_t */
+#define	PRIdFAST16	"d"	/* int_fast16_t */
+#define	PRIdFAST32	"d"	/* int_fast32_t */
+#define	PRIdFAST64	"ld"	/* int_fast64_t */
+#define	PRIdMAX		"jd"	/* intmax_t */
+#define	PRIdPTR		"ld"	/* intptr_t */
+
+#define	PRIi8		"i"	/* int8_t */
+#define	PRIi16		"i"	/* int16_t */
+#define	PRIi32		"i"	/* int32_t */
+#define	PRIi64		"li"	/* int64_t */
+#define	PRIiLEAST8	"i"	/* int_least8_t  */
+#define	PRIiLEAST16	"i"	/* int_least16_t */
+#define	PRIiLEAST32	"i"	/* int_least32_t */
+#define	PRIiLEAST64	"li"	/* int_least64_t */
+#define	PRIiFAST8	"i"	/* int_fast8_t */
+#define	PRIiFAST16	"i"	/* int_fast16_t */
+#define	PRIiFAST32	"i"	/* int_fast32_t */
+#define	PRIiFAST64	"li"	/* int_fast64_t */
+#define	PRIiMAX		"ji"	/* intmax_t */
+#define	PRIiPTR		"li"	/* intptr_t */
+
+/* fprintf(3) macros for unsigned integers. */
+
+#define	PRIo8		"o"	/* uint8_t */
+#define	PRIo16		"o"	/* uint16_t */
+#define	PRIo32		"o"	/* uint32_t */
+#define	PRIo64		"lo"	/* uint64_t */
+#define	PRIoLEAST8	"o"	/* uint_least8_t */
+#define	PRIoLEAST16	"o"	/* uint_least16_t */
+#define	PRIoLEAST32	"o"	/* uint_least32_t */
+#define	PRIoLEAST64	"lo"	/* uint_least64_t */
+#define	PRIoFAST8	"o"	/* uint_fast8_t */
+#define	PRIoFAST16	"o"	/* uint_fast16_t */
+#define	PRIoFAST32	"o"	/* uint_fast32_t */
+#define	PRIoFAST64	"lo"	/* uint_fast64_t */
+#define	PRIoMAX		"jo"	/* uintmax_t */
+#define	PRIoPTR		"lo"	/* uintptr_t */
+
+#define	PRIu8		"u"	/* uint8_t */
+#define	PRIu16		"u"	/* uint16_t */
+#define	PRIu32		"u"	/* uint32_t */
+#define	PRIu64		"lu"	/* uint64_t */
+#define	PRIuLEAST8	"u"	/* uint_least8_t */
+#define	PRIuLEAST16	"u"	/* uint_least16_t */
+#define	PRIuLEAST32	"u"	/* uint_least32_t */
+#define	PRIuLEAST64	"lu"	/* uint_least64_t */
+#define	PRIuFAST8	"u"	/* uint_fast8_t */
+#define	PRIuFAST16	"u"	/* uint_fast16_t */
+#define	PRIuFAST32	"u"	/* uint_fast32_t */
+#define	PRIuFAST64	"lu"	/* uint_fast64_t */
+#define	PRIuMAX		"ju"	/* uintmax_t */
+#define	PRIuPTR		"lu"	/* uintptr_t */
+
+#define	PRIx8		"x"	/* uint8_t */
+#define	PRIx16		"x"	/* uint16_t */
+#define	PRIx32		"x"	/* uint32_t */
+#define	PRIx64		"lx"	/* uint64_t */
+#define	PRIxLEAST8	"x"	/* uint_least8_t */
+#define	PRIxLEAST16	"x"	/* uint_least16_t */
+#define	PRIxLEAST32	"x"	/* uint_least32_t */
+#define	PRIxLEAST64	"lx"	/* uint_least64_t */
+#define	PRIxFAST8	"x"	/* uint_fast8_t */
+#define	PRIxFAST16	"x"	/* uint_fast16_t */
+#define	PRIxFAST32	"x"	/* uint_fast32_t */
+#define	PRIxFAST64	"lx"	/* uint_fast64_t */
+#define	PRIxMAX		"jx"	/* uintmax_t */
+#define	PRIxPTR		"lx"	/* uintptr_t */
+
+#define	PRIX8		"X"	/* uint8_t */
+#define	PRIX16		"X"	/* uint16_t */
+#define	PRIX32		"X"	/* uint32_t */
+#define	PRIX64		"lX"	/* uint64_t */
+#define	PRIXLEAST8	"X"	/* uint_least8_t */
+#define	PRIXLEAST16	"X"	/* uint_least16_t */
+#define	PRIXLEAST32	"X"	/* uint_least32_t */
+#define	PRIXLEAST64	"lX"	/* uint_least64_t */
+#define	PRIXFAST8	"X"	/* uint_fast8_t */
+#define	PRIXFAST16	"X"	/* uint_fast16_t */
+#define	PRIXFAST32	"X"	/* uint_fast32_t */
+#define	PRIXFAST64	"lX"	/* uint_fast64_t */
+#define	PRIXMAX		"jX"	/* uintmax_t */
+#define	PRIXPTR		"lX"	/* uintptr_t */
+
+/* fscanf(3) macros for signed integers. */
+
+#define	SCNd8		"hhd"	/* int8_t */
+#define	SCNd16		"hd"	/* int16_t */
+#define	SCNd32		"d"	/* int32_t */
+#define	SCNd64		"ld"	/* int64_t */
+#define	SCNdLEAST8	"hhd"	/* int_least8_t */
+#define	SCNdLEAST16	"hd"	/* int_least16_t */
+#define	SCNdLEAST32	"d"	/* int_least32_t */
+#define	SCNdLEAST64	"ld"	/* int_least64_t */
+#define	SCNdFAST8	"d"	/* int_fast8_t */
+#define	SCNdFAST16	"d"	/* int_fast16_t */
+#define	SCNdFAST32	"d"	/* int_fast32_t */
+#define	SCNdFAST64	"ld"	/* int_fast64_t */
+#define	SCNdMAX		"jd"	/* intmax_t */
+#define	SCNdPTR		"ld"	/* intptr_t */
+
+#define	SCNi8		"hhi"	/* int8_t */
+#define	SCNi16		"hi"	/* int16_t */
+#define	SCNi32		"i"	/* int32_t */
+#define	SCNi64		"li"	/* int64_t */
+#define	SCNiLEAST8	"hhi"	/* int_least8_t */
+#define	SCNiLEAST16	"hi"	/* int_least16_t */
+#define	SCNiLEAST32	"i"	/* int_least32_t */
+#define	SCNiLEAST64	"li"	/* int_least64_t */
+#define	SCNiFAST8	"i"	/* int_fast8_t */
+#define	SCNiFAST16	"i"	/* int_fast16_t */
+#define	SCNiFAST32	"i"	/* int_fast32_t */
+#define	SCNiFAST64	"li"	/* int_fast64_t */
+#define	SCNiMAX		"ji"	/* intmax_t */
+#define	SCNiPTR		"li"	/* intptr_t */
+
+/* fscanf(3) macros for unsigned integers. */
+
+#define	SCNo8		"hho"	/* uint8_t */
+#define	SCNo16		"ho"	/* uint16_t */
+#define	SCNo32		"o"	/* uint32_t */
+#define	SCNo64		"lo"	/* uint64_t */
+#define	SCNoLEAST8	"hho"	/* uint_least8_t */
+#define	SCNoLEAST16	"ho"	/* uint_least16_t */
+#define	SCNoLEAST32	"o"	/* uint_least32_t */
+#define	SCNoLEAST64	"lo"	/* uint_least64_t */
+#define	SCNoFAST8	"o"	/* uint_fast8_t */
+#define	SCNoFAST16	"o"	/* uint_fast16_t */
+#define	SCNoFAST32	"o"	/* uint_fast32_t */
+#define	SCNoFAST64	"lo"	/* uint_fast64_t */
+#define	SCNoMAX		"jo"	/* uintmax_t */
+#define	SCNoPTR		"lo"	/* uintptr_t */
+
+#define	SCNu8		"hhu"	/* uint8_t */
+#define	SCNu16		"hu"	/* uint16_t */
+#define	SCNu32		"u"	/* uint32_t */
+#define	SCNu64		"lu"	/* uint64_t */
+#define	SCNuLEAST8	"hhu"	/* uint_least8_t */
+#define	SCNuLEAST16	"hu"	/* uint_least16_t */
+#define	SCNuLEAST32	"u"	/* uint_least32_t */
+#define	SCNuLEAST64	"lu"	/* uint_least64_t */
+#define	SCNuFAST8	"u"	/* uint_fast8_t */
+#define	SCNuFAST16	"u"	/* uint_fast16_t */
+#define	SCNuFAST32	"u"	/* uint_fast32_t */
+#define	SCNuFAST64	"lu"	/* uint_fast64_t */
+#define	SCNuMAX		"ju"	/* uintmax_t */
+#define	SCNuPTR		"lu"	/* uintptr_t */
+
+#define	SCNx8		"hhx"	/* uint8_t */
+#define	SCNx16		"hx"	/* uint16_t */
+#define	SCNx32		"x"	/* uint32_t */
+#define	SCNx64		"lx"	/* uint64_t */
+#define	SCNxLEAST8	"hhx"	/* uint_least8_t */
+#define	SCNxLEAST16	"hx"	/* uint_least16_t */
+#define	SCNxLEAST32	"x"	/* uint_least32_t */
+#define	SCNxLEAST64	"lx"	/* uint_least64_t */
+#define	SCNxFAST8	"x"	/* uint_fast8_t */
+#define	SCNxFAST16	"x"	/* uint_fast16_t */
+#define	SCNxFAST32	"x"	/* uint_fast32_t */
+#define	SCNxFAST64	"lx"	/* uint_fast64_t */
+#define	SCNxMAX		"jx"	/* uintmax_t */
+#define	SCNxPTR		"lx"	/* uintptr_t */
+
+#endif /* !_MACHINE_INTTYPES_H_ */
diff --git a/sys/sun4v/include/_limits.h b/sys/sun4v/include/_limits.h
new file mode 100644
index 000000000000..248369288cae
--- /dev/null
+++ b/sys/sun4v/include/_limits.h
@@ -0,0 +1,87 @@
+/*-
+ * Copyright (c) 1988, 1993
+ *	The Regents of the University of California.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	@(#)limits.h	8.3 (Berkeley) 1/4/94
+ * $FreeBSD$
+ */
+
+#ifndef	_MACHINE__LIMITS_H_
+#define	_MACHINE__LIMITS_H_
+
+/*
+ * According to ANSI (section 2.2.4.2), the values below must be usable by
+ * #if preprocessing directives.  Additionally, the expression must have the
+ * same type as would an expression that is an object of the corresponding
+ * type converted according to the integral promotions.  The subtraction for
+ * INT_MIN, etc., is so the value is not unsigned; e.g., 0x80000000 is an
+ * unsigned int for 32-bit two's complement ANSI compilers (section 3.1.3.2).
+ * These numbers are for the default configuration of gcc.  They work for
+ * some other compilers as well, but this should not be depended on.
+ */
+
+#define	__CHAR_BIT	8		/* number of bits in a char */
+
+#define	__SCHAR_MAX	0x7f		/* max value for a signed char */
+#define	__SCHAR_MIN	(-0x7f-1)	/* min value for a signed char */
+
+#define	__UCHAR_MAX	0xffU		/* max value for an unsigned char */
+
+#define	__USHRT_MAX	0xffffU		/* max value for an unsigned short */
+#define	__SHRT_MAX	0x7fff		/* max value for a short */
+#define	__SHRT_MIN	(-0x7fff-1)	/* min value for a short */
+
+#define	__UINT_MAX	0xffffffffU	/* max value for an unsigned int */
+#define	__INT_MAX	0x7fffffff	/* max value for an int */
+#define	__INT_MIN	(-0x7fffffff-1)	/* min value for an int */
+
+#define	__ULONG_MAX	0xffffffffffffffffUL	/* max for an unsigned long */
+#define	__LONG_MAX	0x7fffffffffffffffL	/* max for a long */
+#define	__LONG_MIN	(-0x7fffffffffffffffL-1) /* min for a long */
+
+/* Long longs and longs are the same size on sparc64. */
+					/* max for an unsigned long long */
+#define	__ULLONG_MAX	0xffffffffffffffffULL
+#define	__LLONG_MAX	0x7fffffffffffffffLL	/* max for a long long */
+#define	__LLONG_MIN	(-0x7fffffffffffffffLL-1) /* min for a long long */
+
+#define	__SSIZE_MAX	__LONG_MAX	/* max value for a ssize_t */
+
+#define	__SIZE_T_MAX	__ULONG_MAX	/* max value for a size_t */
+
+#define	__OFF_MAX	__LONG_MAX	/* max value for an off_t */
+#define	__OFF_MIN	__LONG_MIN	/* min value for an off_t */
+
+/* Quads and longs are the same size.  Ensure they stay in sync. */
+#define	__UQUAD_MAX	(__ULONG_MAX)	/* max value for a uquad_t */
+#define	__QUAD_MAX	(__LONG_MAX)	/* max value for a quad_t */
+#define	__QUAD_MIN	(__LONG_MIN)	/* min value for a quad_t */
+
+#define	__LONG_BIT	64
+#define	__WORD_BIT	32
+
+/* Minimum signal stack size. */
+#define	__MINSIGSTKSZ	(1024 * 4)
+
+#endif /* !_MACHINE__LIMITS_H_ */
diff --git a/sys/sun4v/include/_stdint.h b/sys/sun4v/include/_stdint.h
new file mode 100644
index 000000000000..1aed3e3b4efc
--- /dev/null
+++ b/sys/sun4v/include/_stdint.h
@@ -0,0 +1,171 @@
+/*-
+ * Copyright (c) 2001, 2002 Mike Barcroft <mike@FreeBSD.org>
+ * Copyright (c) 2001 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The NetBSD Foundation
+ * by Klaus Klein.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ *    must display the following acknowledgement:
+ *        This product includes software developed by the NetBSD
+ *        Foundation, Inc. and its contributors.
+ * 4. Neither the name of The NetBSD Foundation nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef	_MACHINE__STDINT_H_
+#define	_MACHINE__STDINT_H_
+
+#if !defined(__cplusplus) || defined(__STDC_CONSTANT_MACROS)
+
+#define	INT8_C(c)		(c)
+#define	INT16_C(c)		(c)
+#define	INT32_C(c)		(c)
+#define	INT64_C(c)		(c ## L)
+
+#define	UINT8_C(c)		(c)
+#define	UINT16_C(c)		(c)
+#define	UINT32_C(c)		(c ## U)
+#define	UINT64_C(c)		(c ## UL)
+
+#define	INTMAX_C(c)		(c ## L)
+#define	UINTMAX_C(c)		(c ## UL)
+
+#endif /* !defined(__cplusplus) || defined(__STDC_CONSTANT_MACROS) */
+
+#if !defined(__cplusplus) || defined(__STDC_LIMIT_MACROS)
+
+/*
+ * ISO/IEC 9899:1999
+ * 7.18.2.1 Limits of exact-width integer types
+ */
+/* Minimum values of exact-width signed integer types. */
+#define	INT8_MIN	(-0x7f-1)
+#define	INT16_MIN	(-0x7fff-1)
+#define	INT32_MIN	(-0x7fffffff-1)
+#define	INT64_MIN	(-0x7fffffffffffffffL-1)
+
+/* Maximum values of exact-width signed integer types. */
+#define	INT8_MAX	0x7f
+#define	INT16_MAX	0x7fff
+#define	INT32_MAX	0x7fffffff
+#define	INT64_MAX	0x7fffffffffffffffL
+
+/* Maximum values of exact-width unsigned integer types. */
+#define	UINT8_MAX	0xff
+#define	UINT16_MAX	0xffff
+#define	UINT32_MAX	0xffffffffU
+#define	UINT64_MAX	0xffffffffffffffffUL
+
+/*
+ * ISO/IEC 9899:1999
+ * 7.18.2.2  Limits of minimum-width integer types
+ */
+/* Minimum values of minimum-width signed integer types. */
+#define	INT_LEAST8_MIN	INT8_MIN
+#define	INT_LEAST16_MIN	INT16_MIN
+#define	INT_LEAST32_MIN	INT32_MIN
+#define	INT_LEAST64_MIN	INT64_MIN
+
+/* Maximum values of minimum-width signed integer types. */
+#define	INT_LEAST8_MAX	INT8_MAX
+#define	INT_LEAST16_MAX	INT16_MAX
+#define	INT_LEAST32_MAX	INT32_MAX
+#define	INT_LEAST64_MAX	INT64_MAX
+
+/* Maximum values of minimum-width unsigned integer types. */
+#define	UINT_LEAST8_MAX	 UINT8_MAX
+#define	UINT_LEAST16_MAX UINT16_MAX
+#define	UINT_LEAST32_MAX UINT32_MAX
+#define	UINT_LEAST64_MAX UINT64_MAX
+
+/*
+ * ISO/IEC 9899:1999
+ * 7.18.2.3  Limits of fastest minimum-width integer types
+ */
+/* Minimum values of fastest minimum-width signed integer types. */
+#define	INT_FAST8_MIN	INT32_MIN
+#define	INT_FAST16_MIN	INT32_MIN
+#define	INT_FAST32_MIN	INT32_MIN
+#define	INT_FAST64_MIN	INT64_MIN
+
+/* Maximum values of fastest minimum-width signed integer types. */
+#define	INT_FAST8_MAX	INT32_MAX
+#define	INT_FAST16_MAX	INT32_MAX
+#define	INT_FAST32_MAX	INT32_MAX
+#define	INT_FAST64_MAX	INT64_MAX
+
+/* Maximum values of fastest minimum-width unsigned integer types. */
+#define	UINT_FAST8_MAX	UINT32_MAX
+#define	UINT_FAST16_MAX	UINT32_MAX
+#define	UINT_FAST32_MAX	UINT32_MAX
+#define	UINT_FAST64_MAX	UINT64_MAX
+
+/*
+ * ISO/IEC 9899:1999
+ * 7.18.2.4  Limits of integer types capable of holding object pointers
+ */
+#define	INTPTR_MIN	INT64_MIN
+#define	INTPTR_MAX	INT64_MAX
+#define	UINTPTR_MAX	UINT64_MAX
+
+/*
+ * ISO/IEC 9899:1999
+ * 7.18.2.5  Limits of greatest-width integer types
+ */
+#define	INTMAX_MIN	INT64_MIN
+#define	INTMAX_MAX	INT64_MAX
+#define	UINTMAX_MAX	UINT64_MAX
+
+/*
+ * ISO/IEC 9899:1999
+ * 7.18.3  Limits of other integer types
+ */
+/* Limits of ptrdiff_t. */
+#define	PTRDIFF_MIN	INT64_MIN	
+#define	PTRDIFF_MAX	INT64_MAX
+
+/* Limits of sig_atomic_t. */
+#define	SIG_ATOMIC_MIN	INT32_MIN
+#define	SIG_ATOMIC_MAX	INT32_MAX
+
+/* Limit of size_t. */
+#define	SIZE_MAX	UINT64_MAX
+
+#ifndef WCHAR_MIN /* Also possibly defined in <wchar.h> */
+/* Limits of wchar_t. */
+#define	WCHAR_MIN	INT32_MIN
+#define	WCHAR_MAX	INT32_MAX
+#endif
+
+/* Limits of wint_t. */
+#define	WINT_MIN	INT32_MIN
+#define	WINT_MAX	INT32_MAX
+
+#endif /* !defined(__cplusplus) || defined(__STDC_LIMIT_MACROS) */
+
+#endif /* !_MACHINE__STDINT_H_ */
diff --git a/sys/sun4v/include/_types.h b/sys/sun4v/include/_types.h
new file mode 100644
index 000000000000..612bda749a85
--- /dev/null
+++ b/sys/sun4v/include/_types.h
@@ -0,0 +1,113 @@
+/*-
+ * Copyright (c) 2002 Mike Barcroft <mike@FreeBSD.org>
+ * Copyright (c) 1990, 1993
+ *	The Regents of the University of California.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 4. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	From: @(#)ansi.h	8.2 (Berkeley) 1/4/94
+ *	From: @(#)types.h	8.3 (Berkeley) 1/5/94
+ * $FreeBSD$
+ */
+
+#ifndef _MACHINE__TYPES_H_
+#define	_MACHINE__TYPES_H_
+
+#ifndef _SYS_CDEFS_H_
+#error this file needs sys/cdefs.h as a prerequisite
+#endif
+
+/*
+ * Basic types upon which most other types are built.
+ */
+typedef	__signed char		__int8_t;
+typedef	unsigned char		__uint8_t;
+typedef	short			__int16_t;
+typedef	unsigned short		__uint16_t;
+typedef	int			__int32_t;
+typedef	unsigned int		__uint32_t;
+typedef	long			__int64_t;
+typedef	unsigned long		__uint64_t;
+
+/*
+ * Standard type definitions.
+ */
+typedef	__int32_t	__clock_t;		/* clock()... */
+typedef	__uint32_t	__cpumask_t;
+typedef	__int64_t	__critical_t;
+typedef	double		__double_t;
+typedef	float		__float_t;
+typedef	__int64_t	__intfptr_t;
+typedef	__int64_t	__intmax_t;
+typedef	__int64_t	__intptr_t;
+typedef	__int32_t	__int_fast8_t;
+typedef	__int32_t	__int_fast16_t;
+typedef	__int32_t	__int_fast32_t;
+typedef	__int64_t	__int_fast64_t;
+typedef	__int8_t	__int_least8_t;
+typedef	__int16_t	__int_least16_t;
+typedef	__int32_t	__int_least32_t;
+typedef	__int64_t	__int_least64_t;
+typedef	__int64_t	__ptrdiff_t;		/* ptr1 - ptr2 */
+typedef	__int64_t	__register_t;
+typedef	__int64_t	__segsz_t;		/* segment size (in pages) */
+typedef	__uint64_t	__size_t;		/* sizeof() */
+typedef	__int64_t	__ssize_t;		/* byte count or error */
+typedef	__int64_t	__time_t;		/* time()... */
+typedef	__uint64_t	__uintfptr_t;
+typedef	__uint64_t	__uintmax_t;
+typedef	__uint64_t	__uintptr_t;
+typedef	__uint32_t	__uint_fast8_t;
+typedef	__uint32_t	__uint_fast16_t;
+typedef	__uint32_t	__uint_fast32_t;
+typedef	__uint64_t	__uint_fast64_t;
+typedef	__uint8_t	__uint_least8_t;
+typedef	__uint16_t	__uint_least16_t;
+typedef	__uint32_t	__uint_least32_t;
+typedef	__uint64_t	__uint_least64_t;
+typedef	__uint64_t	__u_register_t;
+typedef	__uint64_t	__vm_offset_t;
+typedef	__int64_t	__vm_ooffset_t;
+typedef	__uint64_t	__vm_paddr_t;
+typedef	__uint64_t	__vm_pindex_t;
+typedef	__uint64_t	__vm_size_t;
+
+/*
+ * Unusual type definitions.
+ */
+#ifdef __GNUCLIKE_BUILTIN_VARARGS
+typedef __builtin_va_list	__va_list;	/* internally known to gcc */
+#else
+typedef	char *			__va_list;
+#endif /* __GNUCLIKE_BUILTIN_VARARGS */
+#if defined(__GNUCLIKE_BUILTIN_VAALIST) && !defined(__GNUC_VA_LIST) \
+    && !defined(__NO_GNUC_VA_LIST)
+#define __GNUC_VA_LIST
+typedef __va_list		__gnuc_va_list;	/* compatibility w/GNU headers*/
+#endif
+
+typedef __uint64_t      tte_t;
+
+#endif /* !_MACHINE__TYPES_H_ */
diff --git a/sys/sun4v/include/asi.h b/sys/sun4v/include/asi.h
new file mode 100644
index 000000000000..799938beb71d
--- /dev/null
+++ b/sys/sun4v/include/asi.h
@@ -0,0 +1,155 @@
+/*-
+ * Copyright (c) 2006 Kip Macy
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	from: BSDI: asi.h,v 1.3 1997/08/08 14:31:42 torek
+ * $FreeBSD$
+ */
+
+#ifndef	_MACHINE_ASI_H_
+#define	_MACHINE_ASI_H_
+
+/*
+ *  UltraSPARC Architecture 2005 ASIs
+ */
+#define ASI_N                         0x04  /* ASI_NUCLEUS                           */
+
+#define ASI_NL                        0x0c  /* ASI_NUCLEUS_LITTLE                    */
+
+#define ASI_AIUP                      0x10  /* ASI_AS_IF_USER_PRIMARY                */
+#define ASI_AIUS                      0x11  /* ASI_AS_IF_USER_SECONDARY              */
+
+#define ASI_REAL                      0x14
+#define ASI_REAL_IO                   0x15
+#define ASI_BLK_AIUP                  0x16  /* ASI_BLOCK_AS_IF_USER_PRIMARY          */
+#define ASI_BLK_AIUS                  0x17  /* ASI_BLOCK_AS_IF_USER_SECONDARY        */
+#define ASI_AIUPL                     0x18  /* ASI_AS_IF_USER_PRIMARY_LITTLE         */
+#define ASI_AIUSL                     0x19  /* ASI_AS_IF_USER_SECONDARY_LITTLE       */
+
+#define ASI_REAL_L                    0x1C  /* ASI_REAL_LITTLE                       */
+#define ASI_REAL_IO_L                 0x1D  /* ASI_REAL_IO_LITTLE                    */
+#define ASI_BLK_AIUPL                 0x1E  /* ASI_BLOCK_AS_IF_USER_PRIMARY_LITTLE   */
+#define ASI_BLK_AIUSL                 0x1F  /* ASI_BLOCK_AS_IF_USER_SECONDARY_LITTLE */
+#define ASI_SCRATCHPAD                0x20
+#define ASI_MMU_CONTEXTID             0x21
+#define ASI_LDTD_AIUP                 0x22  /* ASI_LOAD_TWIN_DW_AS_IF_USER_PRIMARY   */
+#define ASI_LDSTBI_AIUP               0x22
+#define ASI_LDTD_AIUS                 0x23  /* ASI_LOAD_TWIN_DW_AS_IF_USER_SECONDARY */
+#define ASI_LDSTBI_AIUS               0x23
+#define ASI_QUEUE                     0x25
+#define ASI_LDTD_REAL                 0x26  /* ASI_LOAD_TWIN_DW_REAL                 */
+#define ASI_STBI_REAL                 0x26
+#define ASI_LDTD_N                    0x27  /* ASI_LOAD_TWIN_DW_NUCLEUS              */
+#define ASI_LDSTBI_N                  0x27
+
+#define ASI_LDTD_AIUPL                0x2A  /* ASI_LD_TWIN_DW_AS_IF_USER_PRIMARY_LITTLE   */
+#define ASI_LDTD_AIUSL                0x2B  /* ASI_LD_TWIN_DW_AS_IF_USER_SECONDARY_LITTLE */
+
+#define ASI_LDTD_REAL_L               0x2E  /* ASI_LOAD_TWIN_DW_REAL_LITTLE           */
+#define ASI_LDTD_NL                   0x2F  /* ASI_LOAD_TWIN_DW_NUCLEUS_LITTLE        */
+
+
+
+#define ASI_P                         0x80  /* ASI_PRIMARY                            */
+#define ASI_S                         0x81  /* ASI_SECONDARY                          */
+#define ASI_PNF                       0x82  /* ASI_PRIMARY_NO_FAULT                   */
+#define ASI_SNF                       0x83  /* ASI_SECONDARY_NO_FAULT                 */
+
+#define ASI_PL                        0x88  /* ASI_PRIMARY_LITTLE                     */
+#define ASI_SL                        0x89  /* ASI_SECONDARY_LITTLE                   */
+#define ASI_PNFL                      0x8a  /* ASI_PRIMARY_NO_FAULT_LITTLE            */
+#define ASI_SNFL                      0x8b  /* ASI_SECONDARY_NO_FAULT_LITTLE          */
+
+#define ASI_PST8_P                    0xc0
+#define ASI_PST8_S                    0xc1
+#define ASI_PST16_P                   0xc2
+#define ASI_PST16_S                   0xc3
+#define ASI_PST32_P                   0xc4
+#define ASI_PST32_S                   0xc5
+
+
+#define ASI_PST8_PL                   0xc8
+#define ASI_PST8_SL                   0xc9
+#define ASI_PST16_PL                  0xca
+#define ASI_PST16_SL                  0xcb
+#define ASI_PST32_PL                  0xcc
+#define ASI_PST32_SL                  0xcd
+
+#define ASI_FL8_P                     0xd0
+#define ASI_FL8_S                     0xd1
+#define ASI_FL16_P                    0xd2
+#define ASI_FL16_S                    0xd3
+
+#define ASI_FL8_PL                    0xd8
+#define ASI_FL8_SL                    0xd9
+#define ASI_FL16_PL                   0xda
+#define ASI_FL16_SL                   0xdb
+
+#define ASI_LDTD_P                    0xe2  /* ASI_LOAD_TWIN_DW_PRIMARY              */
+#define ASI_LDSTBI_P                  0xe2
+
+#define ASI_LDTD_S                    0xe3  /* ASI_LOAD_TWIN_DW_SECONDARY            */
+
+#define ASI_LDTD_PL                   0xea  /* ASI_LOAD_TWIN_DW_PRIMARY_LITTLE       */
+#define ASI_LDTD_SL                   0xeb  /* ASI_LOAD_TWIN_DW_SECONDARY_LITTLE     */
+
+#define ASI_BLK_P                     0xf0  /* ASI_BLOCK_PRIMARY                     */
+#define ASI_BLK_S                     0xf1  /* ASI_BLOCK_SECONDARY                   */
+
+#define ASI_BLK_PL                    0xf8  /* ASI_BLOCK_PRIMARY_LITTLE              */
+#define ASI_BLK_SL                    0xf9  /* ASI_BLOCK_SECONDARY_LITTLE            */
+
+
+
+#define ASI_SCRATCHPAD_0_REG          0x00
+#define ASI_SCRATCHPAD_1_REG          0x08
+#define ASI_SCRATCHPAD_2_REG          0x10
+#define ASI_SCRATCHPAD_3_REG          0x18
+#define ASI_SCRATCHPAD_6_REG          0x30
+#define ASI_SCRATCHPAD_7_REG          0x38
+
+
+#define SCRATCH_REG_MMFSA             ASI_SCRATCHPAD_0_REG
+#define SCRATCH_REG_PCPU              ASI_SCRATCHPAD_1_REG
+#define SCRATCH_REG_HASH_KERNEL       ASI_SCRATCHPAD_2_REG
+#define SCRATCH_REG_TSB_KERNEL        ASI_SCRATCHPAD_3_REG
+#define SCRATCH_REG_HASH_USER         ASI_SCRATCHPAD_6_REG
+#define SCRATCH_REG_TSB_USER          ASI_SCRATCHPAD_7_REG
+
+#define MMU_CID_P                     0x08
+#define MMU_CID_S                     0x10
+
+#define CPU_MONDO_QUEUE_HEAD          0x3c0
+#define CPU_MONDO_QUEUE_TAIL          0x3c8
+#define DEV_MONDO_QUEUE_HEAD          0x3d0
+#define DEV_MONDO_QUEUE_TAIL          0x3d8
+#define RESUMABLE_ERROR_QUEUE_HEAD    0x3e0
+#define RESUMABLE_ERROR_QUEUE_TAIL    0x3e8
+#define NONRESUMABLE_ERROR_QUEUE_HEAD 0x3f0
+#define NONRESUMABLE_ERROR_QUEUE_TAIL 0x3f8
+
+#define Q(queue_head) (queue_head >> 4)
+
+
+
+#endif /* !_MACHINE_ASI_H_ */
diff --git a/sys/sun4v/include/asm.h b/sys/sun4v/include/asm.h
new file mode 100644
index 000000000000..2510c1c185fa
--- /dev/null
+++ b/sys/sun4v/include/asm.h
@@ -0,0 +1,117 @@
+/*-
+ * Copyright (c) 1990 The Regents of the University of California.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to Berkeley by
+ * William Jolitz.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 4. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	from: @(#)DEFS.h	5.1 (Berkeley) 4/23/90
+ *	from: FreeBSD: src/sys/i386/include/asm.h,v 1.7 2000/01/25
+ * $FreeBSD$
+ */
+
+#ifndef _MACHINE_ASM_H_
+#define	_MACHINE_ASM_H_
+
+#define	__ASM__
+
+#include <sys/cdefs.h>
+
+#ifdef PIC
+#define	PIC_PROLOGUE(r1, r2) \
+	sethi	%hi(_GLOBAL_OFFSET_TABLE_-4), r1 ; \
+	rd	%pc, r2 ; \
+	or	r1, %lo(_GLOBAL_OFFSET_TABLE_+4), r1 ; \
+	add	r2, r1, r2
+#define	SET(name, r1, r2) \
+	set	name, r2 ; \
+	ldx	[r1 + r2], r2
+#else
+#define	PIC_PROLOGUE(r1, r2)
+#define	SET(name, r1, r2) \
+	set	name, r2
+#endif
+
+/*
+ * CNAME and HIDENAME manage the relationship between symbol names in C
+ * and the equivalent assembly language names.  CNAME is given a name as
+ * it would be used in a C program.  It expands to the equivalent assembly
+ * language name.  HIDENAME is given an assembly-language name, and expands
+ * to a possibly-modified form that will be invisible to C programs.
+ */
+#define CNAME(csym)		csym
+#define HIDENAME(asmsym)	__CONCAT(.,asmsym)
+
+#define	CCFSZ	192
+#define	SPOFF	2047
+
+#define	_ALIGN_TEXT	.align 32
+
+#define _START_ENTRY \
+	.text ; \
+	_ALIGN_TEXT
+
+/*
+ * Define a function entry point.
+ *
+ * The compiler produces #function for the .type pseudo-op, but the '#'
+ * character has special meaning in cpp macros, so we use @function like
+ * other architectures.  The assembler seems to accept both.
+ * The assembler also accepts a .proc pseudo-op, which is used by the
+ * peep hole optimizer, whose argument is the type code of the return
+ * value.  Since this is difficult to predict and its expected that
+ * assembler code is already optimized, we leave it out.
+ */
+#define	_ENTRY(x) \
+	_START_ENTRY ; \
+	.globl	CNAME(x) ; \
+	.type	CNAME(x),@function ; \
+CNAME(x):
+
+#define	ENTRY(x)	_ENTRY(x)
+#define	END(x)		.size x, . - x
+
+#define STACK_ALIGN     64
+#define SET_SIZE(x)     END(x)
+#define SA(X)           (((X)+(STACK_ALIGN-1)) & ~(STACK_ALIGN-1))  
+#define WINDOWSIZE64    (16*8)
+#define MINFRAME64      (WINDOWSIZE64 + 64)
+#define MINFRAME        MINFRAME64
+#define REGOFF          SA(MINFRAME)
+
+/*
+ * Kernel RCS ID tag and copyright macros
+ */
+
+#undef __FBSDID
+#if !defined(lint) && !defined(STRIP_FBSDID)
+#define __FBSDID(s)	.ident s
+#else
+#define __FBSDID(s)	/* nothing */
+#endif /* not lint and not STRIP_FBSDID */
+
+#endif /* !_MACHINE_ASM_H_ */
diff --git a/sys/sun4v/include/asmacros.h b/sys/sun4v/include/asmacros.h
new file mode 100644
index 000000000000..107aab00683d
--- /dev/null
+++ b/sys/sun4v/include/asmacros.h
@@ -0,0 +1,309 @@
+/*-
+ * Copyright (c) 2001 Jake Burkholder.
+ * Copyright (c) 2006 Kip Macy
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef	_MACHINE_ASMACROS_H_
+#define	_MACHINE_ASMACROS_H_
+
+#ifdef _KERNEL
+
+/*
+ *  %g7 points to per-cpu data.
+ */
+#define	PCPU_REG	%g7
+
+
+#ifdef LOCORE
+
+/*
+ * Atomically decrement an integer in memory.
+ */
+#define	ATOMIC_DEC_INT(r1, r2, r3) \
+	lduw	[r1], r2 ; \
+9:	sub	r2, 1, r3 ; \
+	casa	[r1] ASI_N, r2, r3 ; \
+	cmp	r2, r3 ; \
+	bne,pn	%icc, 9b ; \
+	 mov	r3, r2
+
+/*
+ * Atomically increment an integer in memory.
+ */
+#define	ATOMIC_INC_INT(r1, r2, r3) \
+	lduw	[r1], r2 ; \
+9:	add	r2, 1, r3 ; \
+	casa	[r1] ASI_N, r2, r3 ; \
+	cmp	r2, r3 ; \
+	bne,pn	%icc, 9b ; \
+	 mov	r3, r2
+
+/*
+ * Atomically increment an u_long in memory.
+ */
+#define	ATOMIC_INC_ULONG(r1, r2, r3) \
+	ldx	[r1], r2 ; \
+9:	add	r2, 1, r3 ; \
+	casxa	[r1] ASI_N, r2, r3 ; \
+	cmp	r2, r3 ; \
+	bne,pn	%icc, 9b ; \
+	 mov	r3, r2
+
+/*
+ * Atomically clear a number of bits of an integer in memory.
+ */
+#define	ATOMIC_CLEAR_INT(r1, r2, r3, bits) \
+	lduw	[r1], r2 ; \
+9:	andn	r2, bits, r3 ; \
+	casa	[r1] ASI_N, r2, r3 ; \
+	cmp	r2, r3 ; \
+	bne,pn	%icc, 9b ; \
+	 mov	r3, r2
+
+#define	PCPU(member)	PCPU_REG + PC_ ## member
+#define	PCPU_ADDR(member, reg) \
+	add	PCPU_REG, PC_ ## member, reg
+
+#define	DEBUGGER() \
+	ta	%xcc, 1
+
+#define	PANIC(msg, r1) \
+	.sect	.rodata ; \
+9:	.asciz	msg ; \
+	.previous ; \
+	SET(9b, r1, %o0) ; \
+	call	panic ; \
+	 nop
+
+#ifdef INVARIANTS
+#define	KASSERT(r1, msg) \
+	brnz	r1, 8f ; \
+	 nop ; \
+	PANIC(msg, r1) ; \
+8:
+#else
+#define	KASSERT(r1, msg)
+#endif
+
+#define	PUTS(msg, r1) \
+	.sect	.rodata ; \
+9:	.asciz	msg ; \
+	.previous ; \
+	SET(9b, r1, %o0) ; \
+	call	printf ; \
+	 nop
+
+#define	_ALIGN_DATA	.align 8
+
+#define	DATA(name) \
+	.data ; \
+	_ALIGN_DATA ; \
+	.globl	name ; \
+	.type	name, @object ; \
+name:
+
+#define	EMPTY
+
+#define GET_MMFSA_SCRATCH(reg)             \
+	ldxa [%g0 + %g0]ASI_SCRATCHPAD, reg;
+
+
+#define GET_PCPU_PHYS_SCRATCH(tmp)                      \
+        sethi %uhi(VM_MIN_DIRECT_ADDRESS), tmp;         \
+        mov  SCRATCH_REG_PCPU, PCPU_REG;                \
+        sllx tmp, 32, tmp;                              \
+        ldxa [%g0 + PCPU_REG]ASI_SCRATCHPAD, PCPU_REG;  \
+        andn PCPU_REG, tmp, PCPU_REG
+
+#define GET_PCPU_SCRATCH                    \
+	mov  SCRATCH_REG_PCPU, PCPU_REG;    \
+        ldxa [%g0 + PCPU_REG]ASI_SCRATCHPAD, PCPU_REG;
+
+#define GET_PCPU_SCRATCH_SLOW(reg)          \
+	mov  SCRATCH_REG_PCPU, reg;    \
+        ldxa [reg]ASI_SCRATCHPAD, PCPU_REG;
+
+#define GET_HASH_SCRATCH_USER(reg)	   \
+        mov SCRATCH_REG_HASH_USER, reg;    \
+	ldxa [%g0 + reg]ASI_SCRATCHPAD, reg;
+
+#define GET_HASH_SCRATCH_KERNEL(reg)   \
+        mov SCRATCH_REG_HASH_KERNEL, reg;  \
+	ldxa [%g0 + reg]ASI_SCRATCHPAD, reg; 
+
+#define GET_HASH_PHYS_SCRATCH_USER(tmp, reg)	\
+	sethi %uhi(VM_MIN_DIRECT_ADDRESS), tmp; \
+        mov SCRATCH_REG_HASH_USER, reg;         \
+	sllx tmp, 32, tmp;                      \
+	ldxa [%g0 + reg]ASI_SCRATCHPAD, reg;    \
+	andn reg, tmp, reg;
+
+#define GET_HASH_PHYS_SCRATCH_KERNEL(tmp, reg)    \
+	sethi %uhi(VM_MIN_DIRECT_ADDRESS), tmp;   \
+        mov SCRATCH_REG_HASH_KERNEL, reg;         \
+	sllx tmp, 32, tmp;                        \
+	ldxa [%g0 + reg]ASI_SCRATCHPAD, reg;      \
+	andn reg, tmp, reg;
+
+
+
+#define GET_TSB_SCRATCH_USER(reg)         \
+        mov SCRATCH_REG_TSB_USER, reg;    \
+	ldxa [%g0 + reg]ASI_SCRATCHPAD, reg;
+
+#define GET_TSB_SCRATCH_KERNEL(reg)	  \
+        mov SCRATCH_REG_TSB_KERNEL, reg;         \
+	ldxa [%g0 + reg]ASI_SCRATCHPAD, reg; 
+
+#define SET_SCRATCH(offsetreg, reg)   stxa reg, [%g0 + offsetreg]ASI_SCRATCHPAD
+
+
+#define GET_PCB_PHYS(tmp, reg)                    \
+        mov PC_CURPCB, reg;                       \
+        GET_PCPU_PHYS_SCRATCH(tmp);               \
+        ldxa [PCPU_REG + reg]ASI_REAL, reg;       \
+        sub reg, tmp, reg;
+
+
+#define GET_PCB(reg)	    \
+        GET_PCPU_SCRATCH;   \
+        ldx [PCPU_REG + PC_CURPCB], reg;
+
+#define SET_MMU_CONTEXT(typereg, reg)     stxa reg, [typereg]ASI_MMU_CONTEXTID
+#define GET_MMU_CONTEXT(typereg, reg)     ldxa [typereg]ASI_MMU_CONTEXTID, reg
+
+
+
+#define	SAVE_GLOBALS(TF) \
+	stx	%g1, [TF + TF_G1]; \
+	stx	%g2, [TF + TF_G2]; \
+	stx	%g3, [TF + TF_G3]; \
+	stx	%g4, [TF + TF_G4]; \
+	stx	%g5, [TF + TF_G5]; \
+	stx	%g6, [TF + TF_G6]; 
+
+#define	RESTORE_GLOBALS_USER(TF) \
+	ldx	[TF + TF_G1], %g1; \
+	ldx	[TF + TF_G2], %g2; \
+	ldx	[TF + TF_G3], %g3; \
+	ldx	[TF + TF_G4], %g4; \
+	ldx	[TF + TF_G5], %g5; \
+	ldx	[TF + TF_G6], %g6; \
+	ldx	[TF + TF_G7], %g7;
+
+#define	RESTORE_GLOBALS_KERNEL(TF) \
+	mov  SCRATCH_REG_PCPU, %g7; \
+	ldx	[TF + TF_G1], %g1; \
+	ldx	[TF + TF_G2], %g2; \
+	ldx	[TF + TF_G3], %g3; \
+	ldx	[TF + TF_G4], %g4; \
+	ldx	[TF + TF_G5], %g5; \
+	ldx	[TF + TF_G6], %g6; \
+	ldxa [%g0 + %g7]ASI_SCRATCHPAD, %g7;
+
+#define	SAVE_OUTS(TF) \
+	stx	%i0, [TF + TF_O0]; \
+	stx	%i1, [TF + TF_O1]; \
+	stx	%i2, [TF + TF_O2]; \
+	stx	%i3, [TF + TF_O3]; \
+	stx	%i4, [TF + TF_O4]; \
+	stx	%i5, [TF + TF_O5]; \
+	stx	%i6, [TF + TF_O6]; \
+	stx	%i7, [TF + TF_O7];
+
+#define	RESTORE_OUTS(TF) \
+	ldx	[TF + TF_O0], %i0; \
+	ldx	[TF + TF_O1], %i1; \
+	ldx	[TF + TF_O2], %i2; \
+	ldx	[TF + TF_O3], %i3; \
+	ldx	[TF + TF_O4], %i4; \
+	ldx	[TF + TF_O5], %i5; \
+	ldx	[TF + TF_O6], %i6; \
+	ldx	[TF + TF_O7], %i7;
+
+
+#define	SAVE_WINDOW(SBP) \
+	stx	%l0, [SBP + (0*8)]; \
+	stx	%l1, [SBP + (1*8)]; \
+	stx	%l2, [SBP + (2*8)]; \
+	stx	%l3, [SBP + (3*8)]; \
+	stx	%l4, [SBP + (4*8)]; \
+	stx	%l5, [SBP + (5*8)]; \
+	stx	%l6, [SBP + (6*8)]; \
+	stx	%l7, [SBP + (7*8)]; \
+	stx	%i0, [SBP + (8*8)]; \
+	stx	%i1, [SBP + (9*8)]; \
+	stx	%i2, [SBP + (10*8)]; \
+	stx	%i3, [SBP + (11*8)]; \
+	stx	%i4, [SBP + (12*8)]; \
+	stx	%i5, [SBP + (13*8)]; \
+	stx	%i6, [SBP + (14*8)]; \
+	stx	%i7, [SBP + (15*8)];
+
+#define	SAVE_WINDOW_ASI(SBP) \
+	stxa	%l0, [SBP + (0*8)]%asi; \
+	stxa	%l1, [SBP + (1*8)]%asi; \
+	stxa	%l2, [SBP + (2*8)]%asi; \
+	stxa	%l3, [SBP + (3*8)]%asi; \
+	stxa	%l4, [SBP + (4*8)]%asi; \
+	stxa	%l5, [SBP + (5*8)]%asi; \
+	stxa	%l6, [SBP + (6*8)]%asi; \
+	stxa	%l7, [SBP + (7*8)]%asi; \
+	stxa	%i0, [SBP + (8*8)]%asi; \
+	stxa	%i1, [SBP + (9*8)]%asi; \
+	stxa	%i2, [SBP + (10*8)]%asi; \
+	stxa	%i3, [SBP + (11*8)]%asi; \
+	stxa	%i4, [SBP + (12*8)]%asi; \
+	stxa	%i5, [SBP + (13*8)]%asi; \
+	stxa	%i6, [SBP + (14*8)]%asi; \
+	stxa	%i7, [SBP + (15*8)]%asi;
+
+#define	SAVE_LOCALS_ASI(SBP) \
+	stxa	%l0, [SBP + (0*8)]%asi; \
+	stxa	%l1, [SBP + (1*8)]%asi; \
+	stxa	%l2, [SBP + (2*8)]%asi; \
+	stxa	%l3, [SBP + (3*8)]%asi; \
+	stxa	%l4, [SBP + (4*8)]%asi; \
+	stxa	%l5, [SBP + (5*8)]%asi; \
+	stxa	%l6, [SBP + (6*8)]%asi; \
+	stxa	%l7, [SBP + (7*8)]%asi; 
+
+#define	RESTORE_LOCALS_ASI(SBP) \
+	ldxa	[SBP + (0*8)]%asi, %l0;	\
+	ldxa	[SBP + (1*8)]%asi, %l1;	\
+	ldxa	[SBP + (2*8)]%asi, %l2;	\
+	ldxa	[SBP + (3*8)]%asi, %l3;	\
+	ldxa	[SBP + (4*8)]%asi, %l4;	\
+	ldxa	[SBP + (5*8)]%asi, %l5;	\
+	ldxa	[SBP + (6*8)]%asi, %l6;	\
+	ldxa	[SBP + (7*8)]%asi, %l7;	
+
+#endif /* LOCORE */
+
+#endif /* _KERNEL */
+
+#endif /* !_MACHINE_ASMACROS_H_ */
diff --git a/sys/sun4v/include/atomic.h b/sys/sun4v/include/atomic.h
new file mode 100644
index 000000000000..77d455bfce5a
--- /dev/null
+++ b/sys/sun4v/include/atomic.h
@@ -0,0 +1,294 @@
+/*-
+ * Copyright (c) 1998 Doug Rabson.
+ * Copyright (c) 2001 Jake Burkholder.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	from: FreeBSD: src/sys/i386/include/atomic.h,v 1.20 2001/02/11
+ * $FreeBSD$
+ */
+
+#ifndef	_MACHINE_ATOMIC_H_
+#define	_MACHINE_ATOMIC_H_
+
+#include <machine/cpufunc.h>
+
+/* Userland needs different ASI's. */
+#ifdef _KERNEL
+#define	__ASI_ATOMIC	ASI_N
+#else
+#define	__ASI_ATOMIC	ASI_P
+#endif
+
+/*
+ * Various simple arithmetic on memory which is atomic in the presence
+ * of interrupts and multiple processors.  See atomic(9) for details.
+ * Note that efficient hardware support exists only for the 32 and 64
+ * bit variants; the 8 and 16 bit versions are not provided and should
+ * not be used in MI code.
+ *
+ * This implementation takes advantage of the fact that the sparc64
+ * cas instruction is both a load and a store.  The loop is often coded
+ * as follows:
+ *
+ *	do {
+ *		expect = *p;
+ *		new = expect + 1;
+ *	} while (cas(p, expect, new) != expect);
+ *
+ * which performs an unnnecessary load on each iteration that the cas
+ * operation fails.  Modified as follows:
+ *
+ *	expect = *p;
+ *	for (;;) {
+ *		new = expect + 1;
+ *		result = cas(p, expect, new);
+ *		if (result == expect)
+ *			break;
+ *		expect = result;
+ *	}
+ *
+ * the return value of cas is used to avoid the extra reload.
+ *
+ * The memory barriers provided by the acq and rel variants are intended
+ * to be sufficient for use of relaxed memory ordering.  Due to the
+ * suggested assembly syntax of the membar operands containing a #
+ * character, they cannot be used in macros.  The cmask and mmask bits
+ * are hard coded in machine/cpufunc.h and used here through macros.
+ * Hopefully sun will choose not to change the bit numbers.
+ */
+
+#define	itype(sz)	uint ## sz ## _t
+
+#define	atomic_cas_32(p, e, s)	casa(p, e, s, __ASI_ATOMIC)
+#define	atomic_cas_64(p, e, s)	casxa(p, e, s, __ASI_ATOMIC)
+
+#define	atomic_cas(p, e, s, sz)						\
+	atomic_cas_ ## sz(p, e, s)
+
+#define	atomic_cas_acq(p, e, s, sz) ({					\
+	itype(sz) v;							\
+	v = atomic_cas(p, e, s, sz);					\
+	membar(LoadLoad | LoadStore);					\
+	v;								\
+})
+
+#define	atomic_cas_rel(p, e, s, sz) ({					\
+	itype(sz) v;							\
+	membar(LoadStore | StoreStore);					\
+	v = atomic_cas(p, e, s, sz);					\
+	v;								\
+})
+
+#define	atomic_op(p, op, v, sz) ({					\
+	itype(sz) e, r, s;						\
+	for (e = *(volatile itype(sz) *)p;; e = r) {			\
+		s = e op v;						\
+		r = atomic_cas_ ## sz(p, e, s);				\
+		if (r == e)						\
+			break;						\
+	}								\
+	e;								\
+})
+
+#define	atomic_op_acq(p, op, v, sz) ({					\
+	itype(sz) t;							\
+	t = atomic_op(p, op, v, sz);					\
+	membar(LoadLoad | LoadStore);					\
+	t;								\
+})
+
+#define	atomic_op_rel(p, op, v, sz) ({					\
+	itype(sz) t;							\
+	membar(LoadStore | StoreStore);					\
+	t = atomic_op(p, op, v, sz);					\
+	t;								\
+})
+
+#define	atomic_load(p, sz)						\
+	atomic_cas(p, 0, 0, sz)
+
+#define	atomic_load_acq(p, sz) ({					\
+	itype(sz) v;							\
+	v = atomic_load(p, sz);						\
+	membar(LoadLoad | LoadStore);					\
+	v;								\
+})
+
+#define	atomic_load_clear(p, sz) ({					\
+	itype(sz) e, r;							\
+	for (e = *(volatile itype(sz) *)p;; e = r) {			\
+		r = atomic_cas(p, e, 0, sz);				\
+		if (r == e)						\
+			break;						\
+	}								\
+	e;								\
+})
+
+#define	atomic_store(p, v, sz) do {					\
+	itype(sz) e, r;							\
+	for (e = *(volatile itype(sz) *)p;; e = r) {			\
+		r = atomic_cas(p, e, v, sz);				\
+		if (r == e)						\
+			break;						\
+	}								\
+} while (0)
+
+#define	atomic_store_rel(p, v, sz) do {					\
+	membar(LoadStore | StoreStore);					\
+	atomic_store(p, v, sz);						\
+} while (0)
+
+#define	ATOMIC_GEN(name, ptype, vtype, atype, sz)			\
+									\
+static __inline vtype							\
+atomic_add_ ## name(volatile ptype p, atype v)				\
+{									\
+	return ((vtype)atomic_op(p, +, v, sz));				\
+}									\
+static __inline vtype							\
+atomic_add_acq_ ## name(volatile ptype p, atype v)			\
+{									\
+	return ((vtype)atomic_op_acq(p, +, v, sz));			\
+}									\
+static __inline vtype							\
+atomic_add_rel_ ## name(volatile ptype p, atype v)			\
+{									\
+	return ((vtype)atomic_op_rel(p, +, v, sz));			\
+}									\
+									\
+static __inline vtype							\
+atomic_clear_ ## name(volatile ptype p, atype v)			\
+{									\
+	return ((vtype)atomic_op(p, &, ~v, sz));			\
+}									\
+static __inline vtype							\
+atomic_clear_acq_ ## name(volatile ptype p, atype v)			\
+{									\
+	return ((vtype)atomic_op_acq(p, &, ~v, sz));			\
+}									\
+static __inline vtype							\
+atomic_clear_rel_ ## name(volatile ptype p, atype v)			\
+{									\
+	return ((vtype)atomic_op_rel(p, &, ~v, sz));			\
+}									\
+									\
+static __inline int							\
+atomic_cmpset_ ## name(volatile ptype p, vtype e, vtype s)		\
+{									\
+	return (((vtype)atomic_cas(p, e, s, sz)) == e);			\
+}									\
+static __inline int							\
+atomic_cmpset_acq_ ## name(volatile ptype p, vtype e, vtype s)		\
+{									\
+	return (((vtype)atomic_cas_acq(p, e, s, sz)) == e);		\
+}									\
+static __inline int							\
+atomic_cmpset_rel_ ## name(volatile ptype p, vtype e, vtype s)		\
+{									\
+	return (((vtype)atomic_cas_rel(p, e, s, sz)) == e);		\
+}									\
+									\
+static __inline vtype							\
+atomic_load_ ## name(volatile ptype p)					\
+{									\
+	return ((vtype)atomic_cas(p, 0, 0, sz));			\
+}									\
+static __inline vtype							\
+atomic_load_acq_ ## name(volatile ptype p)				\
+{									\
+	return ((vtype)atomic_cas_acq(p, 0, 0, sz));			\
+}									\
+									\
+static __inline vtype							\
+atomic_readandclear_ ## name(volatile ptype p)				\
+{									\
+	return ((vtype)atomic_load_clear(p, sz));			\
+}									\
+									\
+static __inline vtype							\
+atomic_set_ ## name(volatile ptype p, atype v)				\
+{									\
+	return ((vtype)atomic_op(p, |, v, sz));				\
+}									\
+static __inline vtype							\
+atomic_set_acq_ ## name(volatile ptype p, atype v)			\
+{									\
+	return ((vtype)atomic_op_acq(p, |, v, sz));			\
+}									\
+static __inline vtype							\
+atomic_set_rel_ ## name(volatile ptype p, atype v)			\
+{									\
+	return ((vtype)atomic_op_rel(p, |, v, sz));			\
+}									\
+									\
+static __inline vtype							\
+atomic_subtract_ ## name(volatile ptype p, atype v)			\
+{									\
+	return ((vtype)atomic_op(p, -, v, sz));				\
+}									\
+static __inline vtype							\
+atomic_subtract_acq_ ## name(volatile ptype p, atype v)			\
+{									\
+	return ((vtype)atomic_op_acq(p, -, v, sz));			\
+}									\
+static __inline vtype							\
+atomic_subtract_rel_ ## name(volatile ptype p, atype v)			\
+{									\
+	return ((vtype)atomic_op_rel(p, -, v, sz));			\
+}									\
+									\
+static __inline void							\
+atomic_store_ ## name(volatile ptype p, vtype v)			\
+{									\
+	atomic_store(p, v, sz);						\
+}									\
+static __inline void							\
+atomic_store_rel_ ## name(volatile ptype p, vtype v)			\
+{									\
+	atomic_store_rel(p, v, sz);					\
+}
+
+ATOMIC_GEN(int, u_int *, u_int, u_int, 32);
+ATOMIC_GEN(32, uint32_t *, uint32_t, uint32_t, 32);
+
+ATOMIC_GEN(long, u_long *, u_long, u_long, 64);
+ATOMIC_GEN(64, uint64_t *, uint64_t, uint64_t, 64);
+
+ATOMIC_GEN(ptr, uintptr_t *, uintptr_t, uintptr_t, 64);
+
+#define	atomic_fetchadd_int	atomic_add_int
+#define	atomic_fetchadd_32	atomic_add_32
+
+#undef ATOMIC_GEN
+#undef atomic_cas
+#undef atomic_cas_acq
+#undef atomic_cas_rel
+#undef atomic_op
+#undef atomic_op_acq
+#undef atomic_op_rel
+#undef atomic_load_acq
+#undef atomic_store_rel
+#undef atomic_load_clear
+
+#endif /* !_MACHINE_ATOMIC_H_ */
diff --git a/sys/sun4v/include/bus.h b/sys/sun4v/include/bus.h
new file mode 100644
index 000000000000..8a30082d28a3
--- /dev/null
+++ b/sys/sun4v/include/bus.h
@@ -0,0 +1,895 @@
+/*-
+ * Copyright (c) 1996, 1997, 1998, 2001 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The NetBSD Foundation
+ * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
+ * NASA Ames Research Center.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ *    must display the following acknowledgement:
+ *	This product includes software developed by the NetBSD
+ *	Foundation, Inc. and its contributors.
+ * 4. Neither the name of The NetBSD Foundation nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+/*
+ * Copyright (c) 1997-1999 Eduardo E. Horvath. All rights reserved.
+ * Copyright (c) 1996 Charles M. Hannum.  All rights reserved.
+ * Copyright (c) 1996 Christopher G. Demetriou.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ *    must display the following acknowledgement:
+ *      This product includes software developed by Christopher G. Demetriou
+ *	for the NetBSD Project.
+ * 4. The name of the author may not be used to endorse or promote products
+ *    derived from this software without specific prior written permission
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * 	from: NetBSD: bus.h,v 1.28 2001/07/19 15:32:19 thorpej Exp
+ *	and
+ *	from: FreeBSD: src/sys/alpha/include/bus.h,v 1.9 2001/01/09
+ *
+ * $FreeBSD$
+ */
+
+#ifndef	_MACHINE_BUS_H_
+#define	_MACHINE_BUS_H_
+
+#ifdef BUS_SPACE_DEBUG
+#include <sys/ktr.h>
+#endif
+
+#include <machine/_bus.h>
+#include <machine/cpufunc.h>
+#include <machine/upa.h>
+
+/*
+ * UPA and SBUS spaces are non-cached and big endian
+ * (except for RAM and PROM)
+ *
+ * PCI spaces are non-cached and little endian
+ */
+#define	UPA_BUS_SPACE		0
+#define	SBUS_BUS_SPACE		1
+#define	PCI_CONFIG_BUS_SPACE	2
+#define	PCI_IO_BUS_SPACE	3
+#define	PCI_MEMORY_BUS_SPACE	4
+#define	LAST_BUS_SPACE		5
+
+extern int bus_type_asi[];
+extern int bus_stream_asi[];
+
+#define __BUS_SPACE_HAS_STREAM_METHODS	1
+
+#define BUS_SPACE_MAXSIZE_24BIT	0xFFFFFF
+#define BUS_SPACE_MAXSIZE_32BIT 0xFFFFFFFF
+#define BUS_SPACE_MAXSIZE	0xFFFFFFFFFFFFFFFF
+#define BUS_SPACE_MAXADDR_24BIT	0xFFFFFF
+#define BUS_SPACE_MAXADDR_32BIT 0xFFFFFFFF
+#define BUS_SPACE_MAXADDR	0xFFFFFFFF
+
+#define BUS_SPACE_UNRESTRICTED	(~0)
+
+/*
+ * Access methods for bus resources and address space.
+ */
+struct bus_space_tag {
+	void		*bst_cookie;
+	bus_space_tag_t	bst_parent;
+	int		bst_type;
+
+	void		(*bst_bus_barrier)(bus_space_tag_t, bus_space_handle_t,
+	    bus_size_t, bus_size_t, int);
+};
+
+/*
+ * Bus space function prototypes.
+ */
+static void bus_space_barrier(bus_space_tag_t, bus_space_handle_t, bus_size_t,
+    bus_size_t, int);
+static int bus_space_subregion(bus_space_tag_t, bus_space_handle_t,
+    bus_size_t, bus_size_t, bus_space_handle_t *);
+
+/*
+ * Map a region of device bus space into CPU virtual address space.
+ */
+
+static __inline int bus_space_map(bus_space_tag_t t, bus_addr_t addr,
+				  bus_size_t size, int flags,
+				  bus_space_handle_t *bshp);
+
+static __inline int
+bus_space_map(bus_space_tag_t t __unused, bus_addr_t addr,
+	      bus_size_t size __unused, int flags __unused,
+	      bus_space_handle_t *bshp)
+{
+
+	*bshp = addr;
+	return (0);
+}
+
+/*
+ * Unmap a region of device bus space.
+ */
+static __inline void bus_space_unmap(bus_space_tag_t t, bus_space_handle_t bsh,
+				     bus_size_t size);
+
+static __inline void
+bus_space_unmap(bus_space_tag_t t __unused, bus_space_handle_t bsh __unused,
+		bus_size_t size __unused)
+{
+}
+
+/* This macro finds the first "upstream" implementation of method `f' */
+#define _BS_CALL(t,f)							\
+	while (t->f == NULL)						\
+		t = t->bst_parent;						\
+	return (*(t)->f)
+
+static __inline void
+bus_space_barrier(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o,
+    bus_size_t s, int f)
+{
+	_BS_CALL(t, bst_bus_barrier)(t, h, o, s, f);
+}
+
+static __inline int
+bus_space_subregion(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o,
+    bus_size_t s, bus_space_handle_t *hp)
+{
+	*hp = h + o;
+	return (0);
+}
+
+/* flags for bus space map functions */
+#define BUS_SPACE_MAP_CACHEABLE		0x0001
+#define BUS_SPACE_MAP_LINEAR		0x0002
+#define BUS_SPACE_MAP_READONLY		0x0004
+#define BUS_SPACE_MAP_PREFETCHABLE	0x0008
+/* placeholders for bus functions... */
+#define BUS_SPACE_MAP_BUS1		0x0100
+#define BUS_SPACE_MAP_BUS2		0x0200
+#define BUS_SPACE_MAP_BUS3		0x0400
+#define BUS_SPACE_MAP_BUS4		0x0800
+
+/* flags for bus_space_barrier() */
+#define	BUS_SPACE_BARRIER_READ		0x01	/* force read barrier */
+#define	BUS_SPACE_BARRIER_WRITE		0x02	/* force write barrier */
+
+#ifdef BUS_SPACE_DEBUG
+#define	KTR_BUS				KTR_CT2
+#define	BUS_HANDLE_MIN			UPA_MEMSTART
+#define	__BUS_DEBUG_ACCESS(h, o, desc, sz) do {				\
+	CTR4(KTR_BUS, "bus space: %s %d: handle %#lx, offset %#lx",	\
+	    (desc), (sz), (h), (o));					\
+	if ((h) + (o) < BUS_HANDLE_MIN)					\
+		panic("bus space access at %#lx out of range",		\
+		    (h) + (o));						\
+} while (0)
+#else
+#define	__BUS_DEBUG_ACCESS(h, o, desc, sz)
+#endif
+
+static __inline uint8_t
+bus_space_read_1(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o)
+{
+
+	__BUS_DEBUG_ACCESS(h, o, "read", 1);
+	return (lduba_nc((caddr_t)(h + o), bus_type_asi[t->bst_type]));
+}
+
+static __inline uint16_t
+bus_space_read_2(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o)
+{
+
+	__BUS_DEBUG_ACCESS(h, o, "read", 2);
+	return (lduha_nc((caddr_t)(h + o), bus_type_asi[t->bst_type]));
+}
+
+static __inline uint32_t
+bus_space_read_4(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o)
+{
+
+	__BUS_DEBUG_ACCESS(h, o, "read", 4);
+	return (lduwa_nc((caddr_t)(h + o), bus_type_asi[t->bst_type]));
+}
+
+static __inline uint64_t
+bus_space_read_8(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o)
+{
+
+	__BUS_DEBUG_ACCESS(h, o, "read", 8);
+	return (ldxa_nc((caddr_t)(h + o), bus_type_asi[t->bst_type]));
+}
+
+static __inline void
+bus_space_read_multi_1(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o,
+    uint8_t *a, size_t c)
+{
+
+	while (c-- > 0)
+		*a++ = bus_space_read_1(t, h, o);
+}
+
+static __inline void
+bus_space_read_multi_2(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o,
+    uint16_t *a, size_t c)
+{
+
+	while (c-- > 0)
+		*a++ = bus_space_read_2(t, h, o);
+}
+
+static __inline void
+bus_space_read_multi_4(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o,
+    uint32_t *a, size_t c)
+{
+
+	while (c-- > 0)
+		*a++ = bus_space_read_4(t, h, o);
+}
+
+static __inline void
+bus_space_read_multi_8(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o,
+    uint64_t *a, size_t c)
+{
+
+	while (c-- > 0)
+		*a++ = bus_space_read_8(t, h, o);
+}
+
+static __inline void
+bus_space_write_1(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o,
+    uint8_t v)
+{
+
+	__BUS_DEBUG_ACCESS(h, o, "write", 1);
+	stba_nc((caddr_t)(h + o), bus_type_asi[t->bst_type], v);
+}
+
+static __inline void
+bus_space_write_2(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o,
+    uint16_t v)
+{
+
+	__BUS_DEBUG_ACCESS(h, o, "write", 2);
+	stha_nc((caddr_t)(h + o), bus_type_asi[t->bst_type], v);
+}
+
+static __inline void
+bus_space_write_4(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o,
+    uint32_t v)
+{
+
+	__BUS_DEBUG_ACCESS(h, o, "write", 4);
+	stwa_nc((caddr_t)(h + o), bus_type_asi[t->bst_type], v);
+}
+
+static __inline void
+bus_space_write_8(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o,
+    uint64_t v)
+{
+
+	__BUS_DEBUG_ACCESS(h, o, "write", 8);
+	stxa_nc((caddr_t)(h + o), bus_type_asi[t->bst_type], v);
+}
+
+static __inline void
+bus_space_write_multi_1(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o,
+    uint8_t *a, size_t c)
+{
+
+	while (c-- > 0)
+		bus_space_write_1(t, h, o, *a++);
+}
+
+static __inline void
+bus_space_write_multi_2(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o,
+    uint16_t *a, size_t c)
+{
+
+	while (c-- > 0)
+		bus_space_write_2(t, h, o, *a++);
+}
+
+static __inline void
+bus_space_write_multi_4(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o,
+    uint32_t *a, size_t c)
+{
+
+	while (c-- > 0)
+		bus_space_write_4(t, h, o, *a++);
+}
+
+static __inline void
+bus_space_write_multi_8(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o,
+    uint64_t *a, size_t c)
+{
+
+	while (c-- > 0)
+		bus_space_write_8(t, h, o, *a++);
+}
+
+static __inline void
+bus_space_set_multi_1(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o,
+    uint8_t v, size_t c)
+{
+
+	while (c-- > 0)
+		bus_space_write_1(t, h, o, v);
+}
+
+static __inline void
+bus_space_set_multi_2(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o,
+    uint16_t v, size_t c)
+{
+
+	while (c-- > 0)
+		bus_space_write_2(t, h, o, v);
+}
+
+static __inline void
+bus_space_set_multi_4(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o,
+    uint32_t v, size_t c)
+{
+
+	while (c-- > 0)
+		bus_space_write_4(t, h, o, v);
+}
+
+static __inline void
+bus_space_set_multi_8(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o,
+    uint64_t v, size_t c)
+{
+
+	while (c-- > 0)
+		bus_space_write_8(t, h, o, v);
+}
+
+static __inline void
+bus_space_read_region_1(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o,
+    u_int8_t *a, bus_size_t c)
+{
+	for (; c; a++, c--, o++)
+		*a = bus_space_read_1(t, h, o);
+}
+
+static __inline void
+bus_space_read_region_2(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o,
+    u_int16_t *a, bus_size_t c)
+{
+	for (; c; a++, c--, o+=2)
+		*a = bus_space_read_2(t, h, o);
+}
+
+static __inline void
+bus_space_read_region_4(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o,
+    u_int32_t *a, bus_size_t c)
+{
+	for (; c; a++, c--, o+=4)
+		*a = bus_space_read_4(t, h, o);
+}
+
+static __inline void
+bus_space_read_region_8(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o,
+    u_int64_t *a, bus_size_t c)
+{
+	for (; c; a++, c--, o+=8)
+		*a = bus_space_read_8(t, h, o);
+}
+
+static __inline void
+bus_space_write_region_1(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o,
+    const u_int8_t *a, bus_size_t c)
+{
+	for (; c; a++, c--, o++)
+		bus_space_write_1(t, h, o, *a);
+}
+
+static __inline void
+bus_space_write_region_2(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o,
+    const u_int16_t *a, bus_size_t c)
+{
+	for (; c; a++, c--, o+=2)
+		bus_space_write_2(t, h, o, *a);
+}
+
+static __inline void
+bus_space_write_region_4(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o,
+    const u_int32_t *a, bus_size_t c)
+{
+	for (; c; a++, c--, o+=4)
+		bus_space_write_4(t, h, o, *a);
+}
+
+static __inline void
+bus_space_write_region_8(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o,
+    const u_int64_t *a, bus_size_t c)
+{
+	for (; c; a++, c--, o+=8)
+		bus_space_write_8(t, h, o, *a);
+}
+
+static __inline void
+bus_space_set_region_1(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o,
+    const u_int8_t v, bus_size_t c)
+{
+	for (; c; c--, o++)
+		bus_space_write_1(t, h, o, v);
+}
+
+static __inline void
+bus_space_set_region_2(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o,
+    const u_int16_t v, bus_size_t c)
+{
+	for (; c; c--, o+=2)
+		bus_space_write_2(t, h, o, v);
+}
+
+static __inline void
+bus_space_set_region_4(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o,
+    const u_int32_t v, bus_size_t c)
+{
+	for (; c; c--, o+=4)
+		bus_space_write_4(t, h, o, v);
+}
+
+static __inline void
+bus_space_set_region_8(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o,
+    const u_int64_t v, bus_size_t c)
+{
+	for (; c; c--, o+=8)
+		bus_space_write_8(t, h, o, v);
+}
+
+static __inline void
+bus_space_copy_region_1(bus_space_tag_t t, bus_space_handle_t h1,
+    bus_size_t o1, bus_space_handle_t h2, bus_size_t o2, bus_size_t c)
+{
+	for (; c; c--, o1++, o2++)
+	    bus_space_write_1(t, h1, o1, bus_space_read_1(t, h2, o2));
+}
+
+static __inline void
+bus_space_copy_region_2(bus_space_tag_t t, bus_space_handle_t h1,
+    bus_size_t o1, bus_space_handle_t h2, bus_size_t o2, bus_size_t c)
+{
+	for (; c; c--, o1+=2, o2+=2)
+	    bus_space_write_2(t, h1, o1, bus_space_read_2(t, h2, o2));
+}
+
+static __inline void
+bus_space_copy_region_4(bus_space_tag_t t, bus_space_handle_t h1,
+    bus_size_t o1, bus_space_handle_t h2, bus_size_t o2, bus_size_t c)
+{
+	for (; c; c--, o1+=4, o2+=4)
+	    bus_space_write_4(t, h1, o1, bus_space_read_4(t, h2, o2));
+}
+
+static __inline void
+bus_space_copy_region_8(bus_space_tag_t t, bus_space_handle_t h1,
+    bus_size_t o1, bus_space_handle_t h2, bus_size_t o2, bus_size_t c)
+{
+	for (; c; c--, o1+=8, o2+=8)
+	    bus_space_write_8(t, h1, o1, bus_space_read_8(t, h2, o2));
+}
+
+static __inline uint8_t
+bus_space_read_stream_1(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o)
+{
+
+	__BUS_DEBUG_ACCESS(h, o, "read stream", 1);
+	return (lduba_nc((caddr_t)(h + o), bus_stream_asi[t->bst_type]));
+}
+
+static __inline uint16_t
+bus_space_read_stream_2(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o)
+{
+
+	__BUS_DEBUG_ACCESS(h, o, "read stream", 2);
+	return (lduha_nc((caddr_t)(h + o), bus_stream_asi[t->bst_type]));
+}
+
+static __inline uint32_t
+bus_space_read_stream_4(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o)
+{
+
+	__BUS_DEBUG_ACCESS(h, o, "read stream", 4);
+	return (lduwa_nc((caddr_t)(h + o), bus_stream_asi[t->bst_type]));
+}
+
+static __inline uint64_t
+bus_space_read_stream_8(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o)
+{
+
+	__BUS_DEBUG_ACCESS(h, o, "read stream", 8);
+	return (ldxa_nc((caddr_t)(h + o), bus_stream_asi[t->bst_type]));
+}
+
+static __inline void
+bus_space_read_multi_stream_1(bus_space_tag_t t, bus_space_handle_t h,
+    bus_size_t o, uint8_t *a, size_t c)
+{
+
+	while (c-- > 0)
+		*a++ = bus_space_read_stream_1(t, h, o);
+}
+
+static __inline void
+bus_space_read_multi_stream_2(bus_space_tag_t t, bus_space_handle_t h,
+    bus_size_t o, uint16_t *a, size_t c)
+{
+
+	while (c-- > 0)
+		*a++ = bus_space_read_stream_2(t, h, o);
+}
+
+static __inline void
+bus_space_read_multi_stream_4(bus_space_tag_t t, bus_space_handle_t h,
+    bus_size_t o, uint32_t *a, size_t c)
+{
+
+	while (c-- > 0)
+		*a++ = bus_space_read_stream_4(t, h, o);
+}
+
+static __inline void
+bus_space_read_multi_stream_8(bus_space_tag_t t, bus_space_handle_t h,
+    bus_size_t o, uint64_t *a, size_t c)
+{
+
+	while (c-- > 0)
+		*a++ = bus_space_read_stream_8(t, h, o);
+}
+
+static __inline void
+bus_space_write_stream_1(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o,
+    uint8_t v)
+{
+
+	__BUS_DEBUG_ACCESS(h, o, "write stream", 1);
+	stba_nc((caddr_t)(h + o), bus_stream_asi[t->bst_type], v);
+}
+
+static __inline void
+bus_space_write_stream_2(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o,
+    uint16_t v)
+{
+
+	__BUS_DEBUG_ACCESS(h, o, "write stream", 2);
+	stha_nc((caddr_t)(h + o), bus_stream_asi[t->bst_type], v);
+}
+
+static __inline void
+bus_space_write_stream_4(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o,
+    uint32_t v)
+{
+
+	__BUS_DEBUG_ACCESS(h, o, "write stream", 4);
+	stwa_nc((caddr_t)(h + o), bus_stream_asi[t->bst_type], v);
+}
+
+static __inline void
+bus_space_write_stream_8(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o,
+    uint64_t v)
+{
+
+	__BUS_DEBUG_ACCESS(h, o, "write stream", 8);
+	stxa_nc((caddr_t)(h + o), bus_stream_asi[t->bst_type], v);
+}
+
+static __inline void
+bus_space_write_multi_stream_1(bus_space_tag_t t, bus_space_handle_t h,
+    bus_size_t o, const uint8_t *a, size_t c)
+{
+
+	while (c-- > 0)
+		bus_space_write_stream_1(t, h, o, *a++);
+}
+
+static __inline void
+bus_space_write_multi_stream_2(bus_space_tag_t t, bus_space_handle_t h,
+    bus_size_t o, const uint16_t *a, size_t c)
+{
+
+	while (c-- > 0)
+		bus_space_write_stream_2(t, h, o, *a++);
+}
+
+static __inline void
+bus_space_write_multi_stream_4(bus_space_tag_t t, bus_space_handle_t h,
+    bus_size_t o, const uint32_t *a, size_t c)
+{
+
+	while (c-- > 0)
+		bus_space_write_stream_4(t, h, o, *a++);
+}
+
+static __inline void
+bus_space_write_multi_stream_8(bus_space_tag_t t, bus_space_handle_t h,
+    bus_size_t o, const uint64_t *a, size_t c)
+{
+
+	while (c-- > 0)
+		bus_space_write_stream_8(t, h, o, *a++);
+}
+
+static __inline void
+bus_space_set_multi_stream_1(bus_space_tag_t t, bus_space_handle_t h,
+    bus_size_t o, uint8_t v, size_t c)
+{
+
+	while (c-- > 0)
+		bus_space_write_stream_1(t, h, o, v);
+}
+
+static __inline void
+bus_space_set_multi_stream_2(bus_space_tag_t t, bus_space_handle_t h,
+    bus_size_t o, uint16_t v, size_t c)
+{
+
+	while (c-- > 0)
+		bus_space_write_stream_2(t, h, o, v);
+}
+
+static __inline void
+bus_space_set_multi_stream_4(bus_space_tag_t t, bus_space_handle_t h,
+    bus_size_t o, uint32_t v, size_t c)
+{
+
+	while (c-- > 0)
+		bus_space_write_stream_4(t, h, o, v);
+}
+
+static __inline void
+bus_space_set_multi_stream_8(bus_space_tag_t t, bus_space_handle_t h,
+    bus_size_t o, uint64_t v, size_t c)
+{
+
+	while (c-- > 0)
+		bus_space_write_stream_8(t, h, o, v);
+}
+
+static __inline void
+bus_space_read_region_stream_1(bus_space_tag_t t, bus_space_handle_t h,
+    bus_size_t o, u_int8_t *a, bus_size_t c)
+{
+
+	for (; c; a++, c--, o++)
+		*a = bus_space_read_stream_1(t, h, o);
+}
+
+static __inline void
+bus_space_read_region_stream_2(bus_space_tag_t t, bus_space_handle_t h,
+    bus_size_t o, u_int16_t *a, bus_size_t c)
+{
+
+	for (; c; a++, c--, o+=2)
+		*a = bus_space_read_stream_2(t, h, o);
+}
+
+static __inline void
+bus_space_read_region_stream_4(bus_space_tag_t t, bus_space_handle_t h,
+    bus_size_t o, u_int32_t *a, bus_size_t c)
+{
+
+	for (; c; a++, c--, o+=4)
+		*a = bus_space_read_stream_4(t, h, o);
+}
+
+static __inline void
+bus_space_read_region_stream_8(bus_space_tag_t t, bus_space_handle_t h,
+    bus_size_t o, u_int64_t *a, bus_size_t c)
+{
+
+	for (; c; a++, c--, o+=8)
+		*a = bus_space_read_stream_8(t, h, o);
+}
+
+static __inline void
+bus_space_write_region_stream_1(bus_space_tag_t t, bus_space_handle_t h,
+    bus_size_t o, const u_int8_t *a, bus_size_t c)
+{
+
+	for (; c; a++, c--, o++)
+		bus_space_write_stream_1(t, h, o, *a);
+}
+
+static __inline void
+bus_space_write_region_stream_2(bus_space_tag_t t, bus_space_handle_t h,
+    bus_size_t o, const u_int16_t *a, bus_size_t c)
+{
+
+	for (; c; a++, c--, o+=2)
+		bus_space_write_stream_2(t, h, o, *a);
+}
+
+static __inline void
+bus_space_write_region_stream_4(bus_space_tag_t t, bus_space_handle_t h,
+    bus_size_t o, const u_int32_t *a, bus_size_t c)
+{
+
+	for (; c; a++, c--, o+=4)
+		bus_space_write_stream_4(t, h, o, *a);
+}
+
+static __inline void
+bus_space_write_region_stream_8(bus_space_tag_t t, bus_space_handle_t h,
+    bus_size_t o, const u_int64_t *a, bus_size_t c)
+{
+
+	for (; c; a++, c--, o+=8)
+		bus_space_write_stream_8(t, h, o, *a);
+}
+
+static __inline void
+bus_space_set_region_stream_1(bus_space_tag_t t, bus_space_handle_t h,
+    bus_size_t o, const u_int8_t v, bus_size_t c)
+{
+
+	for (; c; c--, o++)
+		bus_space_write_stream_1(t, h, o, v);
+}
+
+static __inline void
+bus_space_set_region_stream_2(bus_space_tag_t t, bus_space_handle_t h,
+    bus_size_t o, const u_int16_t v, bus_size_t c)
+{
+
+	for (; c; c--, o+=2)
+		bus_space_write_stream_2(t, h, o, v);
+}
+
+static __inline void
+bus_space_set_region_stream_4(bus_space_tag_t t, bus_space_handle_t h,
+    bus_size_t o, const u_int32_t v, bus_size_t c)
+{
+
+	for (; c; c--, o+=4)
+		bus_space_write_stream_4(t, h, o, v);
+}
+
+static __inline void
+bus_space_set_region_stream_8(bus_space_tag_t t, bus_space_handle_t h,
+    bus_size_t o, const u_int64_t v, bus_size_t c)
+{
+
+	for (; c; c--, o+=8)
+		bus_space_write_stream_8(t, h, o, v);
+}
+
+static __inline void
+bus_space_copy_region_stream_1(bus_space_tag_t t, bus_space_handle_t h1,
+    bus_size_t o1, bus_space_handle_t h2, bus_size_t o2, bus_size_t c)
+{
+
+	for (; c; c--, o1++, o2++)
+	    bus_space_write_stream_1(t, h1, o1, bus_space_read_stream_1(t, h2,
+		o2));
+}
+
+static __inline void
+bus_space_copy_region_stream_2(bus_space_tag_t t, bus_space_handle_t h1,
+    bus_size_t o1, bus_space_handle_t h2, bus_size_t o2, bus_size_t c)
+{
+
+	for (; c; c--, o1+=2, o2+=2)
+	    bus_space_write_stream_2(t, h1, o1, bus_space_read_stream_2(t, h2,
+		o2));
+}
+
+static __inline void
+bus_space_copy_region_stream_4(bus_space_tag_t t, bus_space_handle_t h1,
+    bus_size_t o1, bus_space_handle_t h2, bus_size_t o2, bus_size_t c)
+{
+
+	for (; c; c--, o1+=4, o2+=4)
+	    bus_space_write_stream_4(t, h1, o1, bus_space_read_stream_4(t, h2,
+		o2));
+}
+
+static __inline void
+bus_space_copy_region_stream_8(bus_space_tag_t t, bus_space_handle_t h1,
+    bus_size_t o1, bus_space_handle_t h2, bus_size_t o2, bus_size_t c)
+{
+
+	for (; c; c--, o1+=8, o2+=8)
+	    bus_space_write_stream_8(t, h1, o1, bus_space_read_8(t, h2, o2));
+}
+
+/* Back-compat functions for old ISA drivers */
+extern bus_space_tag_t isa_io_bt;
+extern bus_space_handle_t isa_io_hdl;
+extern bus_space_tag_t isa_mem_bt;
+extern bus_space_handle_t isa_mem_hdl;
+
+#define inb(o)		bus_space_read_1(isa_io_bt, isa_io_hdl, o)
+#define inw(o)		bus_space_read_2(isa_io_bt, isa_io_hdl, o)
+#define inl(o)		bus_space_read_4(isa_io_bt, isa_io_hdl, o)
+#define outb(o, v)	bus_space_write_1(isa_io_bt, isa_io_hdl, o, v)
+#define outw(o, v)	bus_space_write_2(isa_io_bt, isa_io_hdl, o, v)
+#define outl(o, v)	bus_space_write_4(isa_io_bt, isa_io_hdl, o, v)
+
+#define readb(o)	bus_space_read_1(isa_mem_bt, isa_mem_hdl, o)
+#define readw(o)	bus_space_read_2(isa_mem_bt, isa_mem_hdl, o)
+#define readl(o)	bus_space_read_4(isa_mem_bt, isa_mem_hdl, o)
+#define writeb(o, v)	bus_space_write_1(isa_mem_bt, isa_mem_hdl, o, v)
+#define writew(o, v)	bus_space_write_2(isa_mem_bt, isa_mem_hdl, o, v)
+#define writel(o, v)	bus_space_write_4(isa_mem_bt, isa_mem_hdl, o, v)
+
+#define insb(o, a, c) \
+	bus_space_read_multi_1(isa_io_bt, isa_io_hdl, o, (void*)a, c)
+#define insw(o, a, c) \
+	bus_space_read_multi_2(isa_io_bt, isa_io_hdl, o, (void*)a, c)
+#define insl(o, a, c) \
+	bus_space_read_multi_4(isa_io_bt, isa_io_hdl, o, (void*)a, c)
+#define outsb(o, a, c) \
+	bus_space_write_multi_1(isa_io_bt, isa_io_hdl, o, (void*)a, c)
+#define outsw(o, a, c) \
+	bus_space_write_multi_2(isa_io_bt, isa_io_hdl, o, (void*)a, c)
+#define outsl(o, a, c) \
+	bus_space_write_multi_4(isa_io_bt, isa_io_hdl, o, (void*)a, c)
+
+#define memcpy_fromio(d, s, c) \
+	bus_space_read_region_1(isa_mem_bt, isa_mem_hdl, s, d, c)
+#define memcpy_toio(d, s, c) \
+	bus_space_write_region_1(isa_mem_bt, isa_mem_hdl, d, s, c)
+#define memcpy_io(d, s, c) \
+	bus_space_copy_region_1(isa_mem_bt, isa_mem_hdl, s, isa_mem_hdl, d, c)
+#define memset_io(d, v, c) \
+	bus_space_set_region_1(isa_mem_bt, isa_mem_hdl, d, v, c)
+#define memsetw_io(d, v, c) \
+	bus_space_set_region_2(isa_mem_bt, isa_mem_hdl, d, v, c)
+
+static __inline void
+memsetw(void *d, int val, size_t size)
+{
+    u_int16_t *sp = d;
+
+    while (size--)
+	*sp++ = val;
+}
+
+#include <machine/bus_dma.h>
+
+#endif /* !_MACHINE_BUS_H_ */
diff --git a/sys/sun4v/include/bus_common.h b/sys/sun4v/include/bus_common.h
new file mode 100644
index 000000000000..216f592c8d5c
--- /dev/null
+++ b/sys/sun4v/include/bus_common.h
@@ -0,0 +1,65 @@
+/*-
+ * Copyright (c) 1992, 1993
+ *	The Regents of the University of California.  All rights reserved.
+ *
+ * This software was developed by the Computer Systems Engineering group
+ * at Lawrence Berkeley Laboratory under DARPA contract BG 91-66 and
+ * contributed to Berkeley.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 4. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	form: @(#)sbusreg.h	8.1 (Berkeley) 6/11/93
+ *	from: NetBSD: iommureg.h,v 1.6 2001/07/20 00:07:13 eeh Exp
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _MACHINE_BUS_COMMON_H_
+#define _MACHINE_BUS_COMMON_H_
+
+#define INTMAP_V		0x080000000LL	/* Interrupt valid (enabled) */
+#define INTMAP_TID_MASK		0x07c000000LL	/* UPA target ID */
+#define INTMAP_TID_SHIFT	26
+#define INTMAP_IGN_MASK		0x0000007c0LL	/* Interrupt group no. */
+#define INTMAP_IGN_SHIFT	6
+#define INTMAP_INO_MASK		0x00000003fLL	/* Interrupt number */
+#define INTMAP_INR_MASK		(INTMAP_IGN_MASK | INTMAP_INO_MASK)
+#define INTMAP_SBUSSLOT_MASK	0x000000018LL	/* SBUS slot # */
+#define INTMAP_PCIBUS_MASK	0x000000010LL	/* PCI bus number (A or B) */
+#define INTMAP_PCISLOT_MASK	0x00000000cLL	/* PCI slot # */
+#define INTMAP_PCIINT_MASK	0x000000003LL	/* PCI interrupt #A,#B,#C,#D */
+#define INTMAP_OBIO_MASK	0x000000020LL	/* Onboard device */
+#define INTVEC(x)		((x) & INTMAP_INR_MASK)
+#define INTSLOT(x)		(((x) >> 3) & 0x7)
+#define	INTPRI(x)		((x) & 0x7)
+#define	INTINO(x)		((x) & INTMAP_INO_MASK)
+#define	INTMAP_ENABLE(mr, mid) \
+	(((mr) & ~INTMAP_TID_MASK) | ((mid) << INTMAP_TID_SHIFT) | INTMAP_V)
+
+/* counter-timer support. */
+void sparc64_counter_init(bus_space_tag_t tag, bus_space_handle_t handle,
+    bus_addr_t offset);
+
+#endif	/* !_MACHINE_BUS_COMMON_H_ */
diff --git a/sys/sun4v/include/bus_dma.h b/sys/sun4v/include/bus_dma.h
new file mode 100644
index 000000000000..e2b89a1263a5
--- /dev/null
+++ b/sys/sun4v/include/bus_dma.h
@@ -0,0 +1,152 @@
+/*-
+ * Copyright (c) 1996, 1997, 1998, 2001 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The NetBSD Foundation
+ * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
+ * NASA Ames Research Center.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ *    must display the following acknowledgement:
+ *	This product includes software developed by the NetBSD
+ *	Foundation, Inc. and its contributors.
+ * 4. Neither the name of The NetBSD Foundation nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+/*
+ * Copyright (c) 1997-1999 Eduardo E. Horvath. All rights reserved.
+ * Copyright (c) 1996 Charles M. Hannum.  All rights reserved.
+ * Copyright (c) 1996 Christopher G. Demetriou.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ *    must display the following acknowledgement:
+ *      This product includes software developed by Christopher G. Demetriou
+ *	for the NetBSD Project.
+ * 4. The name of the author may not be used to endorse or promote products
+ *    derived from this software without specific prior written permission
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * 	from: NetBSD: bus.h,v 1.28 2001/07/19 15:32:19 thorpej Exp
+ *	and
+ *	from: FreeBSD: src/sys/alpha/include/bus.h,v 1.9 2001/01/09
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _SPARC64_BUS_DMA_H
+#define _SPARC64_BUS_DMA_H
+
+#include <sys/bus_dma.h>
+
+/* DMA support */
+
+/*
+ * Method table for a bus_dma_tag.
+ */
+struct bus_dma_methods {
+	int	(*dm_dmamap_create)(bus_dma_tag_t, int, bus_dmamap_t *);
+	int	(*dm_dmamap_destroy)(bus_dma_tag_t, bus_dmamap_t);
+	int	(*dm_dmamap_load)(bus_dma_tag_t, bus_dmamap_t, void *,
+	    bus_size_t, bus_dmamap_callback_t *, void *, int);
+	int	(*dm_dmamap_load_mbuf)(bus_dma_tag_t, bus_dmamap_t,
+	    struct mbuf *, bus_dmamap_callback2_t *, void *, int);
+	int	(*dm_dmamap_load_mbuf_sg)(bus_dma_tag_t, bus_dmamap_t,
+	    struct mbuf *, bus_dma_segment_t *segs, int *nsegs, int);
+	int	(*dm_dmamap_load_uio)(bus_dma_tag_t, bus_dmamap_t, struct uio *,
+	    bus_dmamap_callback2_t *, void *, int);
+	void	(*dm_dmamap_unload)(bus_dma_tag_t, bus_dmamap_t);
+	void	(*dm_dmamap_sync)(bus_dma_tag_t, bus_dmamap_t,
+	    bus_dmasync_op_t);
+	int	(*dm_dmamem_alloc)(bus_dma_tag_t, void **, int, bus_dmamap_t *);
+	void	(*dm_dmamem_free)(bus_dma_tag_t, void *, bus_dmamap_t);
+};
+
+/*
+ * bus_dma_tag_t
+ *
+ * A machine-dependent opaque type describing the implementation of
+ * DMA for a given bus.
+ */
+struct bus_dma_tag {
+	void		*dt_cookie;		/* cookie used in the guts */
+	bus_dma_tag_t	dt_parent;
+	bus_size_t	dt_alignment;
+	bus_size_t	dt_boundary;
+	bus_addr_t	dt_lowaddr;
+	bus_addr_t	dt_highaddr;
+	bus_dma_filter_t	*dt_filter;
+	void		*dt_filterarg;
+	bus_size_t	dt_maxsize;
+	int		dt_nsegments;
+	bus_size_t	dt_maxsegsz;
+	int		dt_flags;
+	int		dt_ref_count;
+	int		dt_map_count;
+	bus_dma_lock_t	*dt_lockfunc;
+	void *		*dt_lockfuncarg;
+	bus_dma_segment_t *dt_segments;
+
+	struct bus_dma_methods	*dt_mt;
+};
+
+#define	bus_dmamap_create(t, f, p)					\
+	((t)->dt_mt->dm_dmamap_create((t), (f), (p)))
+#define	bus_dmamap_destroy(t, p)					\
+	((t)->dt_mt->dm_dmamap_destroy((t), (p)))
+#define	bus_dmamap_load(t, m, p, s, cb, cba, f)				\
+	((t)->dt_mt->dm_dmamap_load((t), (m), (p), (s), (cb), (cba), (f)))
+#define	bus_dmamap_load_mbuf(t, m, mb, cb, cba, f)			\
+	((t)->dt_mt->dm_dmamap_load_mbuf((t), (m), (mb), (cb), (cba), (f)))
+#define	bus_dmamap_load_mbuf_sg(t, m, mb, segs, nsegs, f)		\
+	((t)->dt_mt->dm_dmamap_load_mbuf_sg((t), (m), (mb), (segs), (nsegs), (f)))
+#define	bus_dmamap_load_uio(t, m, ui, cb, cba, f)			\
+	((t)->dt_mt->dm_dmamap_load_uio((t), (m), (ui), (cb), (cba), (f)))
+#define	bus_dmamap_unload(t, p)						\
+	((t)->dt_mt->dm_dmamap_unload((t), (p)))
+#define	bus_dmamap_sync(t, m, op)					\
+	((t)->dt_mt->dm_dmamap_sync((t), (m), (op)))
+#define	bus_dmamem_alloc(t, v, f, m)					\
+	((t)->dt_mt->dm_dmamem_alloc((t), (v), (f), (m)))
+#define	bus_dmamem_free(t, v, m)					\
+	((t)->dt_mt->dm_dmamem_free((t), (v), (m)))
+
+#endif /* !_SPARC64_BUS_DMA_H_ */
diff --git a/sys/sun4v/include/bus_private.h b/sys/sun4v/include/bus_private.h
new file mode 100644
index 000000000000..ca6536d1c968
--- /dev/null
+++ b/sys/sun4v/include/bus_private.h
@@ -0,0 +1,84 @@
+/*-
+ * Copyright (c) 1997, 1998 Justin T. Gibbs.
+ * Copyright (c) 2002 by Thomas Moestl <tmm@FreeBSD.org>.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+ * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ *	from: FreeBSD: src/sys/i386/i386/busdma_machdep.c,v 1.25 2002/01/05
+ *
+ * $FreeBSD$
+ */
+
+#ifndef	_MACHINE_BUS_PRIVATE_H_
+#define	_MACHINE_BUS_PRIVATE_H_
+
+#include <sys/queue.h>
+
+/*
+ * Helpers
+ */
+int sparc64_bus_mem_map(bus_space_tag_t, bus_space_handle_t, bus_size_t,
+    int, vm_offset_t, void **);
+int sparc64_bus_mem_unmap(void *, bus_size_t);
+bus_space_handle_t sparc64_fake_bustag(int, bus_addr_t, struct bus_space_tag *);
+
+struct bus_dmamap_res {
+	struct resource		*dr_res;
+	bus_size_t		dr_used;
+	bus_size_t		dr_offset;
+	SLIST_ENTRY(bus_dmamap_res)	dr_link;
+};
+
+/*
+ * Callers of the bus_dma interfaces must always protect their tags and maps
+ * appropriately against concurrent access. However, when a map is on a LRU
+ * queue, there is a second access path to it; for this case, the locking rules
+ * are given in the parenthesized comments below:
+ *	q - locked by the mutex protecting the queue.
+ *	p - private to the owner of the map, no access through the queue.
+ *	* - comment refers to pointer target.
+ * Only the owner of the map is allowed to insert the map into a queue. Removal
+ * and repositioning (i.e. temporal removal and reinsertion) is allowed to all
+ * if the queue lock is held.
+ */
+struct bus_dmamap {
+	TAILQ_ENTRY(bus_dmamap)	dm_maplruq;		/* (q) */
+	SLIST_HEAD(, bus_dmamap_res)	dm_reslist;	/* (q, *q) */
+	int			dm_onq;			/* (q) */
+	int			dm_flags;		/* (p) */
+};
+
+/* Flag values. */
+#define	DMF_LOADED	1	/* Map is loaded */
+#define	DMF_COHERENT	2	/* Coherent mapping requested */
+
+int sparc64_dma_alloc_map(bus_dma_tag_t dmat, bus_dmamap_t *mapp);
+void sparc64_dma_free_map(bus_dma_tag_t dmat, bus_dmamap_t map);
+
+/*
+ * XXX: This is a kluge. It would be better to handle dma tags in a hierarchical
+ * way, and have a BUS_GET_DMA_TAG(); however, since this is not currently the
+ * case, save a root tag in the relevant bus attach function and use that.
+ */
+extern bus_dma_tag_t sparc64_root_dma_tag;
+
+#endif /* !_MACHINE_BUS_PRIVATE_H_ */
diff --git a/sys/sun4v/include/cache.h b/sys/sun4v/include/cache.h
new file mode 100644
index 000000000000..8154b273fc1e
--- /dev/null
+++ b/sys/sun4v/include/cache.h
@@ -0,0 +1,117 @@
+/*-
+ * Copyright (c) 1996
+ * 	The President and Fellows of Harvard College. All rights reserved.
+ * Copyright (c) 1992, 1993
+ *	The Regents of the University of California.  All rights reserved.
+ *
+ * This software was developed by the Computer Systems Engineering group
+ * at Lawrence Berkeley Laboratory under DARPA contract BG 91-66 and
+ * contributed to Berkeley.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ *    must display the following acknowledgement:
+ *	This product includes software developed by Aaron Brown and
+ *	Harvard University.
+ * 4. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	from: @(#)cache.h	8.1 (Berkeley) 6/11/93
+ *	from: NetBSD: cache.h,v 1.3 2000/08/01 00:28:02 eeh Exp
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _MACHINE_CACHE_H_
+#define _MACHINE_CACHE_H_
+
+#include <dev/ofw/openfirm.h>
+
+#define	DC_TAG_SHIFT	2
+#define	DC_VALID_SHIFT	0
+
+#define	DC_TAG_BITS	28
+#define	DC_VALID_BITS	2
+
+#define	DC_TAG_MASK	((1 << DC_TAG_BITS) - 1)
+#define	DC_VALID_MASK	((1 << DC_VALID_BITS) - 1)
+
+#define	IC_TAG_SHIFT	7
+#define	IC_VALID_SHIFT	36
+
+#define	IC_TAG_BITS	28
+#define	IC_VALID_BITS	1
+
+#define	IC_TAG_MASK	((1 << IC_TAG_BITS) - 1)
+#define	IC_VALID_MASK	((1 << IC_VALID_BITS) - 1)
+
+/*
+ * Cache control information.
+ */
+struct cacheinfo {
+	u_int	c_enabled;		/* true => cache is enabled */
+	u_int 	ic_size;		/* instruction cache */
+	u_int	ic_set;
+	u_int	ic_l2set;
+	u_int 	ic_assoc;
+	u_int 	ic_linesize;
+	u_int 	dc_size;		/* data cache */
+	u_int	dc_l2size;
+	u_int 	dc_assoc;
+	u_int 	dc_linesize;
+	u_int	ec_size;		/* external cache info */
+	u_int 	ec_assoc;
+	u_int	ec_l2set;
+	u_int	ec_linesize;
+	u_int	ec_l2linesize;
+};
+
+#ifdef _KERNEL
+
+typedef void cache_enable_t(void);
+typedef void cache_flush_t(void);
+typedef void dcache_page_inval_t(vm_paddr_t pa);
+typedef void icache_page_inval_t(vm_paddr_t pa);
+
+void	cache_init(phandle_t node);
+
+cache_enable_t cheetah_cache_enable;
+cache_flush_t cheetah_cache_flush;
+dcache_page_inval_t cheetah_dcache_page_inval;
+icache_page_inval_t cheetah_icache_page_inval;
+
+cache_enable_t spitfire_cache_enable;
+cache_flush_t spitfire_cache_flush;
+dcache_page_inval_t spitfire_dcache_page_inval;
+icache_page_inval_t spitfire_icache_page_inval;
+
+extern cache_enable_t *cache_enable;
+extern cache_flush_t *cache_flush;
+extern dcache_page_inval_t *dcache_page_inval;
+extern icache_page_inval_t *icache_page_inval;
+
+extern struct cacheinfo cache;
+
+#endif
+
+#endif	/* !_MACHINE_CACHE_H_ */
diff --git a/sys/sun4v/include/ccr.h b/sys/sun4v/include/ccr.h
new file mode 100644
index 000000000000..2b7ac84ec9b6
--- /dev/null
+++ b/sys/sun4v/include/ccr.h
@@ -0,0 +1,46 @@
+/*-
+ * Copyright 2001 by Thomas Moestl <tmm@FreeBSD.org>.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+ * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef	_MACHINE_CCR_H_
+#define	_MACHINE_CCR_H_
+
+#define	ICC_SHIFT	0
+#define	ICC_BITS	4
+#define	ICC_MASK	((1UL << ICC_BITS) - 1)
+#define	ICC_C		(1UL << 0)
+#define	ICC_V		(1UL << 1)
+#define	ICC_Z		(1UL << 2)
+#define	ICC_N		(1UL << 3)
+
+#define	XCC_SHIFT	4
+#define	XCC_BITS	4
+#define	XCC_MASK	(((1UL << XCC_BITS) - 1) << XCC_SHIFT)
+#define	XCC_C		(1UL << 4)
+#define	XCC_V		(1UL << 5)
+#define	XCC_Z		(1UL << 6)
+#define	XCC_N		(1UL << 7)
+
+#endif /* !_MACHINE_CCR_H_ */
diff --git a/sys/sun4v/include/cddl/mdesc.h b/sys/sun4v/include/cddl/mdesc.h
new file mode 100644
index 000000000000..9c8242fd2914
--- /dev/null
+++ b/sys/sun4v/include/cddl/mdesc.h
@@ -0,0 +1,214 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ * $FreeBSD$
+ */
+
+/*
+ * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#ifndef	_MDESC_H_
+#define	_MDESC_H_
+
+#include <sys/types.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+/*
+ * Each logical domain is detailed via a (Virtual) Machine Description
+ * available to each guest Operating System courtesy of a
+ * Hypervisor service.
+ */
+
+
+
+#ifdef	_ASM
+#define	U8(_s)	_s
+#define	U16(_s)	_s
+#define	U32(_s)	_s
+#define	U64(_s)	_s
+#else
+#define	U8(_s)	((uint8_t)(_s))
+#define	U16(_s)	((uint16_t)(_s))
+#define	U32(_s)	((uint32_t)(_s))
+#define	U64(_s)	((uint64_t)(_s))
+#endif
+
+
+
+
+
+	/* the version this library understands */
+
+#define	MD_HEADER_VERS_OFF	0x0
+#define	MD_HEADER_NODE_OFF	0x4
+#define	MD_HEADER_NAME_OFF	0x8
+#define	MD_HEADER_DATA_OFF	0xc
+
+#define	MD_HEADER_SIZE	0x10
+
+#define	MD_TRANSPORT_VERSION	U32(0x10000)
+
+#define	MD_ELEMENT_SIZE	0x10
+
+#define	MDE_ILLEGAL_IDX		U64(-1)
+
+#define	MDET_LIST_END	U8(0x0)
+#define	MDET_NULL	U8(' ')
+#define	MDET_NODE	U8('N')
+#define	MDET_NODE_END	U8('E')
+#define	MDET_PROP_ARC	U8('a')
+#define	MDET_PROP_VAL	U8('v')
+#define	MDET_PROP_STR	U8('s')
+#define	MDET_PROP_DAT	U8('d')
+
+
+#ifndef _ASM	/* { */
+
+/*
+ * Opaque handles for use in external interfaces
+ */
+
+typedef void			*md_t;
+
+typedef uint64_t		mde_cookie_t;
+#define	MDE_INVAL_ELEM_COOKIE	((mde_cookie_t)-1)
+
+typedef	uint32_t		mde_str_cookie_t;
+#define	MDE_INVAL_STR_COOKIE	((mde_str_cookie_t)-1)
+
+typedef uint64_t		md_diff_cookie_t;
+#define	MD_INVAL_DIFF_COOKIE	((md_diff_cookie_t)-1)
+
+#define	MDESC_INVAL_GEN		(0)
+
+/*
+ * External structure for MD diff interface
+ */
+typedef struct {
+	uint8_t		type;		/* property type */
+	char		*namep;		/* property name */
+} md_prop_match_t;
+
+
+/*
+ * External Interface
+ */
+
+extern md_t		*md_init_intern(uint64_t *,
+				void *(*allocp)(size_t),
+				void (*freep)(void *, size_t));
+
+extern int		md_fini(md_t *);
+
+extern int		md_node_count(md_t *);
+
+extern mde_str_cookie_t md_find_name(md_t *, char *namep);
+
+extern mde_cookie_t	md_root_node(md_t *);
+
+extern uint64_t		md_get_gen(md_t *);
+
+extern size_t		md_get_bin_size(md_t *);
+
+extern int		md_scan_dag(md_t *,
+				mde_cookie_t,
+				mde_str_cookie_t,
+				mde_str_cookie_t,
+				mde_cookie_t *);
+
+extern int		md_get_prop_val(md_t *,
+				mde_cookie_t,
+				char *,
+				uint64_t *);
+
+extern int		md_get_prop_str(md_t *,
+				mde_cookie_t,
+				char *,
+				char **);
+
+extern int		md_get_prop_data(md_t *,
+				mde_cookie_t,
+				char *,
+				uint8_t **,
+				int *);
+
+extern md_diff_cookie_t	md_diff_init(md_t *,
+				mde_cookie_t,
+				md_t *,
+				mde_cookie_t,
+				char *,
+				md_prop_match_t *);
+
+extern int		md_diff_added(md_diff_cookie_t,
+				mde_cookie_t **);
+
+extern int		md_diff_removed(md_diff_cookie_t,
+				mde_cookie_t **);
+
+extern int		md_diff_matched(md_diff_cookie_t,
+				mde_cookie_t **,
+				mde_cookie_t **);
+
+extern int		md_diff_fini(md_diff_cookie_t);
+
+/***************** NON-CDDL BEGIN *******************************/
+
+#include <sys/malloc.h>
+extern int              md_get_prop_alloc(md_t *, mde_cookie_t, char *, 
+					  int, uint8_t **);
+extern int              md_vdev_find_val(device_t dev, char *namep, 
+					 uint64_t *valp);
+
+
+MALLOC_DECLARE(M_MDPROP);
+extern void             mdesc_init(void);
+extern int              mdesc_update(void);
+
+extern md_t *           md_get(void);
+extern void             md_put(md_t *);
+
+/***************** NON-CDDL END   *******************************/
+
+
+
+#endif	/* } _ASM */
+
+
+
+/*
+ * ioctl info for mdesc device
+ */
+
+#define	MDESCIOC	('m' << 24 | 'd' << 16 | 'd' << 8)
+
+#define	MDESCIOCGSZ	(MDESCIOC | 1)   /* Get quote buffer size */
+#define	MDESCIOCSSZ	(MDESCIOC | 2)   /* Set new quote buffer size */
+#define	MDESCIOCDISCARD	(MDESCIOC | 3)   /* Discard quotes and reset */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif	/* _MDESC_H_ */
diff --git a/sys/sun4v/include/cddl/mdesc_impl.h b/sys/sun4v/include/cddl/mdesc_impl.h
new file mode 100644
index 000000000000..aa9ff27a8bb1
--- /dev/null
+++ b/sys/sun4v/include/cddl/mdesc_impl.h
@@ -0,0 +1,162 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ * $FreeBSD$
+ */
+
+/*
+ * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#ifndef	_MDESC_IMPL_H_
+#define	_MDESC_IMPL_H_
+
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define	LIBMD_MAGIC	0x4d61636844657363ULL	/* MachDesc */
+
+#ifndef _ASM
+
+	/*
+	 * Internal definitions
+	 */
+
+
+/*
+ * Each MD has the following header to
+ * provide information about each section of the MD.
+ *
+ * There are 3 sections:
+ * The description list, the name table and the data block.
+ *
+ * All values are stored in network byte order.
+ *
+ * Elements in the first (description list) section are defined by their
+ * index location within the node block. An index is simply the byte offset
+ * within the block / element size (16bytes). All elements are refered to
+ * by their index, to avoid bugs related to alignment etc.
+ *
+ * The name_len field holds the storage length of an ASCII name, NOT the strlen.
+ * The header fields are written in network
+ * byte order.
+ */
+
+struct md_header_s {
+	uint32_t	transport_version;
+	uint32_t	node_blk_sz;	/* size in bytes of the node block */
+	uint32_t	name_blk_sz;	/* size in bytes of the name block */
+	uint32_t	data_blk_sz;	/* size in bytes of the data block */
+};
+
+typedef struct md_header_s md_header_t;
+
+
+
+#if defined(_BIG_ENDIAN) && !defined(lint)
+#define	mdtoh8(x)	((uint8_t)(x))
+#define	mdtoh16(x)	((uint16_t)(x))
+#define	mdtoh32(x)	((uint32_t)(x))
+#define	mdtoh64(x)	((uint64_t)(x))
+#define	htomd8(x)	(x)
+#define	htomd16(x)	(x)
+#define	htomd32(x)	(x)
+#define	htomd64(x)	(x)
+#else
+#define	mdtoh8(x)	((uint8_t)(x))
+extern	uint16_t	mdtoh16(uint16_t);
+extern	uint32_t	mdtoh32(uint32_t);
+extern	uint64_t	mdtoh64(uint64_t);
+#define	htomd8(x)	((uint8_t)(x))
+extern	uint16_t	htomd16(uint16_t);
+extern	uint32_t	htomd32(uint32_t);
+extern	uint64_t	htomd64(uint64_t);
+#endif
+
+
+
+struct MD_ELEMENT {
+	uint8_t		tag;
+	uint8_t		name_len;
+	uint16_t	_reserved;
+	uint32_t	name_offset;	/* mde_str_cookie_t */
+	union {
+		struct {
+			uint32_t	len;
+			uint32_t	offset;
+		} prop_data;			/* for PROP_DATA and PROP_STR */
+		uint64_t	prop_val;	/* for PROP_VAL */
+		uint64_t	prop_idx;	/* for PROP_ARC and NODE */
+	} d;
+};
+
+typedef struct MD_ELEMENT md_element_t;
+
+struct MACHINE_DESCRIPTION {
+	caddr_t		caddr;
+
+	void		*(*allocp)(size_t);
+	void		(*freep)(void *, size_t);
+
+	md_header_t	*headerp;
+	md_element_t	*mdep;
+	char		*namep;
+	uint8_t		*datap;
+
+	int		node_blk_size;
+	int		name_blk_size;
+	int		data_blk_size;
+
+	int		element_count;
+	int		node_count;
+
+	mde_cookie_t	root_node;
+
+	int		size;
+	uint64_t	gen;
+
+	uint64_t	md_magic;
+};
+
+typedef struct MACHINE_DESCRIPTION md_impl_t;
+
+#define	MDE_TAG(_p)			mdtoh8((_p)->tag)
+#define	MDE_NAME(_p)			mdtoh32((_p)->name_offset)
+#define	MDE_NAME_LEN(_p)		mdtoh32((_p)->name_len)
+#define	MDE_PROP_DATA_OFFSET(_p)	mdtoh32((_p)->d.prop_data.offset)
+#define	MDE_PROP_DATA_LEN(_p)		mdtoh32((_p)->d.prop_data.len)
+#define	MDE_PROP_VALUE(_p)		mdtoh64((_p)->d.prop_val)
+#define	MDE_PROP_INDEX(_p)		mdtoh64((_p)->d.prop_idx)
+
+extern mde_str_cookie_t md_ident_name_str(char *);
+
+extern mde_cookie_t	md_find_node_prop(md_impl_t *,
+				mde_cookie_t,
+				mde_str_cookie_t,
+				int);
+#endif	/* _ASM */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif	/* _MDESC_IMPL_H_ */
diff --git a/sys/sun4v/include/clock.h b/sys/sun4v/include/clock.h
new file mode 100644
index 000000000000..fbedfea57de9
--- /dev/null
+++ b/sys/sun4v/include/clock.h
@@ -0,0 +1,41 @@
+/*-
+ * Copyright (c) 2001 Jake Burkholder.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef	_MACHINE_CLOCK_H_
+#define	_MACHINE_CLOCK_H_
+
+extern u_long tick_increment;
+extern u_long tick_freq;
+extern u_long tick_MHz;
+
+extern int adjkerntz;
+extern int wall_cmos_clock;
+
+int	sysbeep(int, int);
+
+#endif /* !_MACHINE_CLOCK_H_ */
diff --git a/sys/sun4v/include/cpu.h b/sys/sun4v/include/cpu.h
new file mode 100644
index 000000000000..838d4adeb433
--- /dev/null
+++ b/sys/sun4v/include/cpu.h
@@ -0,0 +1,95 @@
+/*-
+ * Copyright (c) 1990 The Regents of the University of California.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to Berkeley by
+ * William Jolitz.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 4. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	from: @(#)cpu.h 5.4 (Berkeley) 5/9/91
+ *	from: FreeBSD: src/sys/i386/include/cpu.h,v 1.62 2001/06/29
+ * $FreeBSD$
+ */
+
+#ifndef	_MACHINE_CPU_H_
+#define	_MACHINE_CPU_H_
+
+#include <machine/frame.h>
+#include <machine/tstate.h>
+extern void cpu_yield(void);
+
+#define	TRAPF_PC(tfp)		((tfp)->tf_tpc)
+#define	TRAPF_USERMODE(tfp)	(((tfp)->tf_tstate & TSTATE_PRIV) == 0)
+
+#define	cpu_getstack(td)	((td)->td_frame->tf_sp)
+#define	cpu_setstack(td, sp)	((td)->td_frame->tf_sp = (sp))
+#define	cpu_spinwait()          
+
+
+
+/*
+ * CTL_MACHDEP definitions.
+ */
+#define CPU_CONSDEV		1	/* dev_t: console terminal device */
+#define	CPU_ADJKERNTZ		2	/* int:	timezone offset	(seconds) */
+#define	CPU_DISRTCSET		3	/* int: disable resettodr() call */
+#define CPU_BOOTINFO		4	/* struct: bootinfo */
+#define	CPU_WALLCLOCK		5	/* int:	indicates wall CMOS clock */
+#define	CPU_MAXID		6	/* number of valid machdep ids */
+
+#define CTL_MACHDEP_NAMES { \
+	{ 0, 0 }, \
+	{ "console_device", CTLTYPE_STRUCT }, \
+	{ "adjkerntz", CTLTYPE_INT }, \
+	{ "disable_rtc_set", CTLTYPE_INT }, \
+	{ "bootinfo", CTLTYPE_STRUCT }, \
+	{ "wall_cmos_clock", CTLTYPE_INT }, \
+}
+
+#ifdef _KERNEL
+
+extern	char btext[];
+extern	char etext[];
+
+void	cpu_halt(void);
+void	cpu_reset(void);
+void	fork_trampoline(void);
+void	swi_vm(void *v);
+
+static __inline u_int64_t
+get_cyclecount(void)
+{
+
+	return (rd(tick));
+}
+
+#define UNIMPLEMENTED panic("%s not implemented", __FUNCTION__)
+
+#define likely(x)  __builtin_expect((x),1)
+#define unlikely(x)  __builtin_expect((x),0)
+#endif
+
+#endif /* !_MACHINE_CPU_H_ */
diff --git a/sys/sun4v/include/cpufunc.h b/sys/sun4v/include/cpufunc.h
new file mode 100644
index 000000000000..01d51d5ea014
--- /dev/null
+++ b/sys/sun4v/include/cpufunc.h
@@ -0,0 +1,255 @@
+/*-
+ * Copyright (c) 2001 Jake Burkholder.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef	_MACHINE_CPUFUNC_H_
+#define	_MACHINE_CPUFUNC_H_
+
+#include <machine/asi.h>
+#include <machine/pstate.h>
+
+struct thread;
+
+/*
+ * membar operand macros for use in other macros when # is a special
+ * character.  Keep these in sync with what the hardware expects.
+ */
+#define	C_Lookaside	(0)
+#define	C_MemIssue	(1)
+#define	C_Sync		(2)
+#define	M_LoadLoad	(0)
+#define	M_StoreLoad	(1)
+#define	M_LoadStore	(2)
+#define	M_StoreStore	(3)
+
+#define	CMASK_SHIFT	(4)
+#define	MMASK_SHIFT	(0)
+
+#define	CMASK_GEN(bit)	((1 << (bit)) << CMASK_SHIFT)
+#define	MMASK_GEN(bit)	((1 << (bit)) << MMASK_SHIFT)
+
+#define	Lookaside	CMASK_GEN(C_Lookaside)
+#define	MemIssue	CMASK_GEN(C_MemIssue)
+#define	Sync		CMASK_GEN(C_Sync)
+#define	LoadLoad	MMASK_GEN(M_LoadLoad)
+#define	StoreLoad	MMASK_GEN(M_StoreLoad)
+#define	LoadStore	MMASK_GEN(M_LoadStore)
+#define	StoreStore	MMASK_GEN(M_StoreStore)
+
+#define	casa(rs1, rs2, rd, asi) ({					\
+	u_int __rd = (uint32_t)(rd);					\
+	__asm __volatile("casa [%2] %3, %4, %0"				\
+	    : "+r" (__rd), "=m" (*rs1)					\
+	    : "r" (rs1), "n" (asi), "r" (rs2), "m" (*rs1));		\
+	__rd;								\
+})
+
+#define	casxa(rs1, rs2, rd, asi) ({					\
+	u_long __rd = (uint64_t)(rd);					\
+	__asm __volatile("casxa [%2] %3, %4, %0"			\
+	    : "+r" (__rd), "=m" (*rs1)					\
+	    : "r" (rs1), "n" (asi), "r" (rs2), "m" (*rs1));		\
+	__rd;								\
+})
+
+#define	flush(va) do {							\
+	__asm __volatile("flush %0" : : "r" (va));			\
+} while (0)
+
+#define	flushw() do {							\
+	__asm __volatile("flushw" : :);					\
+} while (0)
+
+#define	mov(val, reg) do {						\
+	__asm __volatile("mov %0, %" __XSTRING(reg) : : "r" (val));	\
+} while (0)
+
+/* Generate ld*a/st*a functions for non-constant ASI's. */
+#define LDNC_GEN(tp, o)							\
+	static __inline tp						\
+	o ## _nc(caddr_t va, int asi)					\
+	{								\
+		tp r;							\
+		__asm __volatile("wr %2, 0, %%asi;" #o " [%1] %%asi, %0"\
+		    : "=r" (r) : "r" (va), "r" (asi));			\
+		return (r);						\
+	}
+
+LDNC_GEN(u_char, lduba);
+LDNC_GEN(u_short, lduha);
+LDNC_GEN(u_int, lduwa);
+LDNC_GEN(u_long, ldxa);
+
+#define	LD_GENERIC(va, asi, op, type) ({				\
+	type __r;							\
+	__asm __volatile(#op " [%1] %2, %0"				\
+	    : "=r" (__r) : "r" (va), "n" (asi));			\
+	__r;								\
+})
+
+#define	lduba(va, asi)	LD_GENERIC(va, asi, lduba, u_char)
+#define	lduha(va, asi)	LD_GENERIC(va, asi, lduha, u_short)
+#define	lduwa(va, asi)	LD_GENERIC(va, asi, lduwa, u_int)
+#define	ldxa(va, asi)	LD_GENERIC(va, asi, ldxa, u_long)
+
+#if 0
+#define STNC_GEN(tp, o)							\
+	static __inline void						\
+	o ## _nc(caddr_t va, int asi, tp val)				\
+	{								\
+		__asm __volatile(#o " %0, [%g0 + %1] %2"\
+		    : : "r" (val), "r" (va), "r" (asi));		\
+	}
+#else
+#define STNC_GEN(tp, o)							\
+	static __inline void						\
+	o ## _nc(caddr_t va, int asi, tp val)				\
+	{								\
+		__asm __volatile("wr %2, 0, %%asi;" #o " %0, [%1] %%asi"\
+		    : : "r" (val), "r" (va), "r" (asi));		\
+	}
+#endif
+
+STNC_GEN(u_char, stba);
+STNC_GEN(u_short, stha);
+STNC_GEN(u_int, stwa);
+STNC_GEN(u_long, stxa);
+
+#define	ST_GENERIC(va, asi, val, op)					\
+	__asm __volatile(#op " %0, [%1] %2"				\
+	    : : "r" (val), "r" (va), "n" (asi));			\
+
+#define	stba(va, asi, val)	ST_GENERIC(va, asi, val, stba)
+#define	stha(va, asi, val)	ST_GENERIC(va, asi, val, stha)
+#define	stwa(va, asi, val)	ST_GENERIC(va, asi, val, stwa)
+#define	stxa(va, asi, val)	ST_GENERIC(va, asi, val, stxa)
+
+/*
+ * Attempt to read from addr, val.  If a Data Access Error trap happens,
+ * they return -1 and the contents of val is undefined.  A return of 0
+ * means no trap happened, and the contents of val is valid.
+ */
+int fasword8(u_long asi, void *addr, uint8_t *val);
+int fasword16(u_long asi, void *addr, uint16_t *val);
+int fasword32(u_long asi, void *addr, uint32_t *val);
+
+#define	membar(mask) do {						\
+	__asm __volatile("membar %0" : : "n" (mask) : "memory");	\
+} while (0)
+
+#define	rd(name) ({							\
+	uint64_t __sr;							\
+	__asm __volatile("rd %%" #name ", %0" : "=r" (__sr) :);		\
+	__sr;								\
+})
+
+#define	wr(name, val, xor) do {						\
+	__asm __volatile("wr %0, %1, %%" #name				\
+	    : : "r" (val), "rI" (xor));					\
+} while (0)
+
+#define	rdpr(name) ({							\
+	uint64_t __pr;							\
+	__asm __volatile("rdpr %%" #name", %0" : "=r" (__pr) :);	\
+	__pr;								\
+})
+
+#define	wrpr(name, val, xor) do {					\
+	__asm __volatile("wrpr %0, %1, %%" #name			\
+	    : : "r" (val), "rI" (xor));					\
+} while (0)
+
+/*
+ * Macro intended to be used instead of wr(asr23, val, xor) for writing to
+ * the TICK_CMPR register in order to avoid a bug in BlackBird CPUs that
+ * can cause these writes to fail under certain condidtions which in turn
+ * causes the hardclock to stop. The workaround is to perform the write
+ * at the beginning of an I-Cache line directly followed by a dummy read.
+ */
+#define	wrtickcmpr(val, xor) ({						\
+	__asm __volatile(						\
+	"	ba,pt	%%xcc, 1f ;		"			\
+	"	 nop	 ;			"			\
+	"	.align	64 ;			"			\
+	"1:	wr	%0, %1, %%asr23 ;	"			\
+	"	rd	%%asr23, %%g0 ;		"			\
+	: : "r" (val), "rI" (xor));					\
+})
+
+static __inline void
+breakpoint(void)
+{
+	__asm __volatile("ta %%xcc, 1" : :);
+}
+
+static __inline register_t
+intr_disable_all(void)
+{
+	u_long s;
+
+	s = rdpr(pstate);
+	wrpr(pstate, s & ~PSTATE_IE, 0);
+	return (s);
+}
+#define	intr_restore_all(s)	wrpr(pstate, (s), 0)
+
+static __inline register_t
+intr_disable(void)
+{
+	u_long s;
+
+	s = rdpr(pil);
+	wrpr(pil, 14, 0);
+	return (s);
+}
+#define	intr_restore(s)	wrpr(pil, (s), 0)
+
+
+/*
+ * In some places, it is required that the store is directly followed by a
+ * membar #Sync. Don't trust the compiler to not insert instructions in
+ * between. We also need to disable interrupts completely.
+ */
+#define	stxa_sync(va, asi, val) do {					\
+	u_long s;							\
+	s = intr_disable_all();						\
+	__asm __volatile("stxa %0, [%1] %2; membar #Sync"		\
+	    : : "r" (val), "r" (va), "n" (asi));			\
+	intr_restore_all(s);						\
+} while (0)
+
+void ascopy(u_long asi, vm_offset_t src, vm_offset_t dst, size_t len);
+void ascopyfrom(u_long sasi, vm_offset_t src, caddr_t dst, size_t len);
+void ascopyto(caddr_t src, u_long dasi, vm_offset_t dst, size_t len);
+void aszero(u_long asi, vm_offset_t dst, size_t len);
+
+#include <machine/sun4v_cpufunc.h>
+
+#undef LDNC_GEN
+#undef STNC_GEN
+
+#endif /* !_MACHINE_CPUFUNC_H_ */
diff --git a/sys/sun4v/include/db_machdep.h b/sys/sun4v/include/db_machdep.h
new file mode 100644
index 000000000000..439df3699df5
--- /dev/null
+++ b/sys/sun4v/include/db_machdep.h
@@ -0,0 +1,69 @@
+/*-
+ * Mach Operating System
+ * Copyright (c) 1991,1990 Carnegie Mellon University
+ * All Rights Reserved.
+ *
+ * Permission to use, copy, modify and distribute this software and its
+ * documentation is hereby granted, provided that both the copyright
+ * notice and this permission notice appear in all copies of the
+ * software, derivative works or modified versions, and any portions
+ * thereof, and that both notices appear in supporting documentation.
+ *
+ * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
+ * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
+ * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
+ *
+ * Carnegie Mellon requests users of this software to return to
+ *
+ *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
+ *  School of Computer Science
+ *  Carnegie Mellon University
+ *  Pittsburgh PA 15213-3890
+ *
+ * any improvements or extensions that they make and grant Carnegie Mellon
+ * the rights to redistribute these changes.
+ *
+ *	from: FreeBSD: src/sys/i386/include/db_machdep.h,v 1.16 1999/10/04
+ * $FreeBSD$
+ */
+
+#ifndef	_MACHINE_DB_MACHDEP_H_
+#define	_MACHINE_DB_MACHDEP_H_
+
+#include <machine/frame.h>
+#include <machine/trap.h>
+
+#define	BYTE_MSF	(1)
+
+typedef vm_offset_t	db_addr_t;
+typedef long		db_expr_t;
+
+#define	PC_REGS()	((db_addr_t)kdb_thrctx->pcb_pc)
+
+#define	BKPT_INST	(0x91d03001)
+#define	BKPT_SIZE	(4)
+#define	BKPT_SET(inst)	(BKPT_INST)
+
+#define	BKPT_SKIP do {							\
+	kdb_frame->tf_tpc = kdb_frame->tf_tnpc + 4;			\
+	kdb_frame->tf_tnpc += 8;					\
+} while (0)
+
+#define	db_clear_single_step	kdb_cpu_clear_singlestep
+#define	db_set_single_step	kdb_cpu_set_singlestep
+
+#define	IS_BREAKPOINT_TRAP(type, code)	(type == T_BREAKPOINT)
+#define	IS_WATCHPOINT_TRAP(type, code)	(0)
+
+#define	inst_trap_return(ins)	(0)
+#define	inst_return(ins)	(0)
+#define	inst_call(ins)		(0)
+#define	inst_load(ins)		(0)
+#define	inst_store(ins)		(0)
+
+#define	DB_SMALL_VALUE_MAX	(0x7fffffff)
+#define	DB_SMALL_VALUE_MIN	(-0x40001)
+
+#define	DB_ELFSIZE		64
+
+#endif /* !_MACHINE_DB_MACHDEP_H_ */
diff --git a/sys/sun4v/include/elf.h b/sys/sun4v/include/elf.h
new file mode 100644
index 000000000000..bf632d63c785
--- /dev/null
+++ b/sys/sun4v/include/elf.h
@@ -0,0 +1,198 @@
+/*-
+ * Copyright (c) 1996-1997 John D. Polstra.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef	_MACHINE_ELF_H_
+#define	_MACHINE_ELF_H_ 1
+
+/*
+ * ELF definitions for the sparc64 architecture.
+ */
+
+#include <sys/elf32.h>	/* Definitions common to all 32 bit architectures. */
+#include <sys/elf64.h>	/* Definitions common to all 64 bit architectures. */
+
+#ifndef __ELF_WORD_SIZE
+#define	__ELF_WORD_SIZE	64	/* Used by <sys/elf_generic.h> */
+#endif
+#include <sys/elf_generic.h>
+
+#define	ELF_ARCH	EM_SPARCV9
+
+#define	ELF_MACHINE_OK(x)	((x) == ELF_ARCH)
+
+/*
+ * Auxiliary vector entries for passing information to the interpreter.
+ */
+
+typedef struct {	/* Auxiliary vector entry on initial stack */
+	int	a_type;			/* Entry type. */
+	union {
+		int	a_val;		/* Integer value. */
+	} a_un;
+} Elf32_Auxinfo;
+
+typedef struct {	/* Auxiliary vector entry on initial stack */
+	long	a_type;			/* Entry type. */
+	union {
+		long	a_val;		/* Integer value. */
+		void	*a_ptr;		/* Address. */
+		void	(*a_fcn)(void);	/* Function pointer (not used). */
+	} a_un;
+} Elf64_Auxinfo;
+
+__ElfType(Auxinfo);
+
+/* Values for a_type. */
+#define	AT_NULL		0	/* Terminates the vector. */
+#define	AT_IGNORE	1	/* Ignored entry. */
+#define	AT_EXECFD	2	/* File descriptor of program to load. */
+#define	AT_PHDR		3	/* Program header of program already loaded. */
+#define	AT_PHENT	4	/* Size of each program header entry. */
+#define	AT_PHNUM	5	/* Number of program header entries. */
+#define	AT_PAGESZ	6	/* Page size in bytes. */
+#define	AT_BASE		7	/* Interpreter's base address. */
+#define	AT_FLAGS	8	/* Flags (unused). */
+#define	AT_ENTRY	9	/* Where interpreter should transfer control. */
+
+/*
+ * The following non-standard values are used for passing information
+ * from John Polstra's testbed program to the dynamic linker.  These
+ * are expected to go away soon.
+ *
+ * Unfortunately, these overlap the Linux non-standard values, so they
+ * must not be used in the same context.
+ */
+#define	T_BRK		10	/* Starting point for sbrk and brk. */
+#define	AT_DEBUG	11	/* Debugging level. */
+
+/*
+ * The following non-standard values are used in Linux ELF binaries.
+ */
+#define	AT_NOTELF	10	/* Program is not ELF ?? */
+#define	AT_UID		11	/* Real uid. */
+#define	AT_EUID		12	/* Effective uid. */
+#define	AT_GID		13	/* Real gid. */
+#define	AT_EGID		14	/* Effective gid. */
+
+#define	AT_COUNT	15	/* Count of defined aux entry types. */
+
+/*
+ * Relocation types.
+ */
+#define	R_SPARC_NONE		0
+#define	R_SPARC_8		1
+#define	R_SPARC_16		2
+#define	R_SPARC_32		3
+#define	R_SPARC_DISP8		4
+#define	R_SPARC_DISP16		5
+#define	R_SPARC_DISP32		6
+#define	R_SPARC_WDISP30		7
+#define	R_SPARC_WDISP22		8
+#define	R_SPARC_HI22		9
+#define	R_SPARC_22		10
+#define	R_SPARC_13		11
+#define	R_SPARC_LO10		12
+#define	R_SPARC_GOT10		13
+#define	R_SPARC_GOT13		14
+#define	R_SPARC_GOT22		15
+#define	R_SPARC_PC10		16
+#define	R_SPARC_PC22		17
+#define	R_SPARC_WPLT30		18
+#define	R_SPARC_COPY		19
+#define	R_SPARC_GLOB_DAT	20
+#define	R_SPARC_JMP_SLOT	21
+#define	R_SPARC_RELATIVE	22
+#define	R_SPARC_UA32		23
+#define	R_SPARC_PLT32		24
+#define	R_SPARC_HIPLT22		25
+#define	R_SPARC_LOPLT10		26
+#define	R_SPARC_PCPLT32		27
+#define	R_SPARC_PCPLT22		28
+#define	R_SPARC_PCPLT10		29
+#define	R_SPARC_10		30
+#define	R_SPARC_11		31
+#define	R_SPARC_64		32
+#define	R_SPARC_OLO10		33
+#define	R_SPARC_HH22		34
+#define	R_SPARC_HM10		35
+#define	R_SPARC_LM22		36
+#define	R_SPARC_PC_HH22		37
+#define	R_SPARC_PC_HM10		38
+#define	R_SPARC_PC_LM22		39
+#define	R_SPARC_WDISP16		40
+#define	R_SPARC_WDISP19		41
+#define	R_SPARC_GLOB_JMP	42
+#define	R_SPARC_7		43
+#define	R_SPARC_5		44
+#define	R_SPARC_6		45
+#define	R_SPARC_DISP64		46
+#define	R_SPARC_PLT64		47
+#define	R_SPARC_HIX22		48
+#define	R_SPARC_LOX10		49
+#define	R_SPARC_H44		50
+#define	R_SPARC_M44		51
+#define	R_SPARC_L44		52
+#define	R_SPARC_REGISTER	53
+#define	R_SPARC_UA64		54
+#define	R_SPARC_UA16		55
+#define	R_SPARC_TLS_GD_HI22	56
+#define	R_SPARC_TLS_GD_LO10	57
+#define	R_SPARC_TLS_GD_ADD	58
+#define	R_SPARC_TLS_GD_CALL	59
+#define	R_SPARC_TLS_LDM_HI22	60
+#define	R_SPARC_TLS_LDM_LO10	61
+#define	R_SPARC_TLS_LDM_ADD	62
+#define	R_SPARC_TLS_LDM_CALL	63
+#define	R_SPARC_TLS_LDO_HIX22	64
+#define	R_SPARC_TLS_LDO_LOX10	65
+#define	R_SPARC_TLS_LDO_ADD	66
+#define	R_SPARC_TLS_IE_HI22	67
+#define	R_SPARC_TLS_IE_LO10	68
+#define	R_SPARC_TLS_IE_LD	69
+#define	R_SPARC_TLS_IE_LDX	70
+#define	R_SPARC_TLS_IE_ADD	71
+#define	R_SPARC_TLS_LE_HIX22	72
+#define	R_SPARC_TLS_LE_LOX10	73
+#define	R_SPARC_TLS_DTPMOD32	74
+#define	R_SPARC_TLS_DTPMOD64	75
+#define	R_SPARC_TLS_DTPOFF32	76
+#define	R_SPARC_TLS_DTPOFF64	77
+#define	R_SPARC_TLS_TPOFF32	78
+#define	R_SPARC_TLS_TPOFF64	79
+
+/* Define "machine" characteristics */
+#if __ELF_WORD_SIZE == 32
+#define	ELF_TARG_CLASS  ELFCLASS32
+#else
+#define	ELF_TARG_CLASS  ELFCLASS64
+#endif
+#define	ELF_TARG_DATA	ELFDATA2MSB
+#define	ELF_TARG_MACH	ELF_ARCH
+#define	ELF_TARG_VER	1
+
+#endif /* !_MACHINE_ELF_H_ */
diff --git a/sys/sun4v/include/endian.h b/sys/sun4v/include/endian.h
new file mode 100644
index 000000000000..a643dd3537c8
--- /dev/null
+++ b/sys/sun4v/include/endian.h
@@ -0,0 +1,118 @@
+/*-
+ * Copyright (c) 1987, 1991, 1993
+ *	The Regents of the University of California.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 4. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	@(#)endian.h	8.1 (Berkeley) 6/10/93
+ * $FreeBSD$
+ */
+
+#ifndef _MACHINE_ENDIAN_H_
+#define	_MACHINE_ENDIAN_H_
+
+#include <sys/cdefs.h>
+#include <sys/_types.h>
+
+/*
+ * Define the order of 32-bit words in 64-bit words.
+ */
+#define	_QUAD_HIGHWORD 0
+#define	_QUAD_LOWWORD 1
+
+/*
+ * Definitions for byte order, according to byte significance from low
+ * address to high.
+ */
+#define	_LITTLE_ENDIAN	1234	/* LSB first: i386, vax */
+#define	_BIG_ENDIAN	4321	/* MSB first: 68000, ibm, net */
+#define	_PDP_ENDIAN	3412	/* LSB first in word, MSW first in long */
+
+#define	_BYTE_ORDER	_BIG_ENDIAN
+
+/*
+ * Deprecated variants that don't have enough underscores to be useful in more
+ * strict namespaces.
+ */
+#if __BSD_VISIBLE
+#define	LITTLE_ENDIAN	_LITTLE_ENDIAN
+#define	BIG_ENDIAN	_BIG_ENDIAN
+#define	PDP_ENDIAN	_PDP_ENDIAN
+#define	BYTE_ORDER	_BYTE_ORDER
+#endif
+
+#if defined(__GNUCLIKE_BUILTIN_CONSTANT_P) && defined(__OPTIMIZE__)
+#define	__is_constant(x)	__builtin_constant_p(x)
+#else
+#define	__is_constant(x)	0
+#endif
+
+#define	__bswap16_const(x)	((x >> 8) | ((x << 8) & 0xff00))
+#define	__bswap32_const(x)	((x >> 24) | ((x >> 8) & 0xff00) |	\
+	((x << 8) & 0xff0000) | ((x << 24) & 0xff000000))
+#define	__bswap64_const(x)	((x >> 56) | ((x >> 40) & 0xff00) |	\
+	((x >> 24) & 0xff0000) | ((x >> 8) & 0xff000000) |		\
+	((x << 8) & ((__uint64_t)0xff << 32)) |				\
+	((x << 24) & ((__uint64_t)0xff << 40)) |			\
+	((x << 40) & ((__uint64_t)0xff << 48)) | ((x << 56)))
+
+static __inline __uint16_t
+__bswap16_var(__uint16_t _x)
+{
+
+	return ((_x >> 8) | ((_x << 8) & 0xff00));
+}
+
+static __inline __uint32_t
+__bswap32_var(__uint32_t _x)
+{
+
+	return ((_x >> 24) | ((_x >> 8) & 0xff00) | ((_x << 8) & 0xff0000) |
+	    ((_x << 24) & 0xff000000));
+}
+
+static __inline __uint64_t
+__bswap64_var(__uint64_t _x)
+{
+
+	return ((_x >> 56) | ((_x >> 40) & 0xff00) | ((_x >> 24) & 0xff0000) |
+	    ((_x >> 8) & 0xff000000) | ((_x << 8) & ((__uint64_t)0xff << 32)) |
+	    ((_x << 24) & ((__uint64_t)0xff << 40)) |
+	    ((_x << 40) & ((__uint64_t)0xff << 48)) | ((_x << 56)));
+}
+
+#define	__bswap16(x)	(__is_constant(x) ? __bswap16_const(x) : \
+	__bswap16_var(x))
+#define	__bswap32(x)	(__is_constant(x) ? __bswap32_const(x) : \
+	__bswap32_var(x))
+#define	__bswap64(x)	(__is_constant(x) ? __bswap64_const(x) : \
+	__bswap64_var(x))
+
+#define	__htonl(x)	((__uint32_t)(x))
+#define	__htons(x)	((__uint16_t)(x))
+#define	__ntohl(x)	((__uint32_t)(x))
+#define	__ntohs(x)	((__uint16_t)(x))
+
+#endif /* !_MACHINE_ENDIAN_H_ */
diff --git a/sys/sun4v/include/exec.h b/sys/sun4v/include/exec.h
new file mode 100644
index 000000000000..c0f93b68bf90
--- /dev/null
+++ b/sys/sun4v/include/exec.h
@@ -0,0 +1,34 @@
+/*-
+ * Copyright (c) 2001 Jake Burkholder.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef	_MACHINE_EXEC_H_
+#define	_MACHINE_EXEC_H_
+
+#define	__LDPGSZ	8192
+
+#endif /* !_MACHINE_EXEC_H_ */
diff --git a/sys/sun4v/include/float.h b/sys/sun4v/include/float.h
new file mode 100644
index 000000000000..cf78df0c6b55
--- /dev/null
+++ b/sys/sun4v/include/float.h
@@ -0,0 +1,84 @@
+/*-
+ * Copyright (c) 1992, 1993, 2001
+ *	The Regents of the University of California.  All rights reserved.
+ *
+ * This software was developed by the Computer Systems Engineering group
+ * at Lawrence Berkeley Laboratory under DARPA contract BG 91-66 and
+ * contributed to Berkeley.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 4. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	@(#)float.h	8.1 (Berkeley) 6/11/93
+ *	from: NetBSD: float.h,v 1.3 2001/09/21 20:48:02 eeh Exp
+ * $FreeBSD$
+ */
+
+#ifndef _MACHINE_FLOAT_H_
+#define _MACHINE_FLOAT_H_
+
+#include <sys/cdefs.h>
+
+__BEGIN_DECLS
+extern int __flt_rounds(void);
+__END_DECLS
+
+#define FLT_RADIX	2		/* b */
+#define FLT_ROUNDS	__flt_rounds()
+#if __ISO_C_VISIBLE >= 1999
+#define	FLT_EVAL_METHOD	0		/* no promotion */
+#define	DECIMAL_DIG	35		/* max precision in decimal digits */
+#endif
+
+#define FLT_MANT_DIG	24		/* p */
+#define FLT_EPSILON	1.19209290E-7F	/* b**(1-p) */
+#define FLT_DIG		6		/* floor((p-1)*log10(b))+(b == 10) */
+#define FLT_MIN_EXP	(-125)		/* emin */
+#define FLT_MIN		1.17549435E-38F	/* b**(emin-1) */
+#define FLT_MIN_10_EXP	(-37)		/* ceil(log10(b**(emin-1))) */
+#define FLT_MAX_EXP	128		/* emax */
+#define FLT_MAX		3.40282347E+38F	/* (1-b**(-p))*b**emax */
+#define FLT_MAX_10_EXP	38		/* floor(log10((1-b**(-p))*b**emax)) */
+
+#define DBL_MANT_DIG	53
+#define DBL_EPSILON	2.2204460492503131E-16
+#define DBL_DIG		15
+#define DBL_MIN_EXP	(-1021)
+#define DBL_MIN		2.2250738585072014E-308
+#define DBL_MIN_10_EXP	(-307)
+#define DBL_MAX_EXP	1024
+#define DBL_MAX		1.7976931348623157E+308
+#define DBL_MAX_10_EXP	308
+
+#define LDBL_MANT_DIG	113
+#define LDBL_EPSILON	1.925929944387235853055977942584927319E-34L
+#define LDBL_DIG	33
+#define LDBL_MIN_EXP	(-16381)
+#define LDBL_MIN	3.362103143112093506262677817321752603E-4932L
+#define LDBL_MIN_10_EXP	(-4931)
+#define LDBL_MAX_EXP	(+16384)
+#define LDBL_MAX	1.189731495357231765085759326628007016E+4932L
+#define LDBL_MAX_10_EXP	(+4932)
+
+#endif	/* _MACHINE_FLOAT_H_ */
diff --git a/sys/sun4v/include/floatingpoint.h b/sys/sun4v/include/floatingpoint.h
new file mode 100644
index 000000000000..d47756a00c6c
--- /dev/null
+++ b/sys/sun4v/include/floatingpoint.h
@@ -0,0 +1,37 @@
+/*-
+ * Copyright (c) 2002 David O'Brien <obrien@FreeBSD.org>.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of the author nor the names of any co-contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY DAVID O'BRIEN AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _FLOATINGPOINT_H_
+#define _FLOATINGPOINT_H_
+
+#include <machine/ieeefp.h>
+
+#endif /* !_FLOATINGPOINT_H_ */
diff --git a/sys/sun4v/include/fp.h b/sys/sun4v/include/fp.h
new file mode 100644
index 000000000000..bf0d79a8aba6
--- /dev/null
+++ b/sys/sun4v/include/fp.h
@@ -0,0 +1,39 @@
+/*-
+ * Copyright 2001 by Thomas Moestl <tmm@FreeBSD.org>.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+ * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef	_MACHINE_FP_H_
+#define	_MACHINE_FP_H_
+
+#ifdef _KERNEL
+
+/*
+ * Note: The pointer passed to savefpctx must be aligned on a 64 byte
+ * boundary.
+ */
+void	savefpctx(uint32_t *fp);
+
+#endif /* _KERNEL */
+#endif /* !_MACHINE_FP_H_ */
diff --git a/sys/sun4v/include/frame.h b/sys/sun4v/include/frame.h
new file mode 100644
index 000000000000..b9b10881cac6
--- /dev/null
+++ b/sys/sun4v/include/frame.h
@@ -0,0 +1,90 @@
+/*-
+ * Copyright (c) 2001 Jake Burkholder.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef	_MACHINE_FRAME_H_
+#define	_MACHINE_FRAME_H_
+
+#define	RW_SHIFT	7
+#define	SPOFF		2047
+#define	BIAS		SPOFF		/* XXX - open/netbsd compat */
+
+/*
+ * NOTE: keep this structure in sync with struct reg and struct mcontext.
+ */
+struct trapframe {
+	uint64_t tf_global[8];
+	uint64_t tf_out[8];
+	uint64_t tf_fprs;
+	uint64_t tf_fsr;
+	uint64_t tf_gsr;
+	uint64_t tf_pad0[1];
+	uint64_t tf_pil;
+	uint64_t tf_pad1[3];
+	uint64_t tf_tnpc;
+	uint64_t tf_tpc;
+	uint64_t tf_tstate;
+	uint64_t tf_pad2[2];
+	uint64_t tf_wstate;
+	uint64_t tf_asi;
+	uint64_t tf_pad3[1];
+} __aligned(64);
+/* extra padding can go away once we re-shuffle user-land mcontext
+ */
+
+#define	tf_sp	        tf_out[6]
+ 
+#define	TF_DONE(tf) do { \
+	tf->tf_tpc = tf->tf_tnpc; \
+	tf->tf_tnpc += 4; \
+} while (0)
+
+struct frame {
+	u_long	fr_local[8];
+	u_long	fr_in[8];
+	u_long	fr_pad[8];
+};
+#define	fr_arg	fr_in
+#define	fr_fp	fr_in[6]
+#define	fr_pc	fr_in[7]
+
+#define	v9next_frame(fp)	((struct frame *)(fp->fr_fp + BIAS))
+
+/*
+ * Frame used for pcb_rw.
+ */
+struct rwindow {
+	u_long	rw_local[8];
+	u_long	rw_in[8];
+};
+
+struct thread;
+
+int	rwindow_save(struct thread *td);
+int	rwindow_load(struct thread *td, struct trapframe *tf, int n);
+
+#endif /* !_MACHINE_FRAME_H_ */
diff --git a/sys/sun4v/include/fsr.h b/sys/sun4v/include/fsr.h
new file mode 100644
index 000000000000..61a81f7ef965
--- /dev/null
+++ b/sys/sun4v/include/fsr.h
@@ -0,0 +1,108 @@
+/*-
+ * Copyright 2001 by Thomas Moestl <tmm@FreeBSD.org>.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+ * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef	_MACHINE_FSR_H_
+#define	_MACHINE_FSR_H_
+
+#define	FPRS_DL		(1 << 0)
+#define	FPRS_DU		(1 << 1)
+#define	FPRS_FEF	(1 << 2)
+
+#define	FSR_EXC_BITS	5
+#define	FSR_EXC_MASK	((1UL << FSR_EXC_BITS) - 1)
+#define	FSR_CEXC_SHIFT	0
+#define	FSR_CEXC_MASK	(FSR_EXC_MASK << FSR_CEXC_SHIFT)
+#define	FSR_CEXC(b)	((unsigned long)(b) << FSR_CEXC_SHIFT)
+#define	FSR_GET_CEXC(x)	(((x) & FSR_CEXC_MASK) >> FSR_CEXC_SHIFT)
+#define	FSR_AEXC_SHIFT	5
+#define	FSR_AEXC_MASK	(FSR_EXC_MASK << FSR_AEXC_SHIFT)
+#define	FSR_AEXC(b)	((unsigned long)(b) << FSR_AEXC_SHIFT)
+#define	FSR_GET_AEXC(x)	(((x) & FSR_AEXC_MASK) >> FSR_AEXC_SHIFT)
+#define	FSR_QNE		(1UL << 13)
+#define	FSR_NS		(1UL << 22)
+#define	FSR_TEM_SHIFT	23
+#define	FSR_TEM_MASK	(FSR_EXC_MASK << FSR_TEM_SHIFT)
+#define	FSR_TEM(b)	((unsigned long)(b) << FSR_TEM_SHIFT)
+#define	FSR_GET_TEM(x)	(((x) & FSR_TEM_MASK) >> FSR_TEM_SHIFT)
+#define	FSR_FCC0_SHIFT	10
+#define	FSR_FCC0_BITS	2
+#define	FSR_FCC0_MASK	(((1UL << FSR_FCC0_BITS) - 1) << FSR_FCC0_SHIFT)
+#define	FSR_FCC0(x)	((unsigned long)(x) << FSR_FCC0_SHIFT)
+#define	FSR_GET_FCC0(x)	(((x) & FSR_FCC0_MASK) >> FSR_FCC0_SHIFT)
+#define	FSR_FTT_SHIFT	14
+#define	FSR_FTT_BITS	3
+#define	FSR_FTT_MASK	(((1UL << FSR_FTT_BITS) - 1) << FSR_FTT_SHIFT)
+#define	FSR_FTT(x)	((unsigned long)(x) << FSR_FTT_SHIFT)
+#define	FSR_GET_FTT(x)	(((x) & FSR_FTT_MASK) >> FSR_FTT_SHIFT)
+#define	FSR_VER_SHIFT	17
+#define	FSR_GET_VER(x)	(((x) >> FSR_VER_SHIFT) & 7)
+#define	FSR_RD_SHIFT	30
+#define	FSR_RD_BITS	2
+#define	FSR_RD_MASK	(((1UL << FSR_RD_BITS) - 1) << FSR_RD_SHIFT)
+#define	FSR_RD(x)	((unsigned long)(x) << FSR_RD_SHIFT)
+#define	FSR_GET_RD(x)	(((x) & FSR_RD_MASK) >> FSR_RD_SHIFT)
+#define	FSR_FCC1_SHIFT	32
+#define	FSR_FCC1_BITS	2
+#define	FSR_FCC1_MASK	(((1UL << FSR_FCC1_BITS) - 1) << FSR_FCC1_SHIFT)
+#define	FSR_FCC1(x)	((unsigned long)(x) << FSR_FCC1_SHIFT)
+#define	FSR_GET_FCC1(x)	(((x) & FSR_FCC1_MASK) >> FSR_FCC1_SHIFT)
+#define	FSR_FCC2_SHIFT	34
+#define	FSR_FCC2_BITS	2
+#define	FSR_FCC2_MASK	(((1UL << FSR_FCC2_BITS) - 1) << FSR_FCC2_SHIFT)
+#define	FSR_FCC2(x)	((unsigned long)(x) << FSR_FCC2_SHIFT)
+#define	FSR_GET_FCC2(x)	(((x) & FSR_FCC2_MASK) >> FSR_FCC2_SHIFT)
+#define	FSR_FCC3_SHIFT	36
+#define	FSR_FCC3_BITS	2
+#define	FSR_FCC3_MASK	(((1UL << FSR_FCC3_BITS) - 1) << FSR_FCC3_SHIFT)
+#define	FSR_FCC3(x)	((unsigned long)(x) << FSR_FCC3_SHIFT)
+#define	FSR_GET_FCC3(x)	(((x) & FSR_FCC3_MASK) >> FSR_FCC3_SHIFT)
+
+/* CEXC/AEXC/TEM exception values */
+#define	FSR_NX		(1 << 0)
+#define	FSR_DZ		(1 << 1)
+#define	FSR_UF		(1 << 2)
+#define	FSR_OF		(1 << 3)
+#define	FSR_NV		(1 << 4)
+/* FTT values. */
+#define	FSR_FTT_NONE	0
+#define	FSR_FTT_IEEE	1
+#define	FSR_FTT_UNFIN	2
+#define	FSR_FTT_UNIMP	3
+#define	FSR_FTT_SEQERR	4
+#define	FSR_FTT_HWERR	5
+#define	FSR_FTT_INVREG	6
+/* RD values */
+#define	FSR_RD_N	0		/* nearest */
+#define	FSR_RD_Z	1		/* zero */
+#define	FSR_RD_PINF	2		/* +infinity */
+#define	FSR_RD_NINF	3		/* -infinity */
+/* condition codes */
+#define	FSR_CC_EQ	0	/* a = b */
+#define	FSR_CC_LT	1	/* a < b */
+#define	FSR_CC_GT	2	/* a > b */
+#define	FSR_CC_UO	3	/* unordered */
+
+#endif /* !_MACHINE_FSR_H_ */
diff --git a/sys/sun4v/include/gdb_machdep.h b/sys/sun4v/include/gdb_machdep.h
new file mode 100644
index 000000000000..efbc5e78524b
--- /dev/null
+++ b/sys/sun4v/include/gdb_machdep.h
@@ -0,0 +1,57 @@
+/*-
+ * Copyright (c) 2004 Marcel Moolenaar
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _MACHINE_GDB_MACHDEP_H_
+#define	_MACHINE_GDB_MACHDEP_H_
+
+#define	GDB_BUFSZ	600
+#define	GDB_NREGS	86
+#define	GDB_REG_PC	80
+
+static __inline size_t
+gdb_cpu_regsz(int regnum)
+{
+	return ((regnum >= 32 && regnum < 64) ? sizeof(float) : sizeof(long));
+}
+
+static __inline int
+gdb_cpu_query(void)
+{
+	return (0);
+}
+
+static __inline int
+gdb_cpu_signal(int vector, int _)
+{
+	return (vector);
+}
+
+void *gdb_cpu_getreg(int, size_t *);
+void gdb_cpu_setreg(int, void *);
+
+#endif /* !_MACHINE_GDB_MACHDEP_H_ */
diff --git a/sys/sun4v/include/hv_pcivar.h b/sys/sun4v/include/hv_pcivar.h
new file mode 100644
index 000000000000..37414693b28d
--- /dev/null
+++ b/sys/sun4v/include/hv_pcivar.h
@@ -0,0 +1,51 @@
+/*-
+ * Copyright 2006 John-Mark Gurney.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _HV_PCIVAR_H_
+#define _HV_PCIVAR_H_
+
+struct hvpci_softc {
+	devhandle_t		hs_devhandle;
+	phandle_t		hs_node;
+	uint8_t			hs_busnum;
+
+	struct ofw_bus_iinfo	hs_pci_iinfo;
+
+	struct bus_dma_tag	hs_dmatag;
+
+	struct rman		hs_pci_mem_rman;
+	bus_space_tag_t		hs_pci_memt;
+	bus_space_handle_t	hs_pci_memh;
+
+	struct rman		hs_pci_io_rman;
+	bus_space_tag_t		hs_pci_iot;
+	bus_space_handle_t	hs_pci_ioh;
+};
+
+#endif /* _HV_PCIVAR_H_ */
diff --git a/sys/sun4v/include/hviommu.h b/sys/sun4v/include/hviommu.h
new file mode 100644
index 000000000000..a8f8474cb2ea
--- /dev/null
+++ b/sys/sun4v/include/hviommu.h
@@ -0,0 +1,39 @@
+/*-
+ * Copyright 2006 John-Mark Gurney.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _HVIOMMU_H_
+#define _HVIOMMU_H_
+
+struct hviommu;
+
+extern struct bus_dma_methods hviommu_dma_methods;
+
+struct hviommu *hviommu_init(devhandle_t dh, u_long dvmabase, u_long dvmasize);
+
+#endif /* _HVIOMMU_H_ */
diff --git a/sys/sun4v/include/hypervisor_api.h b/sys/sun4v/include/hypervisor_api.h
new file mode 100644
index 000000000000..b760941db797
--- /dev/null
+++ b/sys/sun4v/include/hypervisor_api.h
@@ -0,0 +1,166 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ * $FreeBSD$
+ */
+
+/*
+ * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#ifndef _MACHINE_HYPERVISOR_API_H
+#define	_MACHINE_HYPERVISOR_API_H
+
+/*
+ * sun4v Hypervisor API
+ *
+ * Reference: api.pdf Revision 0.12 dated May 12, 2004.
+ *	      io-api.txt version 1.11 dated 10/19/2004
+ */
+
+#include <machine/hypervisorvar.h>
+
+#ifndef _ASM
+
+typedef uint64_t devhandle_t;
+typedef uint64_t pci_device_t;
+typedef uint32_t pci_config_offset_t;
+typedef uint8_t pci_config_size_t;
+typedef union pci_cfg_data {
+	uint8_t b;
+	uint16_t w;
+	uint32_t dw;
+	uint64_t qw;
+} pci_cfg_data_t;
+typedef uint64_t tsbid_t;
+typedef uint32_t pages_t;
+typedef enum io_attributes {
+	PCI_MAP_ATTR_READ	= (uint32_t)0x01,
+	PCI_MAP_ATTR_WRITE	= (uint32_t)0x02,
+} io_attributes_t;
+typedef enum io_sync_direction {
+	IO_SYNC_DEVICE		= (uint32_t)0x01,
+	IO_SYNC_CPU		= (uint32_t)0x02,
+} io_sync_direction_t;
+typedef uint64_t io_page_list_t;
+typedef uint64_t r_addr_t;
+typedef uint64_t io_addr_t;
+
+typedef struct trap_trace_entry {
+	uint8_t		tte_type;	/* Hypervisor or guest entry. */
+	uint8_t		tte_hpstat;	/* Hyper-privileged state. */
+	uint8_t		tte_tl;		/* Trap level. */
+	uint8_t		tte_gl;		/* Global register level. */
+	uint16_t	tte_tt;		/* Trap type.*/
+	uint16_t	tte_tag;	/* Extended trap identifier. */
+	uint64_t	tte_tstate;	/* Trap state. */
+	uint64_t	tte_tick;	/* Tick. */
+	uint64_t	tte_tpc;	/* Trap PC. */
+	uint64_t	tte_f1;		/* Entry specific. */
+	uint64_t	tte_f2;		/* Entry specific. */
+	uint64_t	tte_f3;		/* Entry specific. */
+	uint64_t	tte_f4;		/* Entry specific. */
+} trap_trace_entry_t;
+
+extern uint64_t hv_mmu_map_perm_addr(void *, int, uint64_t, int);
+extern uint64_t	hv_mmu_unmap_perm_addr(void *, int, int);
+extern uint64_t	hv_set_ctx0(uint64_t, uint64_t);
+extern uint64_t	hv_set_ctxnon0(uint64_t, uint64_t);
+#ifdef SET_MMU_STATS
+extern uint64_t hv_mmu_set_stat_area(uint64_t, uint64_t);
+#endif /* SET_MMU_STATS */
+
+extern uint64_t hv_cpu_qconf(int queue, uint64_t paddr, int size);
+extern uint64_t hv_cpu_mondo_send(int n, vm_paddr_t cpu_list_ra);
+extern uint64_t hv_cpu_yield(void);
+
+extern uint64_t hv_cpu_state(uint64_t cpuid, uint64_t *cpu_state);
+extern uint64_t hv_mem_scrub(uint64_t real_addr, uint64_t length,
+    uint64_t *scrubbed_len);
+extern uint64_t hv_mem_sync(uint64_t real_addr, uint64_t length,
+    uint64_t *flushed_len);
+
+extern uint64_t hv_service_recv(uint64_t s_id, uint64_t buf_pa,
+    uint64_t size, uint64_t *recv_bytes);
+extern uint64_t hv_service_send(uint64_t s_id, uint64_t buf_pa,
+    uint64_t size, uint64_t *send_bytes);
+extern uint64_t hv_service_getstatus(uint64_t s_id, uint64_t *vreg);
+extern uint64_t hv_service_setstatus(uint64_t s_id, uint64_t bits);
+extern uint64_t hv_service_clrstatus(uint64_t s_id, uint64_t bits);
+
+extern uint64_t hv_mach_desc(uint64_t buffer_ra, uint64_t *buffer_sizep);
+
+extern uint64_t hv_ttrace_buf_info(uint64_t *, uint64_t *);
+extern uint64_t hv_ttrace_buf_conf(uint64_t, uint64_t, uint64_t *);
+extern uint64_t hv_ttrace_enable(uint64_t, uint64_t *);
+extern uint64_t hv_ttrace_freeze(uint64_t, uint64_t *);
+extern uint64_t hv_ttrace_addentry(uint64_t, uint64_t, uint64_t, uint64_t, uint64_t);
+extern uint64_t hv_dump_buf_update(uint64_t, uint64_t, uint64_t *);
+
+extern int64_t hv_cnputchar(uint8_t);
+extern int64_t hv_cngetchar(uint8_t *);
+
+extern uint64_t hv_tod_get(uint64_t *seconds);
+extern uint64_t hv_tod_set(uint64_t);
+
+extern uint64_t hvio_intr_devino_to_sysino(uint64_t dev_hdl, uint32_t devino,
+    uint64_t *sysino);
+extern uint64_t hvio_intr_getvalid(uint64_t sysino,
+    int *intr_valid_state);
+extern uint64_t hvio_intr_setvalid(uint64_t sysino,
+    int intr_valid_state);
+extern uint64_t hvio_intr_getstate(uint64_t sysino,
+    int *intr_state);
+extern uint64_t hvio_intr_setstate(uint64_t sysino, int intr_state);
+extern uint64_t hvio_intr_gettarget(uint64_t sysino, uint32_t *cpuid);
+extern uint64_t hvio_intr_settarget(uint64_t sysino, uint32_t cpuid);
+extern uint64_t hvio_peek(devhandle_t dev_hdl, uint64_t r_addr, uint64_t size,
+			  uint32_t *err_flag, uint64_t *data);
+extern uint64_t hvio_poke(devhandle_t dev_hdl, uint64_t r_addr, uint64_t size,
+			  uint64_t data, uint64_t pcidev, uint32_t *err_flag);
+
+extern uint64_t hvio_config_get(devhandle_t dev_hdl, pci_device_t pci_device, 
+				pci_config_offset_t off, pci_config_size_t size, pci_cfg_data_t *data);
+extern uint64_t hvio_config_put(devhandle_t dev_hdl, pci_device_t pci_device, 
+				pci_config_offset_t off, pci_config_size_t size,
+				pci_cfg_data_t data);
+extern uint64_t hvio_iommu_map(devhandle_t dev_hdl, tsbid_t tsbid,
+				pages_t pages, io_attributes_t io_attributes,
+				io_page_list_t *io_page_list_p,
+				pages_t *pages_mapped);
+extern uint64_t hvio_iommu_demap(devhandle_t dev_hdl, tsbid_t tsbid,
+				pages_t pages, pages_t *pages_demapped);
+extern uint64_t hvio_iommu_getmap(devhandle_t dev_hdl, tsbid_t tsbid,
+				 io_attributes_t *attributes_p, r_addr_t *r_addr_p);
+extern uint64_t hvio_iommu_getbypass(devhandle_t dev_hdl, r_addr_t ra,
+				io_attributes_t io_attributes,
+				io_addr_t *io_addr_p);
+extern uint64_t hvio_dma_sync(devhandle_t dev_hdl, r_addr_t ra,
+				size_t num_bytes, uint64_t io_sync_direction,
+				size_t *bytes_synched);
+
+extern void hv_magic_trap_on(void);
+extern void hv_magic_trap_off(void);
+extern int hv_sim_read(uint64_t offset, vm_paddr_t buffer_ra, uint64_t size);
+extern int hv_sim_write(uint64_t offset, vm_paddr_t buffer_ra, uint64_t size);
+
+#endif
+
+#endif /* _MACHINE_HYPERVISOR_API_H */
diff --git a/sys/sun4v/include/hypervisorvar.h b/sys/sun4v/include/hypervisorvar.h
new file mode 100644
index 000000000000..84b03ebdbeb7
--- /dev/null
+++ b/sys/sun4v/include/hypervisorvar.h
@@ -0,0 +1,329 @@
+/*-
+ * Copyright (c) 2006 Kip Macy
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+
+#ifndef	_MACHINE_HYPERVISORVAR_H_
+#define	_MACHINE_HYPERVISORVAR_H_
+/*
+ * Trap types
+ */
+#define	FAST_TRAP		0x80	/* Function # in %o5 */
+#define	CPU_TICK_NPT		0x81
+#define	CPU_STICK_NPT		0x82
+#define	MMU_MAP_ADDR		0x83
+#define	MMU_UNMAP_ADDR		0x84
+
+/*
+ * Error returns in %o0.
+ * (Additional result is returned in %o1.)
+ */
+#define	H_EOK			0	/* Successful return */
+#define	H_ENOCPU		1	/* Invalid CPU id */
+#define	H_ENORADDR		2	/* Invalid real address */
+#define	H_ENOINTR		3	/* Invalid interrupt id */
+#define	H_EBADPGSZ		4	/* Invalid pagesize encoding */
+#define	H_EBADTSB		5	/* Invalid TSB description */
+#define	H_EINVAL		6	/* Invalid argument */
+#define	H_EBADTRAP		7	/* Invalid function number */
+#define	H_EBADALIGN		8	/* Invalid address alignment */
+#define	H_EWOULDBLOCK		9	/* Cannot complete operation */
+					/* without blocking */
+#define	H_ENOACCESS		10	/* No access to resource */
+#define	H_EIO			11	/* I/O error */
+#define	H_ECPUERROR		12	/* CPU is in error state */
+#define	H_ENOTSUPPORTED		13	/* Function not supported */
+#define	H_ENOMAP		14	/* Mapping is not valid, */
+					/* no translation exists */
+
+#define	H_BREAK			-1	/* Console Break */
+#define	H_HUP			-2	/* Console Break */
+
+/*
+ * Mondo CPU ID argument processing.
+ */
+#define	HV_SEND_MONDO_ENTRYDONE	0xffff
+
+/*
+ * Function numbers for FAST_TRAP.
+ */
+#define	HV_MACH_EXIT		0x00
+#define	HV_MACH_DESC		0x01
+#define	HV_CPU_YIELD		0x12
+#define	CPU_QCONF		0x14
+#define	HV_CPU_STATE		0x17
+#define	MMU_TSB_CTX0		0x20
+#define	MMU_TSB_CTXNON0		0x21
+#define	MMU_DEMAP_PAGE		0x22
+#define	MMU_DEMAP_CTX		0x23
+#define	MMU_DEMAP_ALL		0x24
+#define	MAP_PERM_ADDR		0x25
+#define	MMU_SET_INFOPTR		0x26
+#define	UNMAP_PERM_ADDR		0x28
+#define	HV_MEM_SCRUB		0x31
+#define	HV_MEM_SYNC		0x32
+#define	HV_INTR_SEND		0x42
+#define	TOD_GET			0x50
+#define	TOD_SET			0x51
+#define	CONS_READ		0x60
+#define	CONS_WRITE		0x61
+
+#define	SVC_SEND		0x80
+#define	SVC_RECV		0x81
+#define	SVC_GETSTATUS		0x82
+#define	SVC_SETSTATUS		0x83
+#define	SVC_CLRSTATUS		0x84
+
+#define	TTRACE_BUF_CONF		0x90
+#define	TTRACE_BUF_INFO		0x91
+#define	TTRACE_ENABLE		0x92
+#define	TTRACE_FREEZE		0x93
+
+#define	DUMP_BUF_UPDATE		0x94
+
+#define	HVIO_INTR_DEVINO2SYSINO	0xa0
+#define	HVIO_INTR_GETENABLED	0xa1
+#define	HVIO_INTR_SETENABLED	0xa2
+#define	HVIO_INTR_GETSTATE	0xa3
+#define	HVIO_INTR_SETSTATE	0xa4
+#define	HVIO_INTR_GETTARGET	0xa5
+#define	HVIO_INTR_SETTARGET	0xa6
+
+#define	HVIO_IOMMU_MAP		0xb0
+#define	HVIO_IOMMU_DEMAP	0xb1
+#define	HVIO_IOMMU_GETMAP	0xb2
+#define	HVIO_IOMMU_GETBYPASS	0xb3
+
+#define	HVIO_CONFIG_GET		0xb4
+#define	HVIO_CONFIG_PUT		0xb5
+
+#define	HVIO_PEEK		0xb6
+#define	HVIO_POKE		0xb7
+
+#define	HVIO_DMA_SYNC		0xb8
+
+#define	HVIO_MSIQ_CONF		0xc0
+#define	HVIO_MSIQ_INFO		0xc1
+#define	HVIO_MSIQ_GETVALID	0xc2
+#define	HVIO_MSIQ_SETVALID	0xc3
+#define	HVIO_MSIQ_GETSTATE	0xc4
+#define	HVIO_MSIQ_SETSTATE	0xc5
+#define	HVIO_MSIQ_GETHEAD	0xc6
+#define	HVIO_MSIQ_SETHEAD	0xc7
+#define	HVIO_MSIQ_GETTAIL	0xc8
+
+#define	HVIO_MSI_GETVALID	0xc9
+#define	HVIO_MSI_SETVALID	0xca
+#define	HVIO_MSI_GETMSIQ	0xcb
+#define	HVIO_MSI_SETMSIQ	0xcc
+#define	HVIO_MSI_GETSTATE	0xcd
+#define	HVIO_MSI_SETSTATE	0xce
+
+#define	HVIO_MSG_GETMSIQ	0xd0
+#define	HVIO_MSG_SETMSIQ	0xd1
+#define	HVIO_MSG_GETVALID	0xd2
+#define	HVIO_MSG_SETVALID	0xd3
+
+#define HVIO_SIM_READ           0xf0
+#define HVIO_SIM_WRITE          0xf1
+
+
+#ifdef SET_MMU_STATS
+#define	MMU_STAT_AREA		0xfc
+#endif /* SET_MMU_STATS */
+
+#define	HV_NCS_REQUEST		0x110
+
+#define	FIRE_GET_PERFREG	0x120
+#define	FIRE_SET_PERFREG	0x121
+
+#define	HV_RA2PA		0x200
+#define	HV_HPRIV		0x201
+
+/*
+ * Bits for MMU functions flags argument:
+ *	arg3 of MMU_MAP_ADDR
+ *	arg3 of MMU_DEMAP_CTX
+ *	arg2 of MMU_DEMAP_ALL
+ */
+#define	MAP_DTLB		0x1
+#define	MAP_ITLB		0x2
+
+
+/*
+ * Interrupt state manipulation definitions.
+ */
+
+#define	HV_INTR_IDLE_STATE	0
+#define	HV_INTR_RECEIVED_STATE	1
+#define	HV_INTR_DELIVERED_STATE	2
+
+#define	HV_INTR_NOTVALID	0
+#define	HV_INTR_VALID		1
+
+#ifndef LOCORE
+
+/*
+ * TSB description structure for MMU_TSB_CTX0 and MMU_TSB_CTXNON0.
+ */
+typedef struct hv_tsb_info {
+	uint16_t	hvtsb_idxpgsz;	/* page size used to index TSB */
+	uint16_t	hvtsb_assoc;	/* TSB associativity */
+	uint32_t	hvtsb_ntte;	/* TSB size (#TTE entries) */
+	uint32_t	hvtsb_ctx_index; /* context reg index */
+	uint32_t	hvtsb_pgszs;	/* sizes in use */
+	uint64_t	hvtsb_pa;	/* real address of TSB base */
+	uint64_t	hvtsb_rsvd;	/* reserved */
+} hv_tsb_info_t;
+
+#define	HVTSB_SHARE_INDEX	((uint32_t)-1)
+
+#ifdef SET_MMU_STATS
+#ifndef TTE4V_NPGSZ
+#define	TTE4V_NPGSZ	8
+#endif /* TTE4V_NPGSZ */
+/*
+ * MMU statistics structure for MMU_STAT_AREA
+ */
+struct mmu_stat_one {
+	uint64_t	hit_ctx0[TTE4V_NPGSZ];
+	uint64_t	hit_ctxn0[TTE4V_NPGSZ];
+	uint64_t	tsb_miss;
+	uint64_t	tlb_miss;	/* miss, no TSB set */
+	uint64_t	map_ctx0[TTE4V_NPGSZ];
+	uint64_t	map_ctxn0[TTE4V_NPGSZ];
+};
+
+struct mmu_stat {
+	struct mmu_stat_one	immu_stat;
+	struct mmu_stat_one	dmmu_stat;
+	uint64_t		set_ctx0;
+	uint64_t		set_ctxn0;
+};
+#endif /* SET_MMU_STATS */
+
+#endif /* _ASM */
+
+/*
+ * CPU States
+ */
+#define	CPU_STATE_INVALID	0x0
+#define	CPU_STATE_IDLE		0x1	/* cpu not started */
+#define	CPU_STATE_GUEST		0x2	/* cpu running guest code */
+#define	CPU_STATE_ERROR		0x3	/* cpu is in the error state */
+#define	CPU_STATE_LAST_PUBLIC	CPU_STATE_ERROR	/* last valid state */
+
+/*
+ * MMU fault status area
+ */
+
+#define	MMFSA_TYPE_	0x00	/* fault type */
+#define	MMFSA_ADDR_	0x08	/* fault address */
+#define	MMFSA_CTX_	0x10	/* fault context */
+
+#define	MMFSA_I_	0x00		/* start of fields for I */
+#define	MMFSA_I_TYPE	(MMFSA_I_ + MMFSA_TYPE_) /* instruction fault type */
+#define	MMFSA_I_ADDR	(MMFSA_I_ + MMFSA_ADDR_) /* instruction fault address */
+#define	MMFSA_I_CTX	(MMFSA_I_ + MMFSA_CTX_)	/* instruction fault context */
+
+#define	MMFSA_D_	0x40		/* start of fields for D */
+#define	MMFSA_D_TYPE	(MMFSA_D_ + MMFSA_TYPE_) /* data fault type */
+#define	MMFSA_D_ADDR	(MMFSA_D_ + MMFSA_ADDR_) /* data fault address */
+#define	MMFSA_D_CTX	(MMFSA_D_ + MMFSA_CTX_)	/* data fault context */
+
+#define	MMFSA_F_FMISS	1	/* fast miss */
+#define	MMFSA_F_FPROT	2	/* fast protection */
+#define	MMFSA_F_MISS	3	/* mmu miss */
+#define	MMFSA_F_INVRA	4	/* invalid RA */
+#define	MMFSA_F_PRIV	5	/* privilege violation */
+#define	MMFSA_F_PROT	6	/* protection violation */
+#define	MMFSA_F_NFO	7	/* NFO access */
+#define	MMFSA_F_SOPG	8	/* so page */
+#define	MMFSA_F_INVVA	9	/* invalid VA */
+#define	MMFSA_F_INVASI	10	/* invalid ASI */
+#define	MMFSA_F_NCATM	11	/* non-cacheable atomic */
+#define	MMFSA_F_PRVACT	12	/* privileged action */
+#define	MMFSA_F_WPT	13	/* watchpoint hit */
+#define	MMFSA_F_UNALIGN	14	/* unaligned access */
+#define	MMFSA_F_INVPGSZ	15	/* invalid page size */
+
+#define	MMFSA_SIZE	0x80	/* in bytes, 64 byte aligned */
+
+/*
+ * MMU fault status - MMFSA_IFS and MMFSA_DFS
+ */
+#define	MMFS_FV		0x00000001
+#define	MMFS_OW		0x00000002
+#define	MMFS_W		0x00000004
+#define	MMFS_PR		0x00000008
+#define	MMFS_CT		0x00000030
+#define	MMFS_E		0x00000040
+#define	MMFS_FT		0x00003f80
+#define	MMFS_ME		0x00004000
+#define	MMFS_TM		0x00008000
+#define	MMFS_ASI	0x00ff0000
+#define	MMFS_NF		0x01000000
+
+/*
+ * DMA sync parameter definitions
+ */
+#define	HVIO_DMA_SYNC_DIR_TO_DEV	0x01
+#define	HVIO_DMA_SYNC_DIR_FROM_DEV	0x02
+
+/*
+ * Performance counter register definitions.
+ */
+#define	HVIO_FIRE_PERFREG_JBC_SEL	0
+#define	HVIO_FIRE_PERFREG_JBC_CNT0	1
+#define	HVIO_FIRE_PERFREG_JBC_CNT1	2
+#define	HVIO_FIRE_PERFREG_PCIE_IMU_SEL	3
+#define	HVIO_FIRE_PERFREG_PCIE_IMU_CNT0	4
+#define	HVIO_FIRE_PERFREG_PCIE_IMU_CNT1	5
+#define	HVIO_FIRE_PERFREG_PCIE_MMU_SEL	6
+#define	HVIO_FIRE_PERFREG_PCIE_MMU_CNT0	7
+#define	HVIO_FIRE_PERFREG_PCIE_MMU_CNT1	8
+#define	HVIO_FIRE_PERFREG_PCIE_TLU_SEL	9
+#define	HVIO_FIRE_PERFREG_PCIE_TLU_CNT0	10
+#define	HVIO_FIRE_PERFREG_PCIE_TLU_CNT1	11
+#define	HVIO_FIRE_PERFREG_PCIE_TLU_CNT2	12
+#define	HVIO_FIRE_PERFREG_PCIE_LNK_SEL	13
+#define	HVIO_FIRE_PERFREG_PCIE_LNK_CNT1	14
+#define	HVIO_FIRE_PERFREG_PCIE_LNK_CNT2	15
+
+
+#ifdef SIMULATOR
+#define MAGIC_TRAP_ON	ta	0x77
+#define MAGIC_TRAP_OFF	ta	0x78
+#define MAGIC_EXIT	ta	0x71
+#else
+#define MAGIC_TRAP_ON	nop
+#define MAGIC_TRAP_OFF	nop
+#define MAGIC_EXIT	nop
+#endif
+
+
+#endif	/*_MACHINE_HYPERVISORVAR_H_ */
diff --git a/sys/sun4v/include/idprom.h b/sys/sun4v/include/idprom.h
new file mode 100644
index 000000000000..df76a7f73122
--- /dev/null
+++ b/sys/sun4v/include/idprom.h
@@ -0,0 +1,63 @@
+/*-
+ * Copyright (c) 1993 Adam Glass
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ *    must display the following acknowledgement:
+ *	This product includes software developed by Adam Glass.
+ * 4. The name of the Author may not be used to endorse or promote products
+ *    derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY Adam Glass ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	from: NetBSD: idprom.h,v 1.2 1998/09/05 23:57:26 eeh Exp
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _MACHINE_IDPROM_H_
+#define _MACHINE_IDPROM_H_
+
+/*
+ * ID prom format.  The ``host id'' is set up by taking the machine
+ * ID as the top byte and the hostid field as the remaining three.
+ * The id_xxx0 field appears to contain some other number.  The id_xxx1
+ * contains a bunch of 00's and a5's on my machines, suggesting it is
+ * not actually used.  The checksum seems to include them, however.
+ */
+struct idprom {
+	u_char	id_format;		/* format identifier (= 1) */
+	u_char	id_machine;		/* machine type (see param.h) */
+	u_char	id_ether[6];		/* ethernet address */
+	int	id_date;		/* date of manufacture */
+	u_char	id_hostid[3];		/* ``host id'' bytes */
+	u_char	id_checksum;		/* xor of everything else */
+	char	id_undef[16];		/* undefined */
+};
+
+#define ID_SUN4_100	0x22
+#define ID_SUN4_200	0x21
+#define ID_SUN4_300	0x23
+#define ID_SUN4_400	0x24
+
+#define IDPROM_VERSION 1
+
+#endif /* !_MACHINE_IDPROM_H_ */
diff --git a/sys/sun4v/include/ieee.h b/sys/sun4v/include/ieee.h
new file mode 100644
index 000000000000..82ba9a1ae62f
--- /dev/null
+++ b/sys/sun4v/include/ieee.h
@@ -0,0 +1,137 @@
+/*-
+ * Copyright (c) 1992, 1993
+ *	The Regents of the University of California.  All rights reserved.
+ *
+ * This software was developed by the Computer Systems Engineering group
+ * at Lawrence Berkeley Laboratory under DARPA contract BG 91-66 and
+ * contributed to Berkeley.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 4. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	@(#)ieee.h	8.1 (Berkeley) 6/11/93
+ *	from: NetBSD: ieee.h,v 1.1.1.1 1998/06/20 04:58:51 eeh Exp
+ * $FreeBSD$
+ */
+
+#ifndef _MACHINE_IEEE_H_
+#define	_MACHINE_IEEE_H_
+
+/*
+ * ieee.h defines the machine-dependent layout of the machine's IEEE
+ * floating point.  It does *not* define (yet?) any of the rounding
+ * mode bits, exceptions, and so forth.
+ */
+
+/*
+ * Define the number of bits in each fraction and exponent.
+ *
+ *		     k	         k+1
+ * Note that  1.0 x 2  == 0.1 x 2      and that denorms are represented
+ *
+ *					  (-exp_bias+1)
+ * as fractions that look like 0.fffff x 2             .  This means that
+ *
+ *			 -126
+ * the number 0.10000 x 2    , for instance, is the same as the normalized
+ *
+ *		-127			   -128
+ * float 1.0 x 2    .  Thus, to represent 2    , we need one leading zero
+ *
+ *				  -129
+ * in the fraction; to represent 2    , we need two, and so on.  This
+ *
+ *						     (-exp_bias-fracbits+1)
+ * implies that the smallest denormalized number is 2
+ *
+ * for whichever format we are talking about: for single precision, for
+ *
+ *						-126		-149
+ * instance, we get .00000000000000000000001 x 2    , or 1.0 x 2    , and
+ *
+ * -149 == -127 - 23 + 1.
+ */
+#define	SNG_EXPBITS	8
+#define	SNG_FRACBITS	23
+
+#define	DBL_EXPBITS	11
+#define	DBL_FRACBITS	52
+
+#ifdef notyet
+#define	E80_EXPBITS	15
+#define	E80_FRACBITS	64
+#endif
+
+#define	EXT_EXPBITS	15
+#define	EXT_FRACBITS	112
+
+struct ieee_single {
+	u_int	sng_sign:1;
+	u_int	sng_exp:8;
+	u_int	sng_frac:23;
+};
+
+struct ieee_double {
+	u_int	dbl_sign:1;
+	u_int	dbl_exp:11;
+	u_int	dbl_frach:20;
+	u_int	dbl_fracl;
+};
+
+struct ieee_ext {
+	u_int	ext_sign:1;
+	u_int	ext_exp:15;
+	u_int	ext_frach:16;
+	u_int	ext_frachm;
+	u_int	ext_fraclm;
+	u_int	ext_fracl;
+};
+
+/*
+ * Floats whose exponent is in [1..INFNAN) (of whatever type) are
+ * `normal'.  Floats whose exponent is INFNAN are either Inf or NaN.
+ * Floats whose exponent is zero are either zero (iff all fraction
+ * bits are zero) or subnormal values.
+ *
+ * A NaN is a `signalling NaN' if its QUIETNAN bit is clear in its
+ * high fraction; if the bit is set, it is a `quiet NaN'.
+ */
+#define	SNG_EXP_INFNAN	255
+#define	DBL_EXP_INFNAN	2047
+#define	EXT_EXP_INFNAN	32767
+
+#if 0
+#define	SNG_QUIETNAN	(1 << 22)
+#define	DBL_QUIETNAN	(1 << 19)
+#define	EXT_QUIETNAN	(1 << 15)
+#endif
+
+/*
+ * Exponent biases.
+ */
+#define	SNG_EXP_BIAS	127
+#define	DBL_EXP_BIAS	1023
+#define	EXT_EXP_BIAS	16383
+
+#endif
diff --git a/sys/sun4v/include/ieeefp.h b/sys/sun4v/include/ieeefp.h
new file mode 100644
index 000000000000..11264949acee
--- /dev/null
+++ b/sys/sun4v/include/ieeefp.h
@@ -0,0 +1,26 @@
+/*-
+ * Written by J.T. Conklin, Apr 6, 1995
+ * Public domain.
+ * $FreeBSD$
+ */
+
+#ifndef _MACHINE_IEEEFP_H_
+#define _MACHINE_IEEEFP_H_
+
+#include <machine/fsr.h>
+
+typedef int fp_except_t;
+#define FP_X_IMP	FSR_NX	/* imprecise (loss of precision) */
+#define FP_X_DZ		FSR_DZ	/* divide-by-zero exception */
+#define FP_X_UFL	FSR_UF	/* underflow exception */
+#define FP_X_OFL	FSR_OF	/* overflow exception */
+#define FP_X_INV	FSR_NV	/* invalid operation exception */
+
+typedef enum {
+	FP_RN = FSR_RD_N,	/* round to nearest representable number */
+	FP_RZ = FSR_RD_Z,	/* round to zero (truncate) */
+	FP_RP = FSR_RD_PINF,	/* round toward positive infinity */
+	FP_RM = FSR_RD_NINF	/* round toward negative infinity */
+} fp_rnd_t;
+
+#endif /* _MACHINE_IEEEFP_H_ */
diff --git a/sys/sun4v/include/in_cksum.h b/sys/sun4v/include/in_cksum.h
new file mode 100644
index 000000000000..2e46f4632968
--- /dev/null
+++ b/sys/sun4v/include/in_cksum.h
@@ -0,0 +1,169 @@
+/*-
+ * Copyright (c) 1990 The Regents of the University of California.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 4. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+/*-
+ * Copyright (c) 2001 by Thomas Moestl <tmm@FreeBSD.org>.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+ * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ *	from tahoe:	in_cksum.c	1.2	86/01/05
+ *	from:		@(#)in_cksum.c	1.3 (Berkeley) 1/19/91
+ *	from: Id: in_cksum.c,v 1.8 1995/12/03 18:35:19 bde Exp
+ *	from: FreeBSD: src/sys/alpha/include/in_cksum.h,v 1.5 2000/05/06
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _MACHINE_IN_CKSUM_H_
+#define	_MACHINE_IN_CKSUM_H_	1
+
+#include <sys/cdefs.h>
+
+#define in_cksum(m, len)	in_cksum_skip(m, len, 0)
+
+static __inline void
+in_cksum_update(struct ip *ip)
+{
+	int __tmp;
+
+	__tmp = (int)ip->ip_sum + 1;
+	ip->ip_sum = __tmp + (__tmp >> 16);
+}
+
+static __inline u_short
+in_addword(u_short sum, u_short b)
+{
+	u_long __ret, __tmp;
+
+	__asm(
+	    "sll %2, 16, %0\n"
+	    "sll %3, 16, %1\n"
+	    "addcc %0, %1, %0\n"
+	    "srl %0, 16, %0\n"
+	    "addc %0, 0, %0\n"
+	    : "=&r" (__ret), "=&r" (__tmp) : "r" (sum), "r" (b));
+	return (__ret);
+}
+
+static __inline u_short
+in_pseudo(u_int sum, u_int b, u_int c)
+{
+	u_long __tmp;
+		    
+	__asm(
+	    "addcc %0, %3, %0\n"
+	    "addccc %0, %4, %0\n"
+	    "addc %0, 0, %0\n"
+	    "sll %0, 16, %1\n"
+	    "addcc %0, %1, %0\n"
+	    "srl %0, 16, %0\n"
+	    "addc %0, 0, %0\n"
+	    : "=r" (sum), "=&r" (__tmp) : "0" (sum), "r" (b), "r" (c));
+	return (sum);
+}
+
+static __inline u_int
+in_cksum_hdr(struct ip *ip)
+{
+	u_long __ret, __tmp1, __tmp2, __tmp3, __tmp4;
+
+	/*
+	 * Use 32 bit memory accesses and additions - addition with carry only
+	 * works for 32 bits, and fixing up alignment issues for 64 is probably
+	 * more trouble than it's worth.
+	 * This may read outside of the ip header, but does not cross a page
+	 * boundary in doing so, so that should be OK.
+	 * Actually, this specialized implementation might be overkill - using
+	 * a generic implementation for both in_cksum_skip and in_cksum_hdr
+	 * should not be too much more expensive.
+	 */
+#define __LD_ADD(addr, tmp, sum, offs, mod) \
+    "lduw [" #addr " + " #offs "], " #tmp "\n" \
+    "add" # mod " " #sum ", " #tmp ", " #sum "\n"
+
+	__asm(
+	    "and %5, 3, %3\n"
+	    "andn %5, 3, %1\n"
+	    "brz,pt %3, 0f\n"
+	    " lduw [%1], %0\n"
+	    "mov 4, %4\n"
+	    "sub %4, %3, %4\n"
+	    "sll %4, 3, %4\n"		/* fix up unaligned buffers */
+	    "mov 1, %2\n"
+	    "sll %2, %4, %4\n"
+	    "sub %4, 1, %4\n"
+	    "lduw [%1 + 20], %2\n"
+	    "andn %2, %4, %2\n"
+	    "and %0, %4, %0\n"
+	    "or %0, %2, %0\n"
+	    "0:\n"
+	    __LD_ADD(%1, %2, %0, 4, cc)
+	    __LD_ADD(%1, %2, %0, 8, ccc)
+	    __LD_ADD(%1, %2, %0, 12, ccc)
+	    __LD_ADD(%1, %2, %0, 16, ccc)
+	    "addc %0, 0, %0\n"		/* reduce */
+	    "1:\n"
+	    "sll %0, 16, %2\n"
+	    "addcc %0, %2, %0\n"
+	    "srl %0, 16, %0\n"
+	    "addc %0, 0, %0\n"
+	    "andcc %3, 1, %3\n"		/* need to byte-swap? */
+	    "clr %3\n"
+	    "bne,a,pn %%xcc, 1b\n"
+	    " sll %0, 8, %0\n"
+	    "not %0\n"
+	    "sll %0, 16, %0\n"
+	    "srl %0, 16, %0\n"
+	    : "=&r" (__ret), "=r" (__tmp1), "=&r" (__tmp2), "=&r" (__tmp3),
+		"=&r" (__tmp4) : "1" (ip));
+#undef __LD_ADD
+	return (__ret);
+}
+
+u_short	in_cksum_skip(struct mbuf *m, int len, int skip);
+
+#endif /* _MACHINE_IN_CKSUM_H_ */
diff --git a/sys/sun4v/include/instr.h b/sys/sun4v/include/instr.h
new file mode 100644
index 000000000000..d6befbbc3ffe
--- /dev/null
+++ b/sys/sun4v/include/instr.h
@@ -0,0 +1,618 @@
+/*-
+ * Copyright (c) 1994 David S. Miller, davem@nadzieja.rutgers.edu
+ * Copyright (c) 1995 Paul Kranenburg
+ * Copyright (c) 2001 Thomas Moestl <tmm@FreeBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ *    must display the following acknowledgement:
+ *      This product includes software developed by David Miller.
+ * 4. The name of the author may not be used to endorse or promote products
+ *    derived from this software without specific prior written permission
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ *	from: NetBSD: db_disasm.c,v 1.9 2000/08/16 11:29:42 pk Exp
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _MACHINE_INSTR_H_
+#define _MACHINE_INSTR_H_
+
+/*
+ * Definitions for all instruction formats
+ */
+#define	IF_OP_SHIFT		30
+#define	IF_OP_BITS		 2
+#define	IF_IMM_SHIFT		 0	/* Immediate/Displacement */
+
+/*
+ * Definitions for format 2
+ */
+#define	IF_F2_RD_SHIFT		25
+#define	IF_F2_RD_BITS		 5
+#define	IF_F2_A_SHIFT		29
+#define	IF_F2_A_BITS		 1
+#define	IF_F2_COND_SHIFT	25
+#define	IF_F2_COND_BITS		 4
+#define	IF_F2_RCOND_SHIFT	25
+#define	IF_F2_RCOND_BITS	 3
+#define	IF_F2_OP2_SHIFT		22
+#define	IF_F2_OP2_BITS		 3
+#define	IF_F2_CC1_SHIFT		21
+#define	IF_F2_CC1_BITS		 1
+#define	IF_F2_CC0_SHIFT		20
+#define	IF_F2_CC0_BITS		 1
+#define	IF_F2_CC_SHIFT		20	/* CC0 and CC1 combined. */
+#define	IF_F2_CC_BITS		 2
+#define	IF_F2_D16HI_SHIFT	20
+#define	IF_F2_D16HI_BITS	 2
+#define	IF_F2_P_SHIFT		19
+#define	IF_F2_P_BITS		 1
+#define	IF_F2_RS1_SHIFT		14
+#define	IF_F2_RS1_BITS		 5
+
+/*
+ * Definitions for format 3
+ */
+#define	IF_F3_OP3_SHIFT		19
+#define	IF_F3_OP3_BITS		 6
+#define	IF_F3_RD_SHIFT		IF_F2_RD_SHIFT
+#define	IF_F3_RD_BITS		IF_F2_RD_BITS
+#define	IF_F3_FCN_SHIFT		25
+#define	IF_F3_FCN_BITS		 5
+#define	IF_F3_CC1_SHIFT		26
+#define	IF_F3_CC1_BITS		 1
+#define	IF_F3_CC0_SHIFT		25
+#define	IF_F3_CC0_BITS		 1
+#define	IF_F3_CC_SHIFT		25	/* CC0 and CC1 combined. */
+#define	IF_F3_CC_BITS		 2
+#define	IF_F3_RS1_SHIFT		IF_F2_RS1_SHIFT
+#define	IF_F3_RS1_BITS		IF_F2_RS1_BITS
+#define	IF_F3_I_SHIFT		13
+#define	IF_F3_I_BITS		 1
+#define	IF_F3_X_SHIFT		12
+#define	IF_F3_X_BITS		 1
+#define	IF_F3_RCOND_SHIFT	10
+#define	IF_F3_RCOND_BITS	 3
+#define	IF_F3_IMM_ASI_SHIFT	 5
+#define	IF_F3_IMM_ASI_BITS	 8
+#define	IF_F3_OPF_SHIFT		 5
+#define	IF_F3_OPF_BITS		 9
+#define	IF_F3_CMASK_SHIFT	 4
+#define	IF_F3_CMASK_BITS	 3
+#define	IF_F3_RS2_SHIFT		 0
+#define	IF_F3_RS2_BITS		 5
+#define	IF_F3_SHCNT32_SHIFT	 0
+#define	IF_F3_SHCNT32_BITS	 5
+#define	IF_F3_SHCNT64_SHIFT	 0
+#define	IF_F3_SHCNT64_BITS	 6
+
+/*
+ * Definitions for format 4
+ */
+#define	IF_F4_OP3_SHIFT		IF_F3_OP3_SHIFT
+#define	IF_F4_OP3_BITS		IF_F3_OP3_BITS
+#define	IF_F4_RD_SHIFT		IF_F2_RD_SHIFT
+#define	IF_F4_RD_BITS		IF_F2_RD_BITS
+#define	IF_F4_RS1_SHIFT		IF_F2_RS1_SHIFT
+#define	IF_F4_RS1_BITS		IF_F2_RS1_BITS
+#define	IF_F4_TCOND_SHIFT	IF_F2_COND_SHIFT	/* cond for Tcc */
+#define	IF_F4_TCOND_BITS	IF_F2_COND_BITS
+#define	IF_F4_CC2_SHIFT		18
+#define	IF_F4_CC2_BITS		 1
+#define	IF_F4_COND_SHIFT	14
+#define	IF_F4_COND_BITS		 4
+#define	IF_F4_I_SHIFT		IF_F3_I_SHIFT
+#define	IF_F4_I_BITS		IF_F3_I_BITS
+#define	IF_F4_OPF_CC_SHIFT	11
+#define	IF_F4_OPF_CC_BITS	 3
+#define	IF_F4_CC1_SHIFT		12
+#define	IF_F4_CC1_BITS		 1
+#define	IF_F4_CC0_SHIFT		11
+#define	IF_F4_CC0_BITS		 1
+#define	IF_F4_RCOND_SHIFT	IF_F3_RCOND_SHIFT
+#define	IF_F4_RCOND_BITS	IF_F3_RCOND_BITS
+#define	IF_F4_OPF_LOW_SHIFT	 5
+#define	IF_F4_RS2_SHIFT		IF_F3_RS2_SHIFT
+#define	IF_F4_RS2_BITS		IF_F3_RS2_BITS
+#define	IF_F4_SW_TRAP_SHIFT	 0
+#define	IF_F4_SW_TRAP_BITS	 7
+
+/*
+ * Macros to decode instructions
+ */
+/* Extract a field */
+#define	IF_MASK(s, w)		(((1 << (w)) - 1) << (s))
+#define	IF_EXTRACT(x, s, w)	(((x) & IF_MASK((s), (w))) >> (s))
+#define	IF_DECODE(x, f) \
+	IF_EXTRACT((x), IF_ ## f ## _SHIFT, IF_ ## f ## _BITS)
+
+/* Sign-extend a field of width W */
+#define	IF_SEXT(x, w) \
+	(((x) & (1L << ((w) - 1))) != 0 ? \
+	    (-1L - ((x) ^ ((1L << (w)) - 1))) : (x))
+
+#if 0
+/*
+ * The following C variant is from db_disassemble.c, and surely faster, but it
+ * relies on behaviour that is undefined by the C standard (>> in conjunction
+ * with signed negative arguments).
+ */
+#define	IF_SEXT(v, w)	((((long long)(v)) << (64 - w)) >> (64 - w))
+/* Assembler version of the above */
+#define	IF_SEXT(v, w) \
+	{ u_long t; ( __asm __volatile("sllx %1, %2, %0; srax %0, %2, %0" :
+	    "=r" (t) : "r" (v) : "i" (64 - w)); t)}
+#endif
+
+/* All instruction formats */
+#define	IF_OP(i)		IF_DECODE(i, OP)
+
+/* Instruction format 2 */
+#define	IF_F2_RD(i)		IF_DECODE((i), F2_RD)
+#define	IF_F2_A(i)		IF_DECODE((i), F2_A)
+#define	IF_F2_COND(i)		IF_DECODE((i), F2_COND)
+#define	IF_F2_RCOND(i)		IF_DECODE((i), F2_RCOND)
+#define	IF_F2_OP2(i)		IF_DECODE((i), F2_OP2)
+#define	IF_F2_CC1(i)		IF_DECODE((i), F2_CC1)
+#define	IF_F2_CC0(i)		IF_DECODE((i), F2_CC0)
+#define	IF_F2_CC(i)		IF_DECODE((i), F2_CC)
+#define	IF_F2_D16HI(i)		IF_DECODE((i), F2_D16HI)
+#define	IF_F2_P(i)		IF_DECODE((i), F2_P)
+#define	IF_F2_RS1(i)		IF_DECODE((i), F2_RS1)
+
+/* Instruction format 3 */
+#define	IF_F3_OP3(i)		IF_DECODE((i), F3_OP3)
+#define	IF_F3_RD(i)		IF_F2_RD((i))
+#define	IF_F3_FCN(i)		IF_DECODE((i), F3_FCN)
+#define	IF_F3_CC1(i)		IF_DECODE((i), F3_CC1)
+#define	IF_F3_CC0(i)		IF_DECODE((i), F3_CC0)
+#define	IF_F3_CC(i)		IF_DECODE((i), F3_CC)
+#define	IF_F3_RS1(i)		IF_F2_RS1((i))
+#define	IF_F3_I(i)		IF_DECODE((i), F3_I)
+#define	IF_F3_X(i)		IF_DECODE((i), F3_X)
+#define	IF_F3_RCOND(i)		IF_DECODE((i), F3_RCOND)
+#define	IF_F3_IMM_ASI(i)	IF_DECODE((i), F3_IMM_ASI)
+#define	IF_F3_OPF(i)		IF_DECODE((i), F3_OPF)
+#define	IF_F3_CMASK(i)		IF_DECODE((i), F3_CMASK)
+#define	IF_F3_RS2(i)		IF_DECODE((i), F3_RS2)
+#define	IF_F3_SHCNT32(i)	IF_DECODE((i), F3_SHCNT32)
+#define	IF_F3_SHCNT64(i)	IF_DECODE((i), F3_SHCNT64)
+
+/* Instruction format 4 */
+#define	IF_F4_OP3(i)		IF_F3_OP3((i))
+#define	IF_F4_RD(i)		IF_F3_RD((i))
+#define	IF_F4_TCOND(i)		IF_DECODE((i), F4_TCOND)
+#define	IF_F4_RS1(i)		IF_F3_RS1((i))
+#define	IF_F4_CC2(i)		IF_DECODE((i), F4_CC2)
+#define	IF_F4_COND(i)		IF_DECODE((i), F4_COND)
+#define	IF_F4_I(i)		IF_F3_I((i))
+#define	IF_F4_OPF_CC(i)		IF_DECODE((i), F4_OPF_CC)
+#define	IF_F4_RCOND(i)		IF_F3_RCOND((i))
+#define	IF_F4_OPF_LOW(i, w)	IF_EXTRACT((i), IF_F4_OPF_LOW_SHIFT, (w))
+#define	IF_F4_RS2(i)		IF_F3_RS2((i))
+#define	IF_F4_SW_TRAP(i)	IF_DECODE((i), F4_SW_TRAP)
+
+/* Extract an immediate from an instruction, with an without sign extension */
+#define	IF_IMM(i, w)	IF_EXTRACT((i), IF_IMM_SHIFT, (w))
+#define	IF_SIMM(i, w)	({ u_long b = (w), x = IF_IMM((i), b); IF_SEXT((x), b); })
+
+/*
+ * Macros to encode instructions
+ */
+#define	IF_INSERT(x, s, w)	(((x) & ((1 << (w)) - 1)) << (s))
+#define	IF_ENCODE(x, f) \
+	IF_INSERT((x), IF_ ## f ## _SHIFT, IF_ ## f ## _BITS)
+
+/* All instruction formats */
+#define	EIF_OP(x)		IF_ENCODE((x), OP)
+
+/* Instruction format 2 */
+#define	EIF_F2_RD(x)		IF_ENCODE((x), F2_RD)
+#define	EIF_F2_A(x)		IF_ENCODE((x), F2_A)
+#define	EIF_F2_COND(x)		IF_ENCODE((x), F2_COND)
+#define	EIF_F2_RCOND(x)		IF_ENCODE((x), F2_RCOND)
+#define	EIF_F2_OP2(x)		IF_ENCODE((x), F2_OP2)
+#define	EIF_F2_CC1(x)		IF_ENCODE((x), F2_CC1)
+#define	EIF_F2_CC0(x)		IF_ENCODE((x), F2_CC0)
+#define	EIF_F2_D16HI(x)		IF_ENCODE((x), F2_D16HI)
+#define	EIF_F2_P(x)		IF_ENCODE((x), F2_P)
+#define	EIF_F2_RS1(x)		IF_ENCODE((x), F2_RS1)
+
+/* Instruction format 3 */
+#define	EIF_F3_OP3(x)		IF_ENCODE((x), F3_OP3)
+#define	EIF_F3_RD(x)		EIF_F2_RD((x))
+#define	EIF_F3_FCN(x)		IF_ENCODE((x), F3_FCN)
+#define	EIF_F3_CC1(x)		IF_ENCODE((x), F3_CC1)
+#define	EIF_F3_CC0(x)		IF_ENCODE((x), F3_CC0)
+#define	EIF_F3_RS1(x)		EIF_F2_RS1((x))
+#define	EIF_F3_I(x)		IF_ENCODE((x), F3_I)
+#define	EIF_F3_X(x)		IF_ENCODE((x), F3_X)
+#define	EIF_F3_RCOND(x)		IF_ENCODE((x), F3_RCOND)
+#define	EIF_F3_IMM_ASI(x)	IF_ENCODE((x), F3_IMM_ASI)
+#define	EIF_F3_OPF(x)		IF_ENCODE((x), F3_OPF)
+#define	EIF_F3_CMASK(x)		IF_ENCODE((x), F3_CMASK)
+#define	EIF_F3_RS2(x)		IF_ENCODE((x), F3_RS2)
+#define	EIF_F3_SHCNT32(x)	IF_ENCODE((x), F3_SHCNT32)
+#define	EIF_F3_SHCNT64(x)	IF_ENCODE((x), F3_SHCNT64)
+
+/* Instruction format 4 */
+#define	EIF_F4_OP3(x)		EIF_F3_OP3((x))
+#define	EIF_F4_RD(x)		EIF_F2_RD((x))
+#define	EIF_F4_TCOND(x)		IF_ENCODE((x), F4_TCOND)
+#define	EIF_F4_RS1(x)		EIF_F2_RS1((x))
+#define	EIF_F4_CC2(x)		IF_ENCODE((x), F4_CC2)
+#define	EIF_F4_COND(x)		IF_ENCODE((x), F4_COND)
+#define	EIF_F4_I(x)		EIF_F3_I((x))
+#define	EIF_F4_OPF_CC(x)	IF_ENCODE((x), F4_OPF_CC)
+#define	EIF_F4_RCOND(x)		EIF_F3_RCOND((x))
+#define	EIF_F4_OPF_LOW(i, w)	IF_INSERT((x), IF_F4_OPF_CC_SHIFT, (w))
+#define	EIF_F4_RS2(x)		EIF_F3_RS2((x))
+#define	EIF_F4_SW_TRAP(x)	IF_ENCODE((x), F4_SW_TRAP)
+
+/* Immediates */
+#define	EIF_IMM(x, w)	IF_INSERT((x), IF_IMM_SHIFT, (w))
+#define	EIF_SIMM(x, w)	IF_EIMM((x), (w))
+
+/*
+ * OP field values (specifying the instruction format)
+ */
+#define	IOP_FORM2		0x00	/* Format 2: sethi, branches */
+#define	IOP_CALL		0x01	/* Format 1: call */
+#define	IOP_MISC		0x02	/* Format 3 or 4: arith & misc */
+#define	IOP_LDST		0x03	/* Format 4: loads and stores */
+
+/*
+ * OP2/OP3 values (specifying the actual instruction)
+ */
+/* OP2 values for format 2 (OP = 0) */
+#define	INS0_ILLTRAP		0x00
+#define	INS0_BPcc		0x01
+#define	INS0_Bicc		0x02
+#define	INS0_BPr		0x03
+#define	INS0_SETHI	       	0x04	/* with rd = 0 and imm22 = 0, nop */
+#define	INS0_FBPfcc		0x05
+#define	INS0_FBfcc		0x06
+/* undefined			0x07 */
+
+/* OP3 values for Format 3 and 4 (OP = 2) */
+#define	INS2_ADD		0x00
+#define	INS2_AND		0x01
+#define	INS2_OR			0x02
+#define	INS2_XOR		0x03
+#define	INS2_SUB		0x04
+#define	INS2_ANDN		0x05
+#define	INS2_ORN		0x06
+#define	INS2_XNOR		0x07
+#define	INS2_ADDC		0x08
+#define	INS2_MULX		0x09
+#define	INS2_UMUL		0x0a
+#define	INS2_SMUL		0x0b
+#define	INS2_SUBC		0x0c
+#define	INS2_UDIVX		0x0d
+#define	INS2_UDIV		0x0e
+#define	INS2_SDIV		0x0f
+#define	INS2_ADDcc		0x10
+#define	INS2_ANDcc		0x11
+#define	INS2_ORcc		0x12
+#define	INS2_XORcc		0x13
+#define	INS2_SUBcc		0x14
+#define	INS2_ANDNcc		0x15
+#define	INS2_ORNcc		0x16
+#define	INS2_XNORcc		0x17
+#define	INS2_ADDCcc		0x18
+/* undefined			0x19 */
+#define	INS2_UMULcc		0x1a
+#define	INS2_SMULcc		0x1b
+#define	INS2_SUBCcc		0x1c
+/* undefined			0x1d */
+#define	INS2_UDIVcc		0x1e
+#define	INS2_SDIVcc		0x1f
+#define	INS2_TADDcc		0x20
+#define	INS2_TSUBcc		0x21
+#define	INS2_TADDccTV		0x22
+#define	INS2_TSUBccTV		0x23
+#define	INS2_MULScc		0x24
+#define	INS2_SSL		0x25	/* SLLX when IF_X(i) == 1 */
+#define	INS2_SRL		0x26	/* SRLX when IF_X(i) == 1 */
+#define	INS2_SRA		0x27	/* SRAX when IF_X(i) == 1 */
+#define	INS2_RD			0x28	/* and MEMBAR, STBAR */
+/* undefined			0x29 */
+#define	INS2_RDPR		0x2a
+#define	INS2_FLUSHW		0x2b
+#define	INS2_MOVcc		0x2c
+#define	INS2_SDIVX		0x2d
+#define	INS2_POPC		0x2e	/* undefined if IF_RS1(i) != 0 */
+#define	INS2_MOVr		0x2f
+#define	INS2_WR			0x30	/* and SIR */
+#define	INS2_SV_RSTR		0x31	/* saved, restored */
+#define	INS2_WRPR		0x32
+/* undefined			0x33 */
+#define	INS2_FPop1		0x34	/* further encoded in opf field */
+#define	INS2_FPop2		0x35	/* further encoded in opf field */
+#define	INS2_IMPLDEP1		0x36
+#define	INS2_IMPLDEP2		0x37
+#define	INS2_JMPL		0x38
+#define	INS2_RETURN		0x39
+#define	INS2_Tcc		0x3a
+#define	INS2_FLUSH		0x3b
+#define	INS2_SAVE		0x3c
+#define	INS2_RESTORE		0x3d
+#define	INS2_DONE_RETR		0x3e	/* done, retry */
+/* undefined			0x3f */
+
+/* OP3 values for format 3 (OP = 3) */
+#define	INS3_LDUW		0x00
+#define	INS3_LDUB		0x01
+#define	INS3_LDUH		0x02
+#define	INS3_LDD		0x03
+#define	INS3_STW		0x04
+#define	INS3_STB		0x05
+#define	INS3_STH		0x06
+#define	INS3_STD		0x07
+#define	INS3_LDSW		0x08
+#define	INS3_LDSB		0x09
+#define	INS3_LDSH		0x0a
+#define	INS3_LDX		0x0b
+/* undefined			0x0c */
+#define	INS3_LDSTUB		0x0d
+#define	INS3_STX		0x0e
+#define	INS3_SWAP		0x0f
+#define	INS3_LDUWA		0x10
+#define	INS3_LDUBA		0x11
+#define	INS3_LDUHA		0x12
+#define	INS3_LDDA		0x13
+#define	INS3_STWA		0x14
+#define	INS3_STBA		0x15
+#define	INS3_STHA		0x16
+#define	INS3_STDA		0x17
+#define	INS3_LDSWA		0x18
+#define	INS3_LDSBA		0x19
+#define	INS3_LDSHA		0x1a
+#define	INS3_LDXA		0x1b
+/* undefined			0x1c */
+#define	INS3_LDSTUBA		0x1d
+#define	INS3_STXA		0x1e
+#define	INS3_SWAPA		0x1f
+#define	INS3_LDF		0x20
+#define	INS3_LDFSR		0x21	/* and LDXFSR */
+#define	INS3_LDQF		0x22
+#define	INS3_LDDF		0x23
+#define	INS3_STF		0x24
+#define	INS3_STFSR		0x25	/* and STXFSR */
+#define	INS3_STQF		0x26
+#define	INS3_STDF		0x27
+/* undefined			0x28 - 0x2c */
+#define	INS3_PREFETCH		0x2d
+/* undefined			0x2e - 0x2f */
+#define	INS3_LDFA		0x30
+/* undefined			0x31 */
+#define	INS3_LDQFA		0x32
+#define	INS3_LDDFA		0x33
+#define	INS3_STFA		0x34
+/* undefined			0x35 */
+#define	INS3_STQFA		0x36
+#define	INS3_STDFA		0x37
+/* undefined			0x38 - 0x3b */
+#define	INS3_CASA		0x39
+#define	INS3_PREFETCHA		0x3a
+#define	INS3_CASXA		0x3b
+
+/*
+ * OPF values (floating point instructions, IMPLDEP)
+ */
+/*
+ * These values are or'ed to the FPop values to get the instructions.
+ * They describe the operand type(s).
+ */
+#define	INSFP_i			0x000	/* 32-bit int */
+#define	INSFP_s			0x001	/* 32-bit single */
+#define	INSFP_d			0x002	/* 64-bit double */
+#define	INSFP_q			0x003	/* 128-bit quad */
+/* FPop1. The comments give the types for which this instruction is defined. */
+#define	INSFP1_FMOV		0x000	/* s, d, q */
+#define	INSFP1_FNEG		0x004	/* s, d, q */
+#define	INSFP1_FABS		0x008	/* s, d, q */
+#define	INSFP1_FSQRT		0x028	/* s, d, q */
+#define	INSFP1_FADD		0x040	/* s, d, q */
+#define	INSFP1_FSUB		0x044	/* s, d, q */
+#define	INSFP1_FMUL		0x048	/* s, d, q */
+#define	INSFP1_FDIV		0x04c	/* s, d, q */
+#define	INSFP1_FsMULd		0x068	/* s */
+#define	INSFP1_FdMULq		0x06c	/* d */
+#define	INSFP1_FTOx		0x080	/* s, d, q */
+#define	INSFP1_FxTOs		0x084	/* special: i only */
+#define	INSFP1_FxTOd		0x088	/* special: i only */
+#define	INSFP1_FxTOq		0x08c	/* special: i only */
+#define	INSFP1_FTOs		0x0c4	/* i, d, q */
+#define	INSFP1_FTOd		0x0c8	/* i, s, q */
+#define	INSFP1_FTOq		0x0cc	/* i, s, d */
+#define	INSFP1_FTOi		0x0d0	/* i, s, d */
+
+/* FPop2 */
+#define	INSFP2_FMOV_CCMUL	0x40
+#define	INSFP2_FMOV_CCOFFS	0x00
+/* Use the IFCC_* constants for cc. Operand types: s, d, q */
+#define	INSFP2_FMOV_CC(cc)	((cc) * INSFP2_FMOV_CCMUL + INSFP2_FMOV_CCOFFS)
+#define	INSFP2_FMOV_RCMUL	0x20
+#define	INSFP2_FMOV_RCOFFS	0x04
+/* Use the IRCOND_* constants for rc. Operand types: s, d, q */
+#define	INSFP2_FMOV_RC(rc)	((rc) * INSFP2_FMOV_RCMUL + INSFP2_FMOV_RCOFFS)
+#define	INSFP2_FCMP		0x050	/* s, d, q */
+#define	INSFP2_FCMPE		0x054	/* s, d, q */
+
+/* Decode 5-bit register field into 6-bit number (for doubles and quads). */
+#define	INSFPdq_RN(rn)		(((rn) & ~1) | (((rn) & 1) << 5))
+
+/* IMPLDEP1 for Sun UltraSparc */
+#define	IIDP1_EDGE8		0x00
+#define	IIDP1_EDGE8N		0x01	/* US-III */
+#define	IIDP1_EDGE8L		0x02
+#define	IIDP1_EDGE8LN		0x03	/* US-III */
+#define	IIDP1_EDGE16		0x04
+#define	IIDP1_EDGE16N		0x05	/* US-III */
+#define	IIDP1_EDGE16L		0x06
+#define	IIDP1_EDGE16LN		0x07	/* US-III */
+#define	IIDP1_EDGE32		0x08
+#define	IIDP1_EDGE32N		0x09	/* US-III */
+#define	IIDP1_EDGE32L		0x0a
+#define	IIDP1_EDGE32LN		0x0b	/* US-III */
+#define	IIDP1_ARRAY8		0x10
+#define	IIDP1_ARRAY16		0x12
+#define	IIDP1_ARRAY32		0x14
+#define	IIDP1_ALIGNADDRESS	0x18
+#define	IIDP1_BMASK		0x19	/* US-III */
+#define	IIDP1_ALIGNADDRESS_L	0x1a
+#define	IIDP1_FCMPLE16		0x20
+#define	IIDP1_FCMPNE16		0x22
+#define	IIDP1_FCMPLE32		0x24
+#define	IIDP1_FCMPNE32		0x26
+#define	IIDP1_FCMPGT16		0x28
+#define	IIDP1_FCMPEQ16		0x2a
+#define	IIDP1_FCMPGT32		0x2c
+#define	IIDP1_FCMPEQ32		0x2e
+#define	IIDP1_FMUL8x16		0x31
+#define	IIDP1_FMUL8x16AU	0x33
+#define	IIDP1_FMUL8X16AL	0x35
+#define	IIDP1_FMUL8SUx16	0x36
+#define	IIDP1_FMUL8ULx16	0x37
+#define	IIDP1_FMULD8SUx16	0x38
+#define	IIDP1_FMULD8ULx16	0x39
+#define	IIDP1_FPACK32		0x3a
+#define	IIDP1_FPACK16		0x3b
+#define	IIDP1_FPACKFIX		0x3d
+#define	IIDP1_PDIST		0x3e
+#define	IIDP1_FALIGNDATA	0x48
+#define	IIDP1_FPMERGE		0x4b
+#define	IIDP1_BSHUFFLE		0x4c	/* US-III */
+#define	IIDP1_FEXPAND		0x4d
+#define	IIDP1_FPADD16		0x50
+#define	IIDP1_FPADD16S		0x51
+#define	IIDP1_FPADD32		0x52
+#define	IIDP1_FPADD32S		0x53
+#define	IIDP1_SUB16		0x54
+#define	IIDP1_SUB16S		0x55
+#define	IIDP1_SUB32		0x56
+#define	IIDP1_SUB32S		0x57
+#define	IIDP1_FZERO		0x60
+#define	IIDP1_FZEROS		0x61
+#define	IIDP1_FNOR		0x62
+#define	IIDP1_FNORS		0x63
+#define	IIDP1_FANDNOT2		0x64
+#define	IIDP1_FANDNOT2S		0x65
+#define	IIDP1_NOT2		0x66
+#define	IIDP1_NOT2S		0x67
+#define	IIDP1_FANDNOT1		0x68
+#define	IIDP1_FANDNOT1S		0x69
+#define	IIDP1_FNOT1		0x6a
+#define	IIDP1_FNOT1S		0x6b
+#define	IIDP1_FXOR		0x6c
+#define	IIDP1_FXORS		0x6d
+#define	IIDP1_FNAND		0x6e
+#define	IIDP1_FNANDS		0x6f
+#define	IIDP1_FAND		0x70
+#define	IIDP1_FANDS		0x71
+#define	IIDP1_FXNOR		0x72
+#define	IIDP1_FXNORS		0x73
+#define	IIDP1_FSRC1		0x74
+#define	IIDP1_FSRC1S		0x75
+#define	IIDP1_FORNOT2		0x76
+#define	IIDP1_FORNOT2S		0x77
+#define	IIDP1_FSRC2		0x78
+#define	IIDP1_FSRC2S		0x79
+#define	IIDP1_FORNOT1		0x7a
+#define	IIDP1_FORNOT1S		0x7b
+#define	IIDP1_FOR		0x7c
+#define	IIDP1_FORS		0x7d
+#define	IIDP1_FONE		0x7e
+#define	IIDP1_FONES		0x7f
+#define	IIDP1_SHUTDOWN		0x80
+#define	IIDP1_SIAM		0x81	/* US-III */
+
+/*
+ * Instruction modifiers
+ */
+/* cond values for integer ccr's */
+#define	IICOND_N		0x00
+#define	IICOND_E		0x01
+#define	IICOND_LE		0x02
+#define	IICOND_L		0x03
+#define	IICOND_LEU		0x04
+#define	IICOND_CS		0x05
+#define	IICOND_NEG		0x06
+#define	IICOND_VS		0x07
+#define	IICOND_A		0x08
+#define	IICOND_NE		0x09
+#define	IICOND_G		0x0a
+#define	IICOND_GE		0x0b
+#define	IICOND_GU		0x0c
+#define	IICOND_CC		0x0d
+#define	IICOND_POS		0x0e
+#define	IICOND_VC		0x0f
+
+/* cond values for fp ccr's */
+#define	IFCOND_N		0x00
+#define	IFCOND_NE		0x01
+#define	IFCOND_LG		0x02
+#define	IFCOND_UL		0x03
+#define	IFCOND_L		0x04
+#define	IFCOND_UG		0x05
+#define	IFCOND_G		0x06
+#define	IFCOND_U		0x07
+#define	IFCOND_A		0x08
+#define	IFCOND_E		0x09
+#define	IFCOND_UE		0x0a
+#define	IFCOND_GE		0x0b
+#define	IFCOND_UGE		0x0c
+#define	IFCOND_LE		0x0d
+#define	IFCOND_ULE		0x0e
+#define	IFCOND_O		0x0f
+
+/* rcond values for BPr, MOVr, FMOVr */
+#define	IRCOND_Z		0x01
+#define	IRCOND_LEZ		0x02
+#define	IRCOND_LZ		0x03
+#define	IRCOND_NZ		0x05
+#define	IRCOND_GZ		0x06
+#define	IRCOND_GEZ		0x07
+
+/* cc values for MOVcc and FMOVcc */
+#define	IFCC_ICC		0x04
+#define	IFCC_XCC		0x06
+/* if true, the lower 2 bits are the fcc number */
+#define	IFCC_FCC(c)		((c) & 3)
+#define	IFCC_GET_FCC(c)		((c) & 3)
+#define	IFCC_ISFCC(c)		(((c) & 4) == 0)
+
+/* cc values for BPc and Tcc */
+#define	IBCC_ICC		0x00
+#define	IBCC_XCC		0x02
+
+/*
+ * Integer registers
+ */
+#define	IREG_G0			0x00
+#define	IREG_O0			0x08
+#define	IREG_L0			0x10
+#define	IREQ_I0			0x18
+
+#endif /* !_MACHINE_INSTR_H_ */
diff --git a/sys/sun4v/include/intr_machdep.h b/sys/sun4v/include/intr_machdep.h
new file mode 100644
index 000000000000..de59b1e4fe32
--- /dev/null
+++ b/sys/sun4v/include/intr_machdep.h
@@ -0,0 +1,105 @@
+/*-
+ * Copyright (c) 2001 Jake Burkholder.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef	_MACHINE_INTR_MACHDEP_H_
+#define	_MACHINE_INTR_MACHDEP_H_
+
+#define	IRSR_BUSY	(1 << 5)
+
+#define	PIL_MAX		(1 << 4)
+#define	IV_MAX		(1 << 11)
+#define IV_NAMLEN	1024
+
+#define	IR_FREE		(PIL_MAX * 2)
+
+#define	IH_SHIFT	PTR_SHIFT
+#define	IQE_SHIFT	5
+#define	IV_SHIFT	5
+
+#define	PIL_LOW		1	/* stray interrupts */
+#define	PIL_ITHREAD	2	/* interrupts that use ithreads */
+#define	PIL_RENDEZVOUS	3	/* smp rendezvous ipi */
+#define	PIL_AST		4	/* ast ipi */
+#define	PIL_STOP	5	/* stop cpu ipi */
+#define	PIL_FAST	13	/* fast interrupts */
+#define	PIL_TICK	14
+
+struct trapframe;
+
+typedef	void ih_func_t(struct trapframe *);
+typedef	void iv_func_t(void *);
+
+struct ithd;
+
+struct intr_request {
+	struct	intr_request *ir_next;
+	iv_func_t *ir_func;
+	void	*ir_arg;
+	u_int	ir_vec;
+	u_int	ir_pri;
+};
+
+struct intr_vector {
+	iv_func_t *iv_func;
+	void	*iv_arg;
+	struct	intr_event *iv_event;
+	u_int	iv_pri;
+	u_int	iv_vec;
+};
+
+extern ih_func_t *intr_handlers[];
+extern struct intr_vector intr_vectors[];
+
+void	intr_setup(int level, ih_func_t *ihf, int pri, iv_func_t *ivf,
+		   void *iva);
+int	inthand_add(const char *name, int vec, void (*handler)(void *),
+    void *arg, int flags, void **cookiep);
+int	inthand_remove(int vec, void *cookie);
+void cpu_intrq_init(void);
+
+
+ih_func_t intr_fast;
+
+#define CPU_LIST_SIZE  		(MAXCPU * sizeof(uint16_t)) 
+
+#define INTR_REPORT_SIZE        64
+#define INTR_CPU_Q_SIZE         (cpu_q_entries * INTR_REPORT_SIZE)
+#define INTR_DEV_Q_SIZE         (dev_q_entries * INTR_REPORT_SIZE)
+
+#define CPU_RQ_ENTRIES          64
+#define CPU_NRQ_ENTRIES         64
+
+#define Q_ENTRY_SIZE            64
+#define CPU_RQ_SIZE             (CPU_RQ_ENTRIES * Q_ENTRY_SIZE)
+#define CPU_NRQ_SIZE            (CPU_NRQ_ENTRIES * Q_ENTRY_SIZE)
+
+
+
+
+
+#endif
diff --git a/sys/sun4v/include/iommureg.h b/sys/sun4v/include/iommureg.h
new file mode 100644
index 000000000000..e36d837f7a29
--- /dev/null
+++ b/sys/sun4v/include/iommureg.h
@@ -0,0 +1,180 @@
+/*-
+ * Copyright (c) 1992, 1993
+ *	The Regents of the University of California.  All rights reserved.
+ *
+ * This software was developed by the Computer Systems Engineering group
+ * at Lawrence Berkeley Laboratory under DARPA contract BG 91-66 and
+ * contributed to Berkeley.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 4. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	from: @(#)sbusreg.h	8.1 (Berkeley) 6/11/93
+ *	from: NetBSD: iommureg.h,v 1.6 2001/07/20 00:07:13 eeh Exp
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _MACHINE_IOMMUREG_H_
+#define _MACHINE_IOMMUREG_H_
+
+/*
+ * UltraSPARC IOMMU registers, common to both the sbus and PCI
+ * controllers.
+ */
+
+/* iommmu registers */
+#define	IMR_CTL		0x0000	/* IOMMU control register */
+#define	IMR_TSB		0x0008	/* IOMMU TSB base register */
+#define	IMR_FLUSH	0x0010	/* IOMMU flush register */
+
+/* streaming buffer registers */
+#define	ISR_CTL		0x0000	/* streaming buffer control reg */
+#define	ISR_PGFLUSH	0x0008	/* streaming buffer page flush */
+#define	ISR_FLUSHSYNC	0x0010	/* streaming buffer flush sync */
+
+/* streaming buffer diagnostics registers. */
+#define	ISD_DATA_DIAG	0x0000	/* streaming buffer data RAM diag 0..127 */
+#define	ISD_ERROR_DIAG	0x0400	/* streaming buffer error status diag 0..127 */
+#define	ISD_PG_TAG_DIAG	0x0800	/* streaming buffer page tag diag 0..15 */
+#define	ISD_LN_TAG_DIAG	0x0900	/* streaming buffer line tag diag 0..15 */
+
+/* streaming buffer control register */
+#define STRBUF_EN		0x0000000000000001UL
+#define STRBUF_D		0x0000000000000002UL
+
+#define	IOMMU_BITS		34
+#define	IOMMU_MAXADDR		(1UL << IOMMU_BITS)
+
+/*
+ * control register bits
+ */
+/* Nummber of entries in IOTSB */
+#define	IOMMUCR_TSBSZ_SHIFT	16
+#define IOMMUCR_TSB1K		0x0000000000000000UL
+#define IOMMUCR_TSB2K		0x0000000000010000UL
+#define IOMMUCR_TSB4K		0x0000000000020000UL
+#define IOMMUCR_TSB8K		0x0000000000030000UL
+#define IOMMUCR_TSB16K		0x0000000000040000UL
+#define IOMMUCR_TSB32K		0x0000000000050000UL
+#define IOMMUCR_TSB64K		0x0000000000060000UL
+#define IOMMUCR_TSB128K		0x0000000000070000UL
+/* Mask for above */
+#define IOMMUCR_TSBMASK		0xfffffffffff8ffffUL
+/* 8K iommu page size */
+#define IOMMUCR_8KPG		0x0000000000000000UL
+/* 64K iommu page size */
+#define IOMMUCR_64KPG		0x0000000000000004UL
+/* Diag enable */
+#define IOMMUCR_DE		0x0000000000000002UL
+/* Enable IOMMU */
+#define IOMMUCR_EN		0x0000000000000001UL
+
+/*
+ * Diagnostic register definitions.
+ */
+#define	IOMMU_DTAG_VPNBITS	19
+#define	IOMMU_DTAG_VPNMASK	((1 << IOMMU_DTAG_VPNBITS) - 1)
+#define	IOMMU_DTAG_VPNSHIFT	13
+#define IOMMU_DTAG_ERRBITS	3
+#define	IOMMU_DTAG_ERRSHIFT	22
+#define	IOMMU_DTAG_ERRMASK \
+	(((1 << IOMMU_DTAG_ERRBITS) - 1) << IOMMU_DTAG_ERRSHIFT)
+
+#define	IOMMU_DDATA_PGBITS	21
+#define	IOMMU_DDATA_PGMASK	((1 << IOMMU_DDATA_PGBITS) - 1)
+#define	IOMMU_DDATA_PGSHIFT	13
+#define	IOMMU_DDATA_C		(1 << 28)
+#define	IOMMU_DDATA_V		(1 << 30)
+
+/*
+ * IOMMU stuff
+ */
+/* Entry valid */
+#define	IOTTE_V			0x8000000000000000UL
+/* 8K or 64K page? */
+#define IOTTE_64K		0x2000000000000000UL
+#define IOTTE_8K		0x0000000000000000UL
+/* Is page streamable? */
+#define IOTTE_STREAM		0x1000000000000000UL
+/* Accesses to same bus segment? */
+#define	IOTTE_LOCAL		0x0800000000000000UL
+/* Let's assume this is correct */
+#define IOTTE_PAMASK		0x000001ffffffe000UL
+/* Accesses to cacheable space */
+#define IOTTE_C			0x0000000000000010UL
+/* Writeable */
+#define IOTTE_W			0x0000000000000002UL
+
+/* log2 of the IOMMU TTE size. */
+#define	IOTTE_SHIFT		3
+
+/* Streaming buffer line size. */
+#define	STRBUF_LINESZ		64
+
+/*
+ * Number of bytes written by a stream buffer flushsync operation to indicate
+ * completion.
+ */
+#define	STRBUF_FLUSHSYNC_NBYTES	STRBUF_LINESZ
+
+/*
+ * On sun4u each bus controller has a separate IOMMU.  The IOMMU has 
+ * a TSB which must be page aligned and physically contiguous.  Mappings
+ * can be of 8K IOMMU pages or 64K IOMMU pages.  We use 8K for compatibility
+ * with the CPU's MMU.
+ *
+ * On sysio, psycho, and psycho+, IOMMU TSBs using 8K pages can map the
+ * following size segments:
+ *
+ *	VA size		VA base		TSB size	tsbsize
+ *	--------	--------	---------	-------
+ *	8MB		ff800000	8K		0
+ *	16MB		ff000000	16K		1
+ *	32MB		fe000000	32K		2
+ *	64MB		fc000000	64K		3
+ *	128MB		f8000000	128K		4
+ *	256MB		f0000000	256K		5
+ *	512MB		e0000000	512K		6
+ *	1GB		c0000000	1MB		7
+ *
+ * Unfortunately, sabres on UltraSPARC IIi and IIe processors does not use
+ * this scheme to determine the IOVA base address.  Instead, bits 31-29 are
+ * used to check against the Target Address Space register in the IIi and
+ * the the IOMMU is used if they hit.  God knows what goes on in the IIe.
+ *
+ */
+
+#define	IOTSB_BASESZ		(1024 << IOTTE_SHIFT)
+#define IOTSB_VEND		(~IO_PAGE_MASK)
+#define IOTSB_VSTART(sz)	(u_int)(IOTSB_VEND << ((sz) + 10)) 
+
+#define MAKEIOTTE(pa,w,c,s)						\
+	(((pa) & IOTTE_PAMASK) | ((w) ? IOTTE_W : 0) |			\
+	((c) ? IOTTE_C : 0) | ((s) ? IOTTE_STREAM : 0) |		\
+	(IOTTE_V | IOTTE_8K))
+#define IOTSBSLOT(va)						\
+	((u_int)(((vm_offset_t)(va)) - (is->is_dvmabase)) >> IO_PAGE_SHIFT)
+
+#endif /* !_MACHINE_IOMMUREG_H_ */
diff --git a/sys/sun4v/include/iommuvar.h b/sys/sun4v/include/iommuvar.h
new file mode 100644
index 000000000000..9c290c0748cb
--- /dev/null
+++ b/sys/sun4v/include/iommuvar.h
@@ -0,0 +1,99 @@
+/*-
+ * Copyright (c) 1999 Matthew R. Green
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The name of the author may not be used to endorse or promote products
+ *    derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	from: NetBSD: iommuvar.h,v 1.9 2001/07/20 00:07:13 eeh Exp
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _MACHINE_IOMMUVAR_H_
+#define _MACHINE_IOMMUVAR_H_
+
+#define	IO_PAGE_SIZE		PAGE_SIZE_8K
+#define	IO_PAGE_MASK		PAGE_MASK_8K
+#define	IO_PAGE_SHIFT		PAGE_SHIFT_8K
+#define	round_io_page(x)	round_page(x)
+#define	trunc_io_page(x)	trunc_page(x)
+
+/*
+ * Per-IOMMU state. The parenthesized comments indicate the locking strategy:
+ *	i - protected by iommu_mtx.
+ *	r - read-only after initialization.
+ *	* - comment refers to pointer target / target hardware registers
+ *	    (for bus_addr_t).
+ * iommu_map_lruq is also locked by iommu_mtx. Elements of iommu_tsb may only
+ * be accessed from functions operating on the map owning the corresponding
+ * resource, so the locking the user is required to do to protect the map is
+ * sufficient. As soon as the TSBs are divorced, these will be moved into struct
+ * iommu_state, and each state struct will get its own lock.
+ * iommu_dvma_rman needs to be moved there too, but has its own internal lock.
+ */
+struct iommu_state {
+	int			is_tsbsize;	/* (r) 0 = 8K, ... */
+	u_int64_t		is_dvmabase;	/* (r) */
+	int64_t			is_cr;		/* (r) Control reg value */
+
+	vm_paddr_t		is_flushpa[2];	/* (r) */
+	volatile int64_t	*is_flushva[2];	/* (r, *i) */
+	/*
+	 * (i)
+	 * When a flush is completed, 64 bytes will be stored at the given
+	 * location, the first double word being 1, to indicate completion.
+	 * The lower 6 address bits are ignored, so the addresses need to be
+	 * suitably aligned; over-allocate a large enough margin to be able
+	 * to adjust it.
+	 * Two such buffers are needed.
+	 */
+	volatile char		is_flush[STRBUF_FLUSHSYNC_NBYTES * 3 - 1];
+
+	/* copies of our parents state, to allow us to be self contained */
+	bus_space_tag_t		is_bustag;	/* (r) Our bus tag */
+	bus_space_handle_t	is_bushandle;	/* (r) */
+	bus_addr_t		is_iommu;	/* (r, *i) IOMMU registers */
+	bus_addr_t		is_sb[2];	/* (r, *i) Streaming buffer */
+	/* Tag diagnostics access */
+	bus_addr_t		is_dtag;	/* (r, *r) */
+	/* Data RAM diagnostic access */
+	bus_addr_t		is_ddram;	/* (r, *r) */
+	/* LRU queue diag. access */
+	bus_addr_t		is_dqueue;	/* (r, *r) */
+	/* Virtual address diagnostics register */
+	bus_addr_t		is_dva;		/* (r, *r) */
+	/* Tag compare diagnostics access */
+	bus_addr_t		is_dtcmp;	/* (r, *r) */
+
+	STAILQ_ENTRY(iommu_state)	is_link;	/* (r) */
+};
+
+/* interfaces for PCI/SBUS code */
+void iommu_init(char *, struct iommu_state *, int, u_int32_t, int);
+void iommu_reset(struct iommu_state *);
+void iommu_decode_fault(struct iommu_state *, vm_offset_t);
+
+extern struct bus_dma_methods iommu_dma_methods;
+
+#endif /* !_MACHINE_IOMMUVAR_H_ */
diff --git a/sys/sun4v/include/kdb.h b/sys/sun4v/include/kdb.h
new file mode 100644
index 000000000000..e5b39c04d2bb
--- /dev/null
+++ b/sys/sun4v/include/kdb.h
@@ -0,0 +1,50 @@
+/*-
+ * Copyright (c) 2004 Marcel Moolenaar
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _MACHINE_KDB_H_
+#define _MACHINE_KDB_H_
+
+#include <machine/cpufunc.h>
+
+static __inline void
+kdb_cpu_clear_singlestep(void)
+{
+}
+
+static __inline void
+kdb_cpu_set_singlestep(void)
+{
+}
+
+static __inline void
+kdb_cpu_trap(int vector, int _)
+{
+	flushw();
+}
+
+#endif /* _MACHINE_KDB_H_ */
diff --git a/sys/sun4v/include/kerneldump.h b/sys/sun4v/include/kerneldump.h
new file mode 100644
index 000000000000..7fad3fce88dd
--- /dev/null
+++ b/sys/sun4v/include/kerneldump.h
@@ -0,0 +1,51 @@
+/*-
+ * Copyright (c) 2002 by Thomas Moestl <tmm@FreeBSD.org>.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+ * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _MACHINE_KERNELDUMP_H_
+#define	_MACHINE_KERNELDUMP_H_
+
+struct sparc64_dump_reg {
+	vm_paddr_t	dr_pa;
+	vm_offset_t	dr_size;
+	vm_offset_t	dr_offs;
+};
+
+/*
+ * Kernel dump format for sparc64. This does not use ELF because it is of no
+ * avail (only libkvm knows how to translate addresses properly anyway) and
+ * would require some ugly hacks.
+ */
+struct sparc64_dump_hdr {
+	vm_offset_t	dh_hdr_size;
+	vm_paddr_t	dh_tsb_pa;
+	vm_size_t	dh_tsb_size;
+	vm_size_t	dh_tsb_mask;
+	int		dh_nregions;
+	struct sparc64_dump_reg	dh_regions[];
+};
+
+#endif /* _MACHINE_KERNELDUMP_H_ */
diff --git a/sys/sun4v/include/ktr.h b/sys/sun4v/include/ktr.h
new file mode 100644
index 000000000000..10e33cf0386b
--- /dev/null
+++ b/sys/sun4v/include/ktr.h
@@ -0,0 +1,95 @@
+/*-
+ * Copyright (c) 1996 Berkeley Software Design, Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Berkeley Software Design Inc's name may not be used to endorse or
+ *    promote products derived from this software without specific prior
+ *    written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	from BSDI $Id: ktr.h,v 1.10.2.7 2000/03/16 21:44:42 cp Exp $
+ * $FreeBSD$
+ */
+
+#ifndef _MACHINE_KTR_H_
+#define _MACHINE_KTR_H_
+
+#include <sys/ktr.h>
+
+#include <machine/upa.h>
+
+#ifndef LOCORE
+
+#define	KTR_CPU	PCPU_GET(cpuid)
+	
+#else
+
+#define	AND(var, mask, r1, r2) \
+	SET(var, r2, r1) ; \
+	lduw	[r1], r2 ; \
+	and	r2, mask, r1
+
+#define	TEST(var, mask, r1, r2, l1) \
+	AND(var, mask, r1, r2) ; \
+	brz	r1, l1 ## f ; \
+	 nop
+
+/*
+ * XXX could really use another register...
+ */
+#define	ATR(desc, r1, r2, r3, l1, l2) \
+	.sect	.rodata ; \
+l1:	.asciz	desc ; \
+	.previous ; \
+	SET(ktr_idx, r2, r1) ; \
+	lduw	[r1], r2 ; \
+l2:	add	r2, 1, r3 ; \
+	set	KTR_ENTRIES - 1, r1 ; \
+	and	r3, r1, r3 ; \
+	set	ktr_idx, r1 ; \
+	casa	[r1] ASI_N, r2, r3 ; \
+	cmp	r2, r3 ; \
+	bne	%icc, l2 ## b ; \
+	 mov	r3, r2 ; \
+	SET(ktr_buf, r3, r1) ; \
+	mulx	r2, KTR_SIZEOF, r2 ; \
+	add	r1, r2, r1 ; \
+	rd	%tick, r2 ; \
+	stx	r2, [r1 + KTR_TIMESTAMP] ; \
+	UPA_GET_MID(r2) ; \
+	stw	r2, [r1 + KTR_CPU] ; \
+	stw	%g0, [r1 + KTR_LINE] ; \
+	stx	%g0, [r1 + KTR_FILE] ; \
+	SET(l1 ## b, r3, r2) ; \
+	stx	r2, [r1 + KTR_DESC]
+
+#define CATR(mask, desc, r1, r2, r3, l1, l2, l3) \
+	set	mask, r1 ; \
+	TEST(ktr_mask, r1, r2, r2, l3) ; \
+	UPA_GET_MID(r1) ; \
+	mov	1, r2 ; \
+	sllx	r2, r1, r1 ; \
+	TEST(ktr_cpumask, r1, r2, r3, l3) ; \
+	ATR(desc, r1, r2, r3, l1, l2)
+
+#endif /* LOCORE */
+
+#endif /* !_MACHINE_KTR_H_ */
diff --git a/sys/sun4v/include/limits.h b/sys/sun4v/include/limits.h
new file mode 100644
index 000000000000..679dcec51b31
--- /dev/null
+++ b/sys/sun4v/include/limits.h
@@ -0,0 +1,41 @@
+/*-
+ * Copyright (c) 1988, 1993
+ *	The Regents of the University of California.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	@(#)limits.h	8.3 (Berkeley) 1/4/94
+ * $FreeBSD$
+ */
+
+#ifndef _MACHINE_LIMITS_H_
+#define	_MACHINE_LIMITS_H_
+
+#include <sys/cdefs.h>
+
+#if __CC_SUPPORTS_WARNING
+#warning "machine/limits.h is deprecated.  Include sys/limits.h instead."
+#endif
+
+#include <sys/limits.h>
+
+#endif /* !_MACHINE_LIMITS_H_ */
diff --git a/sys/sun4v/include/lsu.h b/sys/sun4v/include/lsu.h
new file mode 100644
index 000000000000..a8787dcabdc7
--- /dev/null
+++ b/sys/sun4v/include/lsu.h
@@ -0,0 +1,68 @@
+/*-
+ * Copyright 2001 by Thomas Moestl <tmm@FreeBSD.org>.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+ * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef	_MACHINE_LSU_H_
+#define	_MACHINE_LSU_H_
+
+/*
+ * Definitions for the Load-Store-Unit Control Register. This is called
+ * Data Cache Unit Control Register (DCUCR) for UltraSPARC-III.
+ */
+#define	LSU_IC		(1UL << 0)
+#define	LSU_DC		(1UL << 1)
+#define	LSU_IM		(1UL << 2)
+#define	LSU_DM		(1UL << 3)
+
+/* Parity control mask, UltraSPARC-I and II series only. */
+#define	LSU_FM_SHIFT	4
+#define	LSU_FM_BITS	16
+#define	LSU_FM_MASK	(((1UL << LSU_FM_BITS) - 1) << LSU_FM_SHIFT)
+
+#define LSU_VM_SHIFT	25
+#define	LSU_VM_BITS	8
+#define	LSU_VM_MASK	(((1UL << LSU_VM_BITS) - 1) << LSU_VM_SHIFT)
+
+#define	LSU_PM_SHIFT	33
+#define	LSU_PM_BITS	8
+#define	LSU_PM_MASK	(((1UL << LSU_PM_BITS) - 1) << LSU_PM_SHIFT)
+
+#define	LSU_VW		(1UL << 21)
+#define	LSU_VR		(1UL << 22)
+#define	LSU_PW		(1UL << 23)
+#define	LSU_PR		(1UL << 24)
+
+/* The following bits are valid for the UltraSPARC-III series only. */
+#define	LSU_WE		(1UL << 41)
+#define	LSU_SL		(1UL << 42)
+#define	LSU_SPE		(1UL << 43)
+#define	LSU_HPE		(1UL << 44)
+#define	LSU_PE		(1UL << 45)
+#define	LSU_RE		(1UL << 46)
+#define	LSU_ME		(1UL << 47)
+#define	LSU_CV		(1UL << 48)
+#define	LSU_CP		(1UL << 49)
+
+#endif	/* _MACHINE_LSU_H_ */
diff --git a/sys/sun4v/include/md_var.h b/sys/sun4v/include/md_var.h
new file mode 100644
index 000000000000..9c48897ac81d
--- /dev/null
+++ b/sys/sun4v/include/md_var.h
@@ -0,0 +1,63 @@
+/*-
+ * Copyright (c) 1995 Bruce D. Evans.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of the author nor the names of contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	from: FreeBSD: src/sys/i386/include/md_var.h,v 1.40 2001/07/12
+ * $FreeBSD$
+ */
+
+#ifndef	_MACHINE_MD_VAR_H_
+#define	_MACHINE_MD_VAR_H_
+
+typedef void cpu_block_copy_t(const void *src, void *dst, size_t len);
+typedef void cpu_block_zero_t(void *dst, size_t len);
+
+extern	char	tl0_base[], tl0_end[];
+extern	char	_end[];
+
+extern	long	Maxmem;
+
+extern	vm_offset_t kstack0;
+extern	vm_paddr_t kstack0_phys;
+
+struct	pcpu;
+struct	md_utrap;
+
+void	cpu_identify(u_long vers, u_int clock, u_int id);
+void	cpu_setregs(struct pcpu *pc);
+int	is_physical_memory(vm_paddr_t addr);
+struct md_utrap *utrap_alloc(void);
+void	utrap_free(struct md_utrap *ut);
+struct md_utrap *utrap_hold(struct md_utrap *ut);
+
+
+extern	cpu_block_copy_t *cpu_block_copy;
+extern	cpu_block_zero_t *cpu_block_zero;
+
+
+
+#endif /* !_MACHINE_MD_VAR_H_ */
diff --git a/sys/sun4v/include/mdesc_bus.h b/sys/sun4v/include/mdesc_bus.h
new file mode 100644
index 000000000000..bdc115153efc
--- /dev/null
+++ b/sys/sun4v/include/mdesc_bus.h
@@ -0,0 +1,72 @@
+/*-
+ * Copyright (c) 2006 Kip Macy
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef	_MACHINE_MDESC_BUS_H_
+#define	_MACHINE_MDESC_BUS_H_
+
+#include <sys/bus.h>
+#include <machine/mdesc_bus_subr.h>
+
+#include "mdesc_bus_if.h"
+
+static __inline const char *
+mdesc_bus_get_compat(device_t dev)
+{
+
+	return (MDESC_BUS_GET_COMPAT(device_get_parent(dev), dev));
+}
+
+static __inline const struct mdesc_bus_devinfo *
+mdesc_bus_get_model(device_t dev)
+{
+
+	return (MDESC_BUS_GET_DEVINFO(device_get_parent(dev), dev));
+}
+
+static __inline const char *
+mdesc_bus_get_name(device_t dev)
+{
+
+	return (MDESC_BUS_GET_NAME(device_get_parent(dev), dev));
+}
+
+static __inline const char *
+mdesc_bus_get_type(device_t dev)
+{
+
+	return (MDESC_BUS_GET_TYPE(device_get_parent(dev), dev));
+}
+
+static __inline uint64_t
+mdesc_bus_get_handle(device_t dev)
+{
+
+	return (MDESC_BUS_GET_HANDLE(device_get_parent(dev), dev));
+}
+
+#endif /* !_MACHINE_MDESC_BUS_H_ */
diff --git a/sys/sun4v/include/mdesc_bus_subr.h b/sys/sun4v/include/mdesc_bus_subr.h
new file mode 100644
index 000000000000..d25e0b5aa357
--- /dev/null
+++ b/sys/sun4v/include/mdesc_bus_subr.h
@@ -0,0 +1,47 @@
+/*-
+ * Copyright (c) 2006 Kip Macy
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _MACHINE_MDESC_MDESC_BUS_SUBR_H_
+#define _MACHINE_MDESC_MDESC_BUS_SUBR_H_
+
+#include <sys/bus.h>
+
+#include <machine/cddl/mdesc.h>
+#include <machine/cddl/mdesc_impl.h>
+
+#include "mdesc_bus_if.h"
+
+int     mdesc_bus_gen_setup_devinfo(struct mdesc_bus_devinfo *, mde_cookie_t);
+void    mdesc_bus_gen_destroy_devinfo(struct mdesc_bus_devinfo *);
+
+mdesc_bus_get_compat_t    mdesc_bus_gen_get_compat;
+mdesc_bus_get_name_t      mdesc_bus_gen_get_name;
+mdesc_bus_get_type_t      mdesc_bus_gen_get_type;
+mdesc_bus_get_handle_t    mdesc_bus_gen_get_handle;
+
+#endif /* !_MACHINE_MDESC_MDESC_BUS_SUBR_H_ */
diff --git a/sys/sun4v/include/memdev.h b/sys/sun4v/include/memdev.h
new file mode 100644
index 000000000000..671a6b467d7e
--- /dev/null
+++ b/sys/sun4v/include/memdev.h
@@ -0,0 +1,38 @@
+/*-
+ * Copyright (c) 2004 Mark R V Murray
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer
+ *    in this position and unchanged.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#define CDEV_MAJOR	2
+#define	CDEV_MINOR_MEM	0
+#define	CDEV_MINOR_KMEM	1
+
+d_open_t	memopen;
+d_read_t	memrw;
+#define		memioctl	(d_ioctl_t *)NULL
+#define		memmmap		(d_mmap_t *)NULL
+
+void		dev_mem_md_init(void);
diff --git a/sys/sun4v/include/metadata.h b/sys/sun4v/include/metadata.h
new file mode 100644
index 000000000000..d14e1f150d63
--- /dev/null
+++ b/sys/sun4v/include/metadata.h
@@ -0,0 +1,45 @@
+/*-
+ * Copyright (c) 2001 Jake Burkholder.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _MACHINE_METADATA_H_
+#define	_MACHINE_METADATA_H_
+
+#define	MODINFOMD_ENVP		0x1001
+#define	MODINFOMD_HOWTO		0x1002
+#define	MODINFOMD_KERNEND	0x1003
+#define	MODINFOMD_DTLB_SLOTS	0x1004
+#define	MODINFOMD_ITLB_SLOTS	0x1005
+#define	MODINFOMD_DTLB		0x1006
+#define	MODINFOMD_ITLB		0x1007
+
+struct tlb_entry {
+	vm_offset_t te_pa;
+	vm_offset_t te_va;
+};
+
+#endif /* !_MACHINE_METADATA_H_ */
diff --git a/sys/sun4v/include/mmu.h b/sys/sun4v/include/mmu.h
new file mode 100644
index 000000000000..838213633a9d
--- /dev/null
+++ b/sys/sun4v/include/mmu.h
@@ -0,0 +1,123 @@
+/*-
+ * Copyright (c) 2006 Kip Macy
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+
+#ifndef _MACHINE_MMU_H_
+#define _MACHINE_MMU_H_
+
+
+#define	FAST_IMMU_MISS_TT	0x64
+#define	FAST_DMMU_MISS_TT	0x68
+#define	FAST_PROT_TT		0x6c
+
+/*
+ * Constants defining alternate spaces
+ * and register layouts within them,
+ * and a few other interesting assembly constants.
+ */
+
+/*
+ * vaddr offsets of various registers
+ */
+#define	MMU_PCONTEXT		0x08 /* primary context number */
+#define	MMU_SCONTEXT		0x10 /* secondary context number */
+
+/*
+ * Pseudo Synchronous Fault Status Register Layout
+ *
+ * IMMU and DMMU maintain their own pseudo SFSR Register
+ *
+ * +------------------------------------------------+
+ * |       Reserved       |   Context   |     FT    |
+ * +----------------------|-------------------------+
+ *  63                  32 31         16 15         0
+ *
+ */
+#define	SFSR_FT		0x0000FFFF	/* fault type mask */
+#define	SFSR_CTX	0xFFFF0000	/* fault context mask */
+
+/*
+ * Definition of FT (Fault Type) bit field of sfsr.
+ */
+#define	FT_NONE		0x00
+#define	FT_PRIV		MMFSA_F_PRIV	/* privilege violation */
+#define	FT_SPEC_LD	MMFSA_F_SOPG	/* speculative ld to e page */
+#define	FT_ATOMIC_NC	MMFSA_F_NCATM	/* atomic to nc page */
+#define	FT_ILL_ALT	MMFSA_F_INVASI	/* illegal lda/sta */
+#define	FT_NFO		MMFSA_F_NFO	/* normal access to nfo page */
+#define	FT_RANGE	MMFSA_F_INVVA	/* dmmu or immu address out of range */
+#define	FT_NEW_FMISS	MMFSA_F_FMISS	/* fast miss */
+#define	FT_NEW_FPROT	MMFSA_F_FPROT	/* fast protection */
+#define	FT_NEW_MISS	MMFSA_F_MISS	/* mmu miss */
+#define	FT_NEW_INVRA	MMFSA_F_INVRA	/* invalid RA */
+#define	FT_NEW_PROT	MMFSA_F_PROT	/* protection violation */
+#define	FT_NEW_PRVACT	MMFSA_F_PRVACT	/* privileged action */
+#define	FT_NEW_WPT	MMFSA_F_WPT	/* watchpoint hit */
+#define	FT_NEW_UNALIGN	MMFSA_F_UNALIGN	/* unaligned access */
+#define	FT_NEW_INVPGSZ	MMFSA_F_INVPGSZ	/* invalid page size */
+
+#define	SFSR_FT_SHIFT	0	/* amt. to shift right to get flt type */
+#define	SFSR_CTX_SHIFT	16	/* to shift right to get context */
+#define	X_FAULT_TYPE(x)	(((x) & SFSR_FT) >> SFSR_FT_SHIFT)
+#define	X_FAULT_CTX(x)	(((x) & SFSR_CTX) >> SFSR_CTX_SHIFT)
+
+/*
+ * MMU TAG TARGET register Layout
+ *
+ * +-----+---------+------+-------------------------+
+ * | 000 | context |  --  | virtual address [63:22] |
+ * +-----+---------+------+-------------------------+
+ *  63 61 60	 48 47	42 41			   0
+ */
+#define	TTARGET_CTX_SHIFT	48
+#define	TTARGET_VA_SHIFT	22
+
+
+#define TTARGET_VA_BITS         42
+#define TTARGET_VA_MASK         ((1UL << TTARGET_VA_BITS) - 1)
+
+
+/*
+ * MMU PRIMARY/SECONDARY CONTEXT register
+ */
+#define	CTXREG_CTX_MASK		0x1FFF
+
+/*
+ * The kernel always runs in KCONTEXT, and no user mappings
+ * are ever valid in it (so any user access pagefaults).
+ */
+#define	KCONTEXT	0
+#define CTX_OTHER_SHIFT 16
+
+/*
+ * FLUSH_ADDR is used in the flush instruction to guarantee stores to mmu
+ * registers complete.  It is selected so it won't miss in the tlb.
+ */
+#define	FLUSH_ADDR	(KERNBASE + 2 * PAGE_SIZE_4M)
+
+#endif /* _MACHINE_MMU_H_ */
diff --git a/sys/sun4v/include/mutex.h b/sys/sun4v/include/mutex.h
new file mode 100644
index 000000000000..c9b2e1ddde07
--- /dev/null
+++ b/sys/sun4v/include/mutex.h
@@ -0,0 +1,32 @@
+/*-
+ * Copyright (c) 2001 Jake Burkholder.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef	_MACHINE_MUTEX_H_
+#define	_MACHINE_MUTEX_H_
+
+#endif /* !_MACHINE_MUTEX_H_ */
diff --git a/sys/sun4v/include/nexusvar.h b/sys/sun4v/include/nexusvar.h
new file mode 100644
index 000000000000..5e28696ff799
--- /dev/null
+++ b/sys/sun4v/include/nexusvar.h
@@ -0,0 +1,44 @@
+/*-
+ * Copyright (c) 2001 by Thomas Moestl <tmm@FreeBSD.org>.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+ * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _MACHINE_NEXUSVAR_H_
+#define _MACHINE_NEXUSVAR_H_
+
+extern struct bus_dma_tag nexus_dmatag;
+
+enum nexus_ivars {
+	NEXUS_IVAR_DEVHANDLE,
+};
+
+#define NEXUS_ACCESSOR(var, ivar, type)					\
+	__BUS_ACCESSOR(nexus, var, NEXUS, ivar, type)
+
+NEXUS_ACCESSOR(devhandle,	DEVHANDLE,	uint64_t)
+
+#undef NEXUS_ACCESSOR
+
+#endif /* _MACHINE_NEXUSVAR_H_ */
diff --git a/sys/sun4v/include/ofw_bus.h b/sys/sun4v/include/ofw_bus.h
new file mode 100644
index 000000000000..1e9004e15d77
--- /dev/null
+++ b/sys/sun4v/include/ofw_bus.h
@@ -0,0 +1,53 @@
+/*-
+ * Copyright (c) 2001 - 2003 by Thomas Moestl <tmm@FreeBSD.org>.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+ * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef	_MACHINE_OFW_BUS_H_
+#define	_MACHINE_OFW_BUS_H_
+
+#define	ORIP_NOINT	-1
+#define	ORIR_NOTFOUND	0xffffffff
+
+/*
+ * Other than in Open Firmware calls, the size of a bus cell seems to be
+ * always the same.
+ */
+typedef u_int32_t pcell_t;
+
+struct ofw_bus_iinfo {
+	u_int8_t		*opi_imap;
+	u_int8_t		*opi_imapmsk;
+	int			opi_imapsz;
+	pcell_t			opi_addrc;
+};
+
+void ofw_bus_setup_iinfo(phandle_t, struct ofw_bus_iinfo *, int);
+int ofw_bus_lookup_imap(phandle_t, struct ofw_bus_iinfo *, void *, int,
+    void *, int, void *, int, void *);
+int ofw_bus_search_intrmap(void *, int, void *, int, void *, int, void *,
+    void *, void *, int);
+
+#endif /* !_MACHINE_OFW_BUS_H_ */
diff --git a/sys/sun4v/include/ofw_machdep.h b/sys/sun4v/include/ofw_machdep.h
new file mode 100644
index 000000000000..ac1fcc1ea1ed
--- /dev/null
+++ b/sys/sun4v/include/ofw_machdep.h
@@ -0,0 +1,38 @@
+/*-
+ * Copyright (c) 2001 by Thomas Moestl <tmm@FreeBSD.org>.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+ * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _MACHINE_OFW_MACHDEP_H_
+#define _MACHINE_OFW_MACHDEP_H_
+
+#include <sys/bus.h>
+
+int  OF_decode_addr(phandle_t, int, int *, bus_addr_t *);
+void OF_getetheraddr(device_t, u_char *);
+void cpu_shutdown(void *);
+void openfirmware_exit(void *);
+
+#endif /* _MACHINE_OFW_MACHDEP_H_ */
diff --git a/sys/sun4v/include/ofw_mem.h b/sys/sun4v/include/ofw_mem.h
new file mode 100644
index 000000000000..af9089ed034a
--- /dev/null
+++ b/sys/sun4v/include/ofw_mem.h
@@ -0,0 +1,46 @@
+/*-
+ * Copyright (c) 2001 Jake Burkholder.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _MACHINE_OFW_MEM_H_
+#define	_MACHINE_OFW_MEM_H_
+
+struct ofw_mem_region {
+	vm_paddr_t mr_start;
+	vm_size_t mr_size;
+};
+
+struct ofw_map {
+	vm_offset_t om_start;
+	vm_size_t om_size;
+	u_long	om_tte;
+};
+
+extern	struct ofw_mem_region sparc64_memreg[];
+extern	int sparc64_nmemreg;
+
+#endif
diff --git a/sys/sun4v/include/ofw_nexus.h b/sys/sun4v/include/ofw_nexus.h
new file mode 100644
index 000000000000..59337d2a0f56
--- /dev/null
+++ b/sys/sun4v/include/ofw_nexus.h
@@ -0,0 +1,56 @@
+/*-
+ * Copyright (c) 1998, 1999 Eduardo E. Horvath
+ * Copyright (c) 1999 Matthew R. Green
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The name of the author may not be used to endorse or promote products
+ *    derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	from: NetBSD: psychoreg.h,v 1.8 2001/09/10 16:17:06 eeh Exp
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _MACHINE_OFW_NEXUS_H_
+#define	_MACHINE_OFW_NEXUS_H_
+
+/*
+ * These are the regs used for devices on the nexus. They apply to all of
+ * Fireplane/Safari, JBus and UPA.
+ */
+
+struct nexus_regs {
+	uint32_t	phys_hi;
+	uint32_t	phys_lo;
+	uint32_t	size_hi;
+	uint32_t	size_lo;
+};
+
+#define	NEXUS_REG_PHYS(r) \
+	(((uint64_t)(r)->phys_hi << 32) | (uint64_t)(r)->phys_lo)
+#define	NEXUS_REG_SIZE(r) \
+	(((uint64_t)(r)->size_hi << 32) | (uint64_t)(r)->size_lo)
+
+#endif /* !_MACHINE_OFW_NEXUS_H_ */
+
+
diff --git a/sys/sun4v/include/ofw_upa.h b/sys/sun4v/include/ofw_upa.h
new file mode 100644
index 000000000000..fbe6d396cdc7
--- /dev/null
+++ b/sys/sun4v/include/ofw_upa.h
@@ -0,0 +1,54 @@
+/*-
+ * Copyright (c) 1998, 1999 Eduardo E. Horvath
+ * Copyright (c) 1999 Matthew R. Green
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The name of the author may not be used to endorse or promote products
+ *    derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	from: NetBSD: psychoreg.h,v 1.8 2001/09/10 16:17:06 eeh Exp
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _MACHINE_OFW_UPA_H_
+#define _MACHINE_OFW_UPA_H_
+
+/*
+ * These are the regs and ranges property the psycho uses. They should be
+ * applicable to all UPA devices. XXX: verify this.
+ */
+
+struct upa_regs {
+	u_int32_t	phys_hi;
+	u_int32_t	phys_lo;
+	u_int32_t	size_hi;
+	u_int32_t	size_lo;
+};
+
+#define	UPA_REG_PHYS(r) \
+	(((u_int64_t)(r)->phys_hi << 32) | (u_int64_t)(r)->phys_lo)
+#define	UPA_REG_SIZE(r) \
+	(((u_int64_t)(r)->size_hi << 32) | (u_int64_t)(r)->size_lo)
+
+#endif /* !_MACHINE_OFW_UPA_H_ */
diff --git a/sys/sun4v/include/param.h b/sys/sun4v/include/param.h
new file mode 100644
index 000000000000..6e1d389b304c
--- /dev/null
+++ b/sys/sun4v/include/param.h
@@ -0,0 +1,139 @@
+/*-
+ * Copyright (c) 1990 The Regents of the University of California.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to Berkeley by
+ * William Jolitz.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	from: @(#)param.h	5.8 (Berkeley) 6/28/91
+ * $FreeBSD$
+ */
+
+/*
+ * Machine dependent constants for sparc64.
+ */
+
+/*
+ * Round p (pointer or byte index) up to a correctly-aligned value
+ * for all data types (int, long, ...).   The result is unsigned int
+ * and must be cast to any desired pointer type.
+ */
+#ifndef _ALIGNBYTES
+#define _ALIGNBYTES	0xf
+#endif
+#ifndef _ALIGN
+#define _ALIGN(p)	(((u_long)(p) + _ALIGNBYTES) & ~_ALIGNBYTES)
+#endif
+
+#ifndef _NO_NAMESPACE_POLLUTION
+
+#ifndef _MACHINE_PARAM_H_
+#define	_MACHINE_PARAM_H_
+
+#ifndef MACHINE
+#define MACHINE		"sun4v"
+#endif
+#ifndef MACHINE_ARCH
+#define	MACHINE_ARCH	"sparc64"
+#endif
+#define MID_MACHINE	MID_SPARC64
+
+#ifdef SMP
+#define MAXCPU		32
+#else
+#define MAXCPU		1
+#endif /* SMP */
+
+#define	INT_SHIFT	2
+#define	PTR_SHIFT	3
+
+#define ALIGNBYTES	_ALIGNBYTES
+#define ALIGN(p)	_ALIGN(p)
+
+#define	PAGE_SHIFT_8K	13
+#define	PAGE_SIZE_8K	(1L<<PAGE_SHIFT_8K)
+#define	PAGE_MASK_8K	(PAGE_SIZE_8K-1)
+
+#define	PAGE_SHIFT_64K	16
+#define	PAGE_SIZE_64K	(1L<<PAGE_SHIFT_64K)
+#define	PAGE_MASK_64K	(PAGE_SIZE_64K-1)
+
+#define	PAGE_SHIFT_512K	19
+#define	PAGE_SIZE_512K	(1L<<PAGE_SHIFT_512K)
+#define	PAGE_MASK_512K	(PAGE_SIZE_512K-1)
+
+#define	PAGE_SHIFT_4M	22
+#define	PAGE_SIZE_4M	(1L<<PAGE_SHIFT_4M)
+#define	PAGE_MASK_4M	(PAGE_SIZE_4M-1)
+
+#define	PAGE_SHIFT_256M	28
+#define	PAGE_SIZE_256M	(1L<<PAGE_SHIFT_256M)
+#define	PAGE_MASK_256M	(PAGE_SIZE_256M-1)
+
+
+#define PAGE_SHIFT_MIN	PAGE_SHIFT_8K
+#define PAGE_SIZE_MIN	PAGE_SIZE_8K
+#define PAGE_MASK_MIN	PAGE_MASK_8K
+#define PAGE_SHIFT	PAGE_SHIFT_8K	/* LOG2(PAGE_SIZE) */
+#define PAGE_SIZE	PAGE_SIZE_8K	/* bytes/page */
+#define PAGE_MASK	PAGE_MASK_8K
+#define PAGE_SHIFT_MAX	PAGE_SHIFT_4M
+#define PAGE_SIZE_MAX	PAGE_SIZE_4M
+#define PAGE_MASK_MAX	PAGE_MASK_4M
+
+#ifndef KSTACK_PAGES
+#define KSTACK_PAGES		4	/* pages of kernel stack (with pcb) */
+#endif
+#define KSTACK_GUARD_PAGES	1	/* pages of kstack guard; 0 disables */
+#define PCPU_PAGES		1
+
+/*
+ * Ceiling on size of buffer cache (really only effects write queueing,
+ * the VM page cache is not effected), can be changed via
+ * the kern.maxbcache /boot/loader.conf variable.
+ */
+#ifndef VM_BCACHE_SIZE_MAX
+#define VM_BCACHE_SIZE_MAX      (400 * 1024 * 1024)
+#endif
+
+/*
+ * Mach derived conversion macros
+ */
+#ifndef LOCORE
+#define round_page(x)		(((unsigned long)(x) + PAGE_MASK) & ~PAGE_MASK)
+#define trunc_page(x)		((unsigned long)(x) & ~PAGE_MASK)
+
+#define atop(x)			((unsigned long)(x) >> PAGE_SHIFT)
+#define ptoa(x)			((unsigned long)(x) << PAGE_SHIFT)
+
+#define sparc64_btop(x)		((unsigned long)(x) >> PAGE_SHIFT)
+#define sparc64_ptob(x)		((unsigned long)(x) << PAGE_SHIFT)
+
+#define	pgtok(x)		((unsigned long)(x) * (PAGE_SIZE / 1024))
+#endif /* LOCORE */
+
+
+
+#endif /* !_MACHINE_PARAM_H_ */
+#endif /* !_NO_NAMESPACE_POLLUTION */
diff --git a/sys/sun4v/include/pcb.h b/sys/sun4v/include/pcb.h
new file mode 100644
index 000000000000..f3112576beae
--- /dev/null
+++ b/sys/sun4v/include/pcb.h
@@ -0,0 +1,57 @@
+/*-
+ * Copyright (c) 2001 Jake Burkholder.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef	_MACHINE_PCB_H_
+#define	_MACHINE_PCB_H_
+
+#include <machine/frame.h>
+
+#define	MAXWIN	8
+
+#define	PCB_FEF	(1 << 0)
+
+/* NOTE: pcb_ufp must be aligned on a 64 byte boundary. */
+struct pcb {
+	struct	rwindow pcb_rw[MAXWIN];    /* wbuf for page faults during spill / fill trap handling */ 
+	uint32_t pcb_kfp[64];              /* used for regs in handling kernel floating point exc */
+	uint32_t pcb_ufp[64];              /* used for regs in handling user floating point exc */
+	uint64_t pcb_rwsp[MAXWIN];         /* spbuf sp's for each wbuf */
+	uint64_t pcb_flags;
+	uint64_t pcb_nsaved;               /* number of windows saved in pcb_rw */
+	uint64_t pcb_pc;
+	uint64_t pcb_sp;
+	uint64_t pcb_kstack;               /* pcb's kernel stack */
+	uint64_t pcb_pad[4]; 
+} __aligned(64);
+
+#ifdef _KERNEL
+void	makectx(struct trapframe *, struct pcb *);
+int	savectx(struct pcb *pcb);
+#endif
+
+#endif /* !_MACHINE_PCB_H_ */
diff --git a/sys/sun4v/include/pcpu.h b/sys/sun4v/include/pcpu.h
new file mode 100644
index 000000000000..a337e4f8d11a
--- /dev/null
+++ b/sys/sun4v/include/pcpu.h
@@ -0,0 +1,101 @@
+/*-
+ * Copyright (c) 1999 Luoqi Chen <luoqi@freebsd.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	from: FreeBSD: src/sys/i386/include/globaldata.h,v 1.27 2001/04/27
+ * $FreeBSD$
+ */
+
+#ifndef	_MACHINE_PCPU_H_
+#define	_MACHINE_PCPU_H_
+
+#ifdef _KERNEL
+
+#include <machine/asmacros.h>
+#include <machine/frame.h>
+#include <machine/intr_machdep.h>
+
+#define	ALT_STACK_SIZE	128
+
+struct pmap;
+
+/*
+ * Inside the kernel, the globally reserved register g7 is used to
+ * point at the globaldata structure.
+ */
+#define	PCPU_MD_FIELDS							\
+	struct	intr_request pc_irpool[IR_FREE];			\
+	struct	intr_request *pc_irhead;				\
+	struct	intr_request **pc_irtail;				\
+	struct	intr_request *pc_irfree;				\
+	struct 	pmap *pc_curpmap;					\
+	vm_offset_t pc_addr;						\
+	vm_offset_t pc_tsb;						\
+	vm_offset_t *pc_mondo_data;                                     \
+        vm_offset_t *pc_cpu_list;                                       \
+	vm_offset_t *pc_cpu_q;                                          \
+	vm_offset_t *pc_dev_q;                                          \
+	vm_offset_t *pc_rq;                                             \
+	vm_offset_t *pc_nrq;                                            \
+	vm_paddr_t pc_mondo_data_ra;                                    \
+        vm_paddr_t pc_cpu_list_ra;                                      \
+	vm_paddr_t pc_cpu_q_ra;                                         \
+	uint64_t pc_cpu_q_size;                                         \
+	vm_paddr_t pc_dev_q_ra;                                         \
+	uint64_t pc_dev_q_size;                                         \
+	vm_paddr_t pc_rq_ra;                                            \
+	uint64_t pc_rq_size;                                            \
+	vm_paddr_t pc_nrq_ra;                                           \
+	uint64_t pc_nrq_size;                                           \
+	u_long	pc_tickref;						\
+	u_long  pc_lookup_field;                                        \
+	u_long  pc_last_field;                                          \
+	u_long	pc_tickadj;						\
+	struct rwindow pc_kwbuf;                                        \
+	u_long  pc_kwbuf_sp;                                            \
+	u_int   pc_kwbuf_full;                                          \
+	struct rwindow pc_tsbwbuf[2];                                   \
+        uint64_t pc_caller;                                             \
+        uint32_t pc_tl;                                                 \
+        uint32_t pc_trapid;                                             \
+        uint16_t pc_cpulist[MAXCPU];                                    \
+        uint64_t pad[6];
+
+	/* XXX SUN4V_FIXME - as we access the *_ra and *_size fields in quick
+	 * succession we _really_ want them to be L1 cache line size aligned
+	 * and it is quite possible that we want all of ASI_QUEUE fields to
+	 * be L2 cache aligned - they're surrounded by per-cpu data, so there is
+	 * no possibility of false sharing, but this might help in reducing misses
+	 */
+struct pcpu;
+
+register struct pcpu *pcpup __asm__(__XSTRING(PCPU_REG));
+
+#define	PCPU_GET(member)	(pcpup->pc_ ## member)
+#define	PCPU_PTR(member)	(&pcpup->pc_ ## member)
+#define	PCPU_SET(member,value)	(pcpup->pc_ ## member = (value))
+
+#endif	/* _KERNEL */
+
+#endif	/* !_MACHINE_PCPU_H_ */
diff --git a/sys/sun4v/include/pmap.h b/sys/sun4v/include/pmap.h
new file mode 100644
index 000000000000..c1a69da95d11
--- /dev/null
+++ b/sys/sun4v/include/pmap.h
@@ -0,0 +1,129 @@
+/*-
+ * Copyright (c) 1991 Regents of the University of California.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to Berkeley by
+ * the Systems Programming Group of the University of Utah Computer
+ * Science Department and William Jolitz of UUNET Technologies Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 4. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	from: hp300: @(#)pmap.h 7.2 (Berkeley) 12/16/90
+ *	from: @(#)pmap.h        7.4 (Berkeley) 5/12/91
+ *	from: FreeBSD: src/sys/i386/include/pmap.h,v 1.70 2000/11/30
+ * $FreeBSD$
+ */
+
+#ifndef	_MACHINE_PMAP_H_
+#define	_MACHINE_PMAP_H_
+
+#include <sys/types.h>
+#include <sys/queue.h>
+#include <sys/_lock.h>
+#include <sys/_mutex.h>
+#include <machine/cache.h>
+#include <machine/hypervisorvar.h>
+
+#define	PMAP_CONTEXT_MAX	8192
+
+typedef	struct pmap *pmap_t;
+typedef uint32_t pmap_cpumask_t;
+
+struct pv_entry;
+struct tte_hash;
+
+struct md_page {
+	int pv_list_count;
+	TAILQ_HEAD(, pv_entry) pv_list;
+};
+
+
+struct pmap {
+	uint64_t                pm_context;
+	uint64_t                pm_hashscratch;
+	uint64_t                pm_tsbscratch;
+	vm_paddr_t              pm_tsb_ra;
+
+	struct	mtx             pm_mtx;
+	struct tte_hash        *pm_hash;
+	TAILQ_HEAD(,pv_entry)	pm_pvlist;	/* list of mappings in pmap */
+	struct hv_tsb_info      pm_tsb;
+	pmap_cpumask_t          pm_active;      /* mask of cpus currently using pmap */
+	pmap_cpumask_t          pm_tlbactive;   /* mask of cpus that have used this pmap */
+	struct	pmap_statistics pm_stats;
+};
+
+#define	PMAP_LOCK(pmap)		mtx_lock(&(pmap)->pm_mtx)
+#define	PMAP_LOCK_ASSERT(pmap, type) \
+				mtx_assert(&(pmap)->pm_mtx, (type))
+#define	PMAP_LOCK_DESTROY(pmap)	mtx_destroy(&(pmap)->pm_mtx)
+#define	PMAP_LOCK_INIT(pmap)	mtx_init(&(pmap)->pm_mtx, "pmap", \
+				    NULL, MTX_DEF | MTX_DUPOK)
+#define	PMAP_LOCKED(pmap)	mtx_owned(&(pmap)->pm_mtx)
+#define	PMAP_MTX(pmap)		(&(pmap)->pm_mtx)
+#define	PMAP_TRYLOCK(pmap)	mtx_trylock(&(pmap)->pm_mtx)
+#define	PMAP_UNLOCK(pmap)	mtx_unlock(&(pmap)->pm_mtx)
+
+/*
+ * For each vm_page_t, there is a list of all currently valid virtual
+ * mappings of that page.  An entry is a pv_entry_t, the list is pv_table.
+ */
+
+typedef struct pv_entry {
+	pmap_t  pv_pmap;
+	vm_offset_t	pv_va;
+	TAILQ_ENTRY(pv_entry) pv_list;
+	TAILQ_ENTRY(pv_entry) pv_plist;
+} *pv_entry_t;
+
+#define pmap_page_is_mapped(m)  (!TAILQ_EMPTY(&(m)->md.pv_list))
+
+void	pmap_bootstrap(vm_offset_t ekva);
+vm_paddr_t pmap_kextract(vm_offset_t va);
+
+void    pmap_invalidate_page(pmap_t pmap, vm_offset_t va, int cleartsb);
+void    pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int cleartsb);
+void    pmap_invalidate_all(pmap_t pmap);
+void	pmap_scrub_pages(vm_paddr_t pa, int64_t size);
+
+#define	vtophys(va)	pmap_kextract((vm_offset_t)(va))
+
+extern	struct pmap kernel_pmap_store;
+#define	kernel_pmap	(&kernel_pmap_store)
+extern	vm_paddr_t phys_avail[];
+extern	vm_offset_t virtual_avail;
+extern	vm_offset_t virtual_end;
+extern	vm_paddr_t msgbuf_phys;
+
+static __inline int
+pmap_track_modified(pmap_t pm, vm_offset_t va)
+{
+	if (pm == kernel_pmap)
+		return ((va < kmi.clean_sva) || (va >= kmi.clean_eva));
+	else
+		return (1);
+}
+
+#endif /* !_MACHINE_PMAP_H_ */
diff --git a/sys/sun4v/include/pmc_mdep.h b/sys/sun4v/include/pmc_mdep.h
new file mode 100644
index 000000000000..cf643c7b4f39
--- /dev/null
+++ b/sys/sun4v/include/pmc_mdep.h
@@ -0,0 +1,24 @@
+/*-
+ * This file is in the public domain.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _MACHINE_PMC_MDEP_H_
+#define	_MACHINE_PMC_MDEP_H_
+
+union pmc_md_op_pmcallocate {
+	uint64_t		__pad[4];
+};
+
+/* Logging */
+#define	PMCLOG_READADDR		PMCLOG_READ64
+#define	PMCLOG_EMITADDR		PMCLOG_EMIT64
+
+#if	_KERNEL
+union pmc_md_pmc {
+};
+
+#endif
+
+#endif /* !_MACHINE_PMC_MDEP_H_ */
diff --git a/sys/sun4v/include/proc.h b/sys/sun4v/include/proc.h
new file mode 100644
index 000000000000..bfd1268e433c
--- /dev/null
+++ b/sys/sun4v/include/proc.h
@@ -0,0 +1,54 @@
+/*-
+ * Copyright (c) 1991 Regents of the University of California.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 4. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *      from: @(#)proc.h        7.1 (Berkeley) 5/15/91
+ *	from: FreeBSD: src/sys/i386/include/proc.h,v 1.11 2001/06/29
+ * $FreeBSD$
+ */
+
+#ifndef	_MACHINE_PROC_H_
+#define	_MACHINE_PROC_H_
+
+#include <machine/utrap.h>
+
+struct md_utrap {
+	utrap_entry_t *ut_precise[UT_MAX];	/* must be first */
+	int	ut_refcnt;
+};
+
+struct mdthread {
+	int	md_spinlock_count;	/* (k) */
+	register_t md_saved_pil;	/* (k) */
+};
+
+struct mdproc {
+	struct	md_utrap *md_utrap;
+	void	*md_sigtramp;
+};
+
+#endif /* !_MACHINE_PROC_H_ */
diff --git a/sys/sun4v/include/profile.h b/sys/sun4v/include/profile.h
new file mode 100644
index 000000000000..8779ea59024c
--- /dev/null
+++ b/sys/sun4v/include/profile.h
@@ -0,0 +1,70 @@
+/*-
+ * Copyright (c) 1994, 1995, 1996 Carnegie-Mellon University.
+ * All rights reserved.
+ *
+ * Author: Chris G. Demetriou
+ * 
+ * Permission to use, copy, modify and distribute this software and
+ * its documentation is hereby granted, provided that both the copyright
+ * notice and this permission notice appear in all copies of the
+ * software, derivative works or modified versions, and any portions
+ * thereof, and that both notices appear in supporting documentation.
+ * 
+ * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 
+ * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND 
+ * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
+ * 
+ * Carnegie Mellon requests users of this software to return to
+ *
+ *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
+ *  School of Computer Science
+ *  Carnegie Mellon University
+ *  Pittsburgh PA 15213-3890
+ *
+ * any improvements or extensions that they make and grant Carnegie the
+ * rights to redistribute these changes.
+ *
+ *	from: NetBSD: profile.h,v 1.9 1997/04/06 08:47:37 cgd Exp
+ *	from: FreeBSD: src/sys/alpha/include/profile.h,v 1.4 1999/12/29
+ * $FreeBSD$
+ */
+
+#ifndef _MACHINE_PROFILE_H_
+#define	_MACHINE_PROFILE_H_
+
+#define	FUNCTION_ALIGNMENT 32
+#define	_MCOUNT_DECL	void mcount
+#define	MCOUNT
+
+typedef u_long	fptrdiff_t;
+
+#ifdef _KERNEL
+
+#include <machine/cpufunc.h>
+
+#define	MCOUNT_DECL(s)	register_t s;
+#define	MCOUNT_ENTER(s)	s = rdpr(pil); wrpr(pil, 0, 14)
+#define	MCOUNT_EXIT(s)	wrpr(pil, 0, s)
+
+void bintr(void);
+void btrap(void);
+void eintr(void);
+void user(void);
+
+#define	MCOUNT_FROMPC_USER(pc)					\
+	((pc < (uintfptr_t)VM_MAXUSER_ADDRESS) ? (uintfptr_t)user : pc)
+
+#define	MCOUNT_FROMPC_INTR(pc)					\
+	((pc >= (uintfptr_t)btrap && pc < (uintfptr_t)eintr) ?	\
+	    ((pc >= (uintfptr_t)bintr) ? (uintfptr_t)bintr :	\
+		(uintfptr_t)btrap) : ~0UL)
+
+void	mcount(uintfptr_t frompc, uintfptr_t selfpc);
+
+#else /* !_KERNEL */
+
+typedef u_long	uintfptr_t;
+
+#endif /* _KERNEL */
+
+#endif /* !_MACHINE_PROFILE_H_ */
diff --git a/sys/sun4v/include/pstate.h b/sys/sun4v/include/pstate.h
new file mode 100644
index 000000000000..f0e5b68d6556
--- /dev/null
+++ b/sys/sun4v/include/pstate.h
@@ -0,0 +1,59 @@
+/*-
+ * Copyright (c) 2001 Jake Burkholder.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef	_MACHINE_PSTATE_H_
+#define	_MACHINE_PSTATE_H_
+
+#define	PSTATE_IE		(1<<1)
+#define	PSTATE_PRIV		(1<<2)
+#define	PSTATE_AM		(1<<3)
+#define	PSTATE_PEF		(1<<4)
+#define	PSTATE_RED		(1<<5)
+
+#define	PSTATE_MM_SHIFT		(6)
+#define	PSTATE_MM_SIZE		(2)
+#define	PSTATE_MM_MASK		(((1<<PSTATE_MM_SIZE)-1)<<PSTATE_MM_SHIFT)
+#define	PSTATE_MM_TSO		(0<<PSTATE_MM_SHIFT)
+#define	PSTATE_MM_PSO		(1<<PSTATE_MM_SHIFT)
+#define	PSTATE_MM_RMO		(2<<PSTATE_MM_SHIFT)
+
+#define	PSTATE_TLE		(1<<8)
+#define	PSTATE_CLE		(1<<9)
+
+
+#define	PSTATE_MM		PSTATE_MM_TSO
+#define PSTATE_MASK             ((1<<10)-1)
+#define PSTATE_INTR_DISABLE     (PSTATE_MASK & ~PSTATE_IE)
+
+#define	PSTATE_NORMAL		(PSTATE_MM | PSTATE_PEF | PSTATE_PRIV)
+#define	PSTATE_KERNEL		(PSTATE_NORMAL | PSTATE_IE)
+
+#define	PSTATE_SECURE(pstate) \
+	(((pstate) & ~(PSTATE_AM|PSTATE_MM_MASK)) == (PSTATE_IE|PSTATE_PEF))
+
+#endif /* !_MACHINE_PSTATE_H_ */
diff --git a/sys/sun4v/include/ptrace.h b/sys/sun4v/include/ptrace.h
new file mode 100644
index 000000000000..49b1e1c0ee9a
--- /dev/null
+++ b/sys/sun4v/include/ptrace.h
@@ -0,0 +1,36 @@
+/*-
+ * Copyright (c) 1992, 1993
+ *	The Regents of the University of California.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 4. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	@(#)ptrace.h	8.1 (Berkeley) 6/11/93
+ * $FreeBSD$
+ */
+
+#ifndef	_MACHINE_PTRACE_H_
+#define	_MACHINE_PTRACE_H_
+
+#endif
diff --git a/sys/sun4v/include/reg.h b/sys/sun4v/include/reg.h
new file mode 100644
index 000000000000..d2241861d209
--- /dev/null
+++ b/sys/sun4v/include/reg.h
@@ -0,0 +1,113 @@
+/*-
+ * Copyright (c) 1990 The Regents of the University of California.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to Berkeley by
+ * William Jolitz.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 4. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	from: @(#)reg.h	5.5 (Berkeley) 1/18/91
+ *	from: FreeBSD: src/sys/i386/include/reg.h,v 1.23 2000/09/21
+ * $FreeBSD$
+ */
+
+#ifndef	_MACHINE_REG_H_
+#define	_MACHINE_REG_H_
+
+/*
+ * Register set accessible via /proc/$pid/regs and PT_{SET,GET}REGS.
+ *
+ * NOTE: DO NOT CHANGE THESE STRUCTURES.  The offsets of the fields are
+ * hardcoded in gdb.  Changing them and recompiling doesn't help, the
+ * constants in nm-fbsd.h must also be updated.
+ */
+
+struct reg32 {
+	uint32_t r_global[8];
+	uint32_t r_out[8];
+	uint32_t r_npc;
+	uint32_t r_pc;
+	uint32_t r_psr;
+	uint32_t r_wim;
+	uint32_t r_pad[4];
+};
+
+struct reg {
+	uint64_t r_global[8];
+	uint64_t r_out[8];
+	uint64_t r_fprs;
+	uint64_t r_fsr;
+	uint64_t r_gsr;
+	uint64_t r_level;
+	uint64_t r_pil;
+	uint64_t r_sfar;
+	uint64_t r_sfsr;
+	uint64_t r_tar;
+	uint64_t r_tnpc;
+	uint64_t r_tpc;
+	uint64_t r_tstate;
+	uint64_t r_type;
+	uint64_t r_y;
+	uint64_t r_wstate;
+	uint64_t r_pad[2];
+};
+
+/*
+ * Register set accessible via /proc/$pid/fpregs.
+ */
+
+struct fpreg32 {
+	uint32_t fr_regs[32];
+	uint32_t fr_fsr;
+};
+
+struct fpreg {
+	uint32_t fr_regs[64];	/* our view is 64 32-bit registers */
+	int64_t	fr_fsr;		/* %fsr */
+	int32_t	fr_gsr;		/* %gsr */
+	int32_t fr_pad[1];
+};
+
+/*
+ * Register set accessible via /proc/$pid/dbregs.
+ */
+struct dbreg {
+	int dummy;
+};
+
+#ifdef _KERNEL
+/*
+ * XXX these interfaces are MI, so they should be declared in a MI place.
+ */
+int	fill_regs(struct thread *, struct reg *);
+int	set_regs(struct thread *, struct reg *);
+int	fill_fpregs(struct thread *, struct fpreg *);
+int	set_fpregs(struct thread *, struct fpreg *);
+int	fill_dbregs(struct thread *, struct dbreg *);
+int	set_dbregs(struct thread *, struct dbreg *);
+#endif
+
+#endif /* !_MACHINE_REG_H_ */
diff --git a/sys/sun4v/include/reloc.h b/sys/sun4v/include/reloc.h
new file mode 100644
index 000000000000..6e5053602772
--- /dev/null
+++ b/sys/sun4v/include/reloc.h
@@ -0,0 +1,33 @@
+/*-
+ * Copyright (c) 1998 John Birrell <jb@cimlogic.com.au>.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ *    must display the following acknowledgement:
+ *	This product includes software developed by John Birrell.
+ * 4. Neither the name of the author nor the names of any co-contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY JOHN BIRRELL AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
diff --git a/sys/sun4v/include/resource.h b/sys/sun4v/include/resource.h
new file mode 100644
index 000000000000..229d14f253b0
--- /dev/null
+++ b/sys/sun4v/include/resource.h
@@ -0,0 +1,45 @@
+/*-
+ * Copyright 1998 Massachusetts Institute of Technology
+ *
+ * Permission to use, copy, modify, and distribute this software and
+ * its documentation for any purpose and without fee is hereby
+ * granted, provided that both the above copyright notice and this
+ * permission notice appear in all copies, that both the above
+ * copyright notice and this permission notice appear in all
+ * supporting documentation, and that the name of M.I.T. not be used
+ * in advertising or publicity pertaining to distribution of the
+ * software without specific, written prior permission.  M.I.T. makes
+ * no representations about the suitability of this software for any
+ * purpose.  It is provided "as is" without express or implied
+ * warranty.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY M.I.T. ``AS IS''.  M.I.T. DISCLAIMS
+ * ALL EXPRESS OR IMPLIED WARRANTIES WITH REGARD TO THIS SOFTWARE,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. IN NO EVENT
+ * SHALL M.I.T. BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _MACHINE_RESOURCE_H_
+#define	_MACHINE_RESOURCE_H_	1
+
+/*
+ * Definitions of resource types for Intel Architecture machines
+ * with support for legacy ISA devices and drivers.
+ */
+
+#define	SYS_RES_IRQ	1	/* interrupt lines */
+#define	SYS_RES_DRQ	2	/* isa dma lines */
+#define	SYS_RES_MEMORY	3	/* i/o memory */
+#define	SYS_RES_IOPORT	4	/* i/o ports */
+
+#endif /* !_MACHINE_RESOURCE_H_ */
diff --git a/sys/sun4v/include/runq.h b/sys/sun4v/include/runq.h
new file mode 100644
index 000000000000..6d03646fd539
--- /dev/null
+++ b/sys/sun4v/include/runq.h
@@ -0,0 +1,58 @@
+/*-
+ * Copyright (c) 2001 Jake Burkholder <jake@FreeBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef	_MACHINE_RUNQ_H_
+#define	_MACHINE_RUNQ_H_
+
+#define	RQB_LEN		(1UL)		/* Number of priority status words. */
+#define	RQB_L2BPW	(6UL)		/* Log2(sizeof(rqb_word_t) * NBBY)). */
+#define	RQB_BPW		(1UL<<RQB_L2BPW)	/* Bits in an rqb_word_t. */
+
+#define	RQB_BIT(pri)	(1UL << ((pri) & (RQB_BPW - 1)))
+#define	RQB_WORD(pri)	((pri) >> RQB_L2BPW)
+
+#define	RQB_FFS(word)	(ffs64(word) - 1)
+
+/*
+ * Type of run queue status word.
+ */
+typedef	u_int64_t	rqb_word_t;
+
+static __inline u_long
+ffs64(u_long mask)
+{
+	u_long bit;
+
+	if (mask == 0)
+		return (0);
+	for (bit = 1; (mask & 1UL) == 0; bit++)
+		mask >>= 1UL;
+	return (bit);
+}
+
+#endif
diff --git a/sys/sun4v/include/sc_machdep.h b/sys/sun4v/include/sc_machdep.h
new file mode 100644
index 000000000000..3a492f2383ba
--- /dev/null
+++ b/sys/sun4v/include/sc_machdep.h
@@ -0,0 +1,76 @@
+/*-
+ * Copyright (c) 2003 Jake Burkholder.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef	_MACHINE_SC_MACHDEP_H_
+#define	_MACHINE_SC_MACHDEP_H_
+
+/* Color attributes for foreground text */
+
+#define	FG_BLACK		0x0
+#define	FG_BLUE			0x1
+#define	FG_GREEN		0x2
+#define	FG_CYAN			0x3
+#define	FG_RED			0x4
+#define	FG_MAGENTA		0x5
+#define	FG_BROWN		0x6
+#define	FG_LIGHTGREY		0x7
+#define	FG_DARKGREY		0x8
+#define	FG_LIGHTBLUE		0x9
+#define	FG_LIGHTGREEN		0xa
+#define	FG_LIGHTCYAN		0xb
+#define	FG_LIGHTRED		0xc
+#define	FG_LIGHTMAGENTA		0xd
+#define	FG_YELLOW		0xe
+#define	FG_WHITE		0xf
+#define	FG_BLINK		0x80
+
+/* Color attributes for text background */
+
+#define	BG_BLACK		0x00
+#define	BG_BLUE			0x10
+#define	BG_GREEN		0x20
+#define	BG_CYAN			0x30
+#define	BG_RED			0x40
+#define	BG_MAGENTA		0x50
+#define	BG_BROWN		0x60
+#define	BG_LIGHTGREY		0x70
+#define	BG_DARKGREY		0x80
+#define	BG_LIGHTBLUE		0x90
+#define	BG_LIGHTGREEN		0xa0
+#define	BG_LIGHTCYAN		0xb0
+#define	BG_LIGHTRED		0xc0
+#define	BG_LIGHTMAGENTA		0xd0
+#define	BG_YELLOW		0xe0
+#define	BG_WHITE		0xf0
+
+#define	SC_NORM_ATTR		(FG_BLACK | BG_WHITE)
+#define	SC_NORM_REV_ATTR	(FG_WHITE | BG_BLACK)
+#define	SC_KERNEL_CONS_ATTR	(FG_BLACK | BG_WHITE)
+#define	SC_KERNEL_CONS_REV_ATTR	(FG_WHITE | BG_BLACK)
+
+#endif /* !_MACHINE_SC_MACHDEP_H_ */
diff --git a/sys/sun4v/include/setjmp.h b/sys/sun4v/include/setjmp.h
new file mode 100644
index 000000000000..c38bcabadd65
--- /dev/null
+++ b/sys/sun4v/include/setjmp.h
@@ -0,0 +1,60 @@
+/*-
+ * Copyright (c) 1998 John Birrell <jb@cimlogic.com.au>.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ *    must display the following acknowledgement:
+ *      This product includes software developed by John Birrell.
+ * 4. Neither the name of the author nor the names of any co-contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY JOHN BIRRELL AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	From: FreeBSD: src/sys/i386/include/setjmp.h,v 1.5 2000/10/06
+ * $FreeBSD$
+ */
+
+#ifndef	_MACHINE_SETJMP_H_
+#define	_MACHINE_SETJMP_H_
+
+#include <sys/cdefs.h>
+
+#define	_JBLEN		5
+
+#define	_JB_FP		0
+#define	_JB_PC		1
+#define	_JB_SP		2
+#define	_JB_SIGMASK	3
+#define	_JB_SIGFLAG	5
+
+/*
+ * jmp_buf and sigjmp_buf are encapsulated in different structs to force
+ * compile-time diagnostics for mismatches.  The structs are the same
+ * internally to avoid some run-time errors for mismatches.
+ */
+#if __BSD_VISIBLE || __POSIX_VISIBLE || __XSI_VISIBLE
+typedef	struct _sigjmp_buf { long _sjb[_JBLEN + 1]; } sigjmp_buf[1];
+#endif
+
+typedef	struct _jmp_buf { long _jb[_JBLEN + 1]; } jmp_buf[1];
+
+#endif /* !_MACHINE_SETJMP_H_ */
diff --git a/sys/sun4v/include/sf_buf.h b/sys/sun4v/include/sf_buf.h
new file mode 100644
index 000000000000..b090cb26601d
--- /dev/null
+++ b/sys/sun4v/include/sf_buf.h
@@ -0,0 +1,59 @@
+/*-
+ * Copyright (c) 2003 Alan L. Cox <alc@cs.rice.edu>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _MACHINE_SF_BUF_H_
+#define _MACHINE_SF_BUF_H_
+
+#include <machine/tlb.h>
+
+#include <vm/vm.h>
+#include <vm/vm_param.h>
+#include <vm/vm_page.h>
+/*
+ * On this machine, the only purpose for which sf_buf is used is to implement
+ * an opaque pointer required by the machine-independent parts of the kernel.
+ * That pointer references the vm_page that is "mapped" by the sf_buf.  The
+ * actual mapping is provided by the direct virtual-to-physical mapping.  
+ */
+struct sf_buf;
+
+static __inline vm_offset_t
+sf_buf_kva(struct sf_buf *sf)
+{
+
+	return (TLB_PHYS_TO_DIRECT(VM_PAGE_TO_PHYS((vm_page_t)sf)));
+}
+
+static __inline struct vm_page *
+sf_buf_page(struct sf_buf *sf)
+{
+
+	return ((vm_page_t)sf);
+}
+
+#endif /* !_MACHINE_SF_BUF_H_ */
diff --git a/sys/sun4v/include/sigframe.h b/sys/sun4v/include/sigframe.h
new file mode 100644
index 000000000000..fd40afdd1209
--- /dev/null
+++ b/sys/sun4v/include/sigframe.h
@@ -0,0 +1,40 @@
+/*-
+ * Copyright (c) 1999 Marcel Moolenaar
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer 
+ *    in this position and unchanged.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The name of the author may not be used to endorse or promote products
+ *    derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ *	from:	FreeBSD: src/sys/alpha/include/sigframe.h,v 1.1 1999/09/29
+ * $FreeBSD$
+ */
+
+#ifndef _MACHINE_SIGFRAME_H_
+#define _MACHINE_SIGFRAME_H_ 1
+
+struct sigframe {
+	ucontext_t	sf_uc;
+	siginfo_t	sf_si;
+};
+
+#endif /* _MACHINE_SIGFRAME_H_ */
diff --git a/sys/sun4v/include/signal.h b/sys/sun4v/include/signal.h
new file mode 100644
index 000000000000..d96cef6dd6c8
--- /dev/null
+++ b/sys/sun4v/include/signal.h
@@ -0,0 +1,47 @@
+/*-
+ * Copyright (c) 1986, 1989, 1991, 1993
+ *      The Regents of the University of California.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 4. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *      @(#)signal.h    8.1 (Berkeley) 6/11/93
+ * 	from: FreeBSD: src/sys/i386/include/signal.h,v 1.13 2000/11/09
+ * $FreeBSD$
+ */
+
+#ifndef	_MACHINE_SIGNAL_H_
+#define	_MACHINE_SIGNAL_H_
+
+#include <sys/cdefs.h>
+
+typedef	long sig_atomic_t;
+
+#if __BSD_VISIBLE
+struct sigcontext {
+	int	_dummy;
+};
+#endif /* __BSD_VISIBLE */
+
+#endif /* !_MACHINE_SIGNAL_H_ */
diff --git a/sys/sun4v/include/smp.h b/sys/sun4v/include/smp.h
new file mode 100644
index 000000000000..1b9044567cb3
--- /dev/null
+++ b/sys/sun4v/include/smp.h
@@ -0,0 +1,263 @@
+/*-
+ * Copyright (c) 2001 Jake Burkholder.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef	_MACHINE_SMP_H_
+#define	_MACHINE_SMP_H_
+
+#define	CPU_CLKSYNC		1
+#define	CPU_INIT		2
+#define	CPU_BOOTSTRAP		3
+
+#ifndef	LOCORE
+
+#include <machine/intr_machdep.h>
+#include <machine/tte.h>
+
+#define	IDR_BUSY	(1<<0)
+#define	IDR_NACK	(1<<1)
+
+#define	IPI_AST		PIL_AST
+#define	IPI_RENDEZVOUS	PIL_RENDEZVOUS
+#define	IPI_STOP	PIL_STOP
+
+#define	IPI_RETRIES	5000
+
+struct cpu_start_args {
+	u_int	csa_count;
+	u_int	csa_state;
+	vm_offset_t csa_pcpu;
+	u_int csa_cpuid;
+};
+
+struct ipi_cache_args {
+	u_int	ica_mask;
+	vm_paddr_t ica_pa;
+};
+
+struct ipi_tlb_args {
+	u_int	ita_mask;
+	struct	pmap *ita_pmap;
+	u_long	ita_start;
+	u_long	ita_end;
+};
+#define	ita_va	ita_start
+
+struct pcpu;
+
+extern struct pcb stoppcbs[];
+
+void	cpu_mp_bootstrap(struct pcpu *pc);
+void	cpu_mp_shutdown(void);
+
+void	cpu_ipi_selected(int cpus, uint16_t *cpulist, u_long d0, u_long d1, u_long d2, uint64_t *ackmask);
+void	cpu_ipi_send(u_int mid, u_long d0, u_long d1, u_long d2);
+
+void cpu_ipi_ast(struct trapframe *tf);
+void cpu_ipi_stop(struct trapframe *tf);
+
+void	ipi_selected(u_int cpus, u_int ipi);
+void	ipi_all(u_int ipi);
+void	ipi_all_but_self(u_int ipi);
+
+vm_offset_t mp_tramp_alloc(void);
+void        mp_set_tsb_desc_ra(vm_paddr_t);
+void        mp_add_nucleus_mapping(vm_offset_t, uint64_t);
+extern	struct mtx ipi_mtx;
+extern	struct ipi_cache_args ipi_cache_args;
+extern	struct ipi_tlb_args ipi_tlb_args;
+
+extern	vm_offset_t mp_tramp;
+extern	char *mp_tramp_code;
+extern	u_long mp_tramp_code_len;
+extern	u_long mp_tramp_tte_slots;
+extern	u_long mp_tramp_tsb_desc_ra;
+extern	u_long mp_tramp_func;
+
+extern	void mp_startup(void);
+
+extern	char tl_ipi_cheetah_dcache_page_inval[];
+extern	char tl_ipi_spitfire_dcache_page_inval[];
+extern	char tl_ipi_spitfire_icache_page_inval[];
+
+extern	char tl_ipi_level[];
+extern	char tl_invltlb[];
+extern	char tl_invlctx[];
+extern	char tl_invlpg[];
+extern	char tl_ipi_tlb_context_demap[];
+extern	char tl_ipi_tlb_page_demap[];
+extern	char tl_ipi_tlb_range_demap[];
+
+#ifdef SMP
+
+#if defined(_MACHINE_PMAP_H_) && defined(_SYS_MUTEX_H_)
+#if 0
+static __inline void *
+ipi_dcache_page_inval(void *func, vm_paddr_t pa)
+{
+	struct ipi_cache_args *ica;
+
+	if (smp_cpus == 1)
+		return (NULL);
+	ica = &ipi_cache_args;
+	mtx_lock_spin(&ipi_mtx);
+	ica->ica_mask = all_cpus;
+	ica->ica_pa = pa;
+	cpu_ipi_selected(PCPU_GET(other_cpus), 0, (u_long)func, (u_long)ica);
+	return (&ica->ica_mask);
+}
+
+static __inline void *
+ipi_icache_page_inval(void *func, vm_paddr_t pa)
+{
+	struct ipi_cache_args *ica;
+
+	if (smp_cpus == 1)
+		return (NULL);
+	ica = &ipi_cache_args;
+	mtx_lock_spin(&ipi_mtx);
+	ica->ica_mask = all_cpus;
+	ica->ica_pa = pa;
+	cpu_ipi_selected(PCPU_GET(other_cpus), 0, (u_long)func, (u_long)ica);
+	return (&ica->ica_mask);
+}
+
+static __inline void *
+ipi_tlb_context_demap(struct pmap *pm)
+{
+	struct ipi_tlb_args *ita;
+	u_int cpus;
+
+	if (smp_cpus == 1)
+		return (NULL);
+	if ((cpus = (pm->pm_active & PCPU_GET(other_cpus))) == 0)
+		return (NULL);
+	ita = &ipi_tlb_args;
+	mtx_lock_spin(&ipi_mtx);
+	ita->ita_mask = cpus | PCPU_GET(cpumask);
+	ita->ita_pmap = pm;
+	cpu_ipi_selected(cpus, 0, (u_long)tl_ipi_tlb_context_demap,
+	    (u_long)ita);
+	return (&ita->ita_mask);
+}
+
+static __inline void *
+ipi_tlb_page_demap(struct pmap *pm, vm_offset_t va)
+{
+	struct ipi_tlb_args *ita;
+	u_int cpus;
+
+	if (smp_cpus == 1)
+		return (NULL);
+	if ((cpus = (pm->pm_active & PCPU_GET(other_cpus))) == 0)
+		return (NULL);
+	ita = &ipi_tlb_args;
+	mtx_lock_spin(&ipi_mtx);
+	ita->ita_mask = cpus | PCPU_GET(cpumask);
+	ita->ita_pmap = pm;
+	ita->ita_va = va;
+	cpu_ipi_selected(cpus, 0, (u_long)tl_ipi_tlb_page_demap, (u_long)ita);
+	return (&ita->ita_mask);
+}
+
+static __inline void *
+ipi_tlb_range_demap(struct pmap *pm, vm_offset_t start, vm_offset_t end)
+{
+	struct ipi_tlb_args *ita;
+	u_int cpus;
+
+	if (smp_cpus == 1)
+		return (NULL);
+	if ((cpus = (pm->pm_active & PCPU_GET(other_cpus))) == 0)
+		return (NULL);
+	ita = &ipi_tlb_args;
+	mtx_lock_spin(&ipi_mtx);
+	ita->ita_mask = cpus | PCPU_GET(cpumask);
+	ita->ita_pmap = pm;
+	ita->ita_start = start;
+	ita->ita_end = end;
+	cpu_ipi_selected(cpus, 0, (u_long)tl_ipi_tlb_range_demap, (u_long)ita);
+	return (&ita->ita_mask);
+}
+#endif
+static __inline void
+ipi_wait(void *cookie)
+{
+	volatile u_int *mask;
+
+	if ((mask = cookie) != NULL) {
+		atomic_clear_int(mask, PCPU_GET(cpumask));
+		while (*mask != 0)
+			;
+		mtx_unlock_spin(&ipi_mtx);
+	}
+}
+
+#endif /* _MACHINE_PMAP_H_ && _SYS_MUTEX_H_ */
+
+#else
+
+static __inline void *
+ipi_dcache_page_inval(void *func, vm_paddr_t pa)
+{
+	return (NULL);
+}
+
+static __inline void *
+ipi_icache_page_inval(void *func, vm_paddr_t pa)
+{
+	return (NULL);
+}
+
+static __inline void *
+ipi_tlb_context_demap(struct pmap *pm)
+{
+	return (NULL);
+}
+
+static __inline void *
+ipi_tlb_page_demap(struct pmap *pm, vm_offset_t va)
+{
+	return (NULL);
+}
+
+static __inline void *
+ipi_tlb_range_demap(struct pmap *pm, vm_offset_t start, vm_offset_t end)
+{
+	return (NULL);
+}
+
+static __inline void
+ipi_wait(void *cookie)
+{
+}
+
+#endif /* SMP */
+
+#endif /* !LOCORE */
+
+#endif /* !_MACHINE_SMP_H_ */
diff --git a/sys/sun4v/include/stdarg.h b/sys/sun4v/include/stdarg.h
new file mode 100644
index 000000000000..a7ff28484389
--- /dev/null
+++ b/sys/sun4v/include/stdarg.h
@@ -0,0 +1,101 @@
+/*-
+ * Copyright (c) 2002 David E. O'Brien.  All rights reserved.
+ * Copyright (c) 1992, 1993
+ *	The Regents of the University of California.  All rights reserved.
+ *
+ * This software was developed by the Computer Systems Engineering group
+ * at Lawrence Berkeley Laboratory under DARPA contract BG 91-66 and
+ * contributed to Berkeley.
+ *
+ * All advertising materials mentioning features or use of this software
+ * must display the following acknowledgement:
+ *	This product includes software developed by the University of
+ *	California, Lawrence Berkeley Laboratory.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	@(#)stdarg.h	8.2 (Berkeley) 9/27/93
+ *	$NetBSD: stdarg.h,v 1.11 2000/07/23 21:36:56 mycroft Exp $
+ * $FreeBSD$
+ */
+
+#ifndef _MACHINE_STDARG_H_
+#define	_MACHINE_STDARG_H_
+
+#include <sys/cdefs.h>
+#include <sys/_types.h>
+
+#ifndef _VA_LIST_DECLARED
+#define	_VA_LIST_DECLARED
+typedef	__va_list	va_list;
+#endif
+
+#ifdef __GNUCLIKE_BUILTIN_STDARG
+
+#define	va_start(ap, last) \
+	__builtin_stdarg_start((ap), (last))
+
+#define	va_arg(ap, type) \
+	__builtin_va_arg((ap), type)
+
+#if __ISO_C_VISIBLE >= 1999
+#define	va_copy(dest, src) \
+	__builtin_va_copy((dest), (src))
+#endif
+
+#define	va_end(ap) \
+	__builtin_va_end(ap)
+
+#else	/* ! __GNUCLIKE_BUILTIN_STDARG */
+
+#if !defined(__GNUCLIKE_BUILTIN_NEXT_ARG) && !defined(lint)
+#error no support for your compiler
+#endif
+
+#define	va_start(ap, last) \
+	(__builtin_next_arg(last), (ap) = (va_list)__builtin_saveregs())
+
+#define va_end(ap)	
+
+#define	__va_arg8(ap, type) \
+	(*(type *)(void *)((ap) += 8, (ap) - 8))
+#define	__va_arg16(ap, type) \
+	(*(type *)(void *)((ap) = (va_list)(((unsigned long)(ap) + 31) & -16),\
+			   (ap) - 16))
+#define	__va_int(ap, type) \
+	(*(type *)(void *)((ap) += 8, (ap) - sizeof(type)))
+
+#define	__REAL_TYPE_CLASS	8
+#define	__RECORD_TYPE_CLASS	12
+#define va_arg(ap, type) \
+	(__builtin_classify_type(*(type *)0) == __REAL_TYPE_CLASS ?	\
+	 (__alignof__(type) == 16 ? __va_arg16(ap, type) :		\
+	  __va_arg8(ap, type)) :					\
+	 (__builtin_classify_type(*(type *)0) < __RECORD_TYPE_CLASS ?	\
+	  __va_int(ap, type) :						\
+	  (sizeof(type) <= 8 ? __va_arg8(ap, type) :			\
+	   (sizeof(type) <= 16 ? __va_arg16(ap, type) :			\
+	    *__va_arg8(ap, type *)))))
+
+#endif /* __GNUCLIKE_BUILTIN_STDARG */
+
+#endif /* !_MACHINE_STDARG_H_ */
diff --git a/sys/sun4v/include/sun4v_cpufunc.h b/sys/sun4v/include/sun4v_cpufunc.h
new file mode 100644
index 000000000000..4870d0dc28bd
--- /dev/null
+++ b/sys/sun4v/include/sun4v_cpufunc.h
@@ -0,0 +1,91 @@
+/*-
+ * Copyright (c) 2006 Kip Macy
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef	_MACHINE_SUN4V_CPUFUNC_H_
+#define	_MACHINE_SUN4V_CPUFUNC_H_
+#include <machine/hypervisor_api.h>
+void set_mmfsa_scratchpad(vm_paddr_t mmfsa);
+
+void set_hash_user_scratchpad(uint64_t);
+void set_tsb_user_scratchpad(uint64_t);
+void set_hash_kernel_scratchpad(uint64_t);
+void set_tsb_kernel_scratchpad(uint64_t);
+void init_mondo(uint64_t func, uint64_t arg1, uint64_t arg2, uint64_t arg3);
+void init_mondo_queue(void);
+
+static __inline void * 
+set_tba(void *ntba)
+{
+	void *otba;
+	otba = (char *)rdpr(tba);
+	wrpr(tba, ntba, 0);
+	return otba;
+}
+
+
+
+static __inline void 
+set_wstate(u_long nwstate)
+{
+	wrpr(wstate, nwstate, 0);
+}
+
+void invlpg(uint16_t ctx, vm_offset_t va);
+
+void invlctx(uint16_t ctx);
+
+void invltlb(void);
+
+static __inline void 
+store_real(vm_paddr_t ra, uint64_t val)
+{
+	stxa(ra, ASI_REAL, val);
+}
+
+static __inline void 
+store_real_sync(vm_paddr_t ra, uint64_t val)
+{
+	stxa_sync(ra, ASI_REAL, val);
+}
+
+static __inline uint64_t 
+load_real(vm_paddr_t ra)
+{
+	uint64_t val;
+	val = ldxa(ra, ASI_REAL);
+	return val;
+}
+
+
+void load_real_dw(vm_paddr_t ra, uint64_t *lo, uint64_t *hi);
+void bzerophyspage(vm_paddr_t ra, uint64_t size);
+int hwblkclr(void *p, uint64_t size);
+int novbcopy(void *src, void *dst, uint64_t size);
+
+
+#endif /* !_MACHINE_CPUFUNC_H_ */
diff --git a/sys/sun4v/include/sysarch.h b/sys/sun4v/include/sysarch.h
new file mode 100644
index 000000000000..c209ef8048d2
--- /dev/null
+++ b/sys/sun4v/include/sysarch.h
@@ -0,0 +1,72 @@
+/*-
+ * Copyright (c) 1993 The Regents of the University of California.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 4. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	from: FreeBSD: src/sys/i386/include/sysarch.h,v 1.14 2000/09/21
+ * $FreeBSD$
+ */
+
+/*
+ * Architecture specific syscalls (sparc64)
+ */
+#ifndef _MACHINE_SYSARCH_H_
+#define _MACHINE_SYSARCH_H_
+
+#define	SPARC_UTRAP_INSTALL	1
+#define	SPARC_SIGTRAMP_INSTALL	2
+
+struct sparc_utrap_install_args {
+	int num;
+	const struct sparc_utrap_args *handlers;
+};
+
+struct sparc_sigtramp_install_args {
+	void	*sia_new;
+	void	**sia_old;
+};
+
+struct sparc_utrap_args {
+	utrap_entry_t type;
+	utrap_handler_t new_precise;
+	utrap_handler_t new_deferred;
+	utrap_handler_t *old_precise;
+	utrap_handler_t *old_deferred;
+};
+
+#ifndef _KERNEL
+#include <sys/cdefs.h>
+
+__BEGIN_DECLS
+int __sparc_utrap_install(utrap_entry_t type, utrap_handler_t new_precise,
+			  utrap_handler_t new_deferred,
+			  utrap_handler_t *old_precise,
+			  utrap_handler_t *old_deferred);
+int sysarch(int, void *);
+__END_DECLS
+#endif
+
+#endif /* !_MACHINE_SYSARCH_H_ */
diff --git a/sys/sun4v/include/tick.h b/sys/sun4v/include/tick.h
new file mode 100644
index 000000000000..8586a0d89f8d
--- /dev/null
+++ b/sys/sun4v/include/tick.h
@@ -0,0 +1,36 @@
+/*-
+ * Copyright (c) 2001 Jake Burkholder.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _MACHINE_TICK_H_
+#define	_MACHINE_TICK_H_
+
+void	tick_init(u_long clock);
+void	tick_start(void);
+void	tick_stop(void);
+
+#endif
diff --git a/sys/sun4v/include/tlb.h b/sys/sun4v/include/tlb.h
new file mode 100644
index 000000000000..74e008ebcf6c
--- /dev/null
+++ b/sys/sun4v/include/tlb.h
@@ -0,0 +1,95 @@
+/*-
+ * Copyright (c) 2001 Jake Burkholder.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef	_MACHINE_TLB_H_
+#define	_MACHINE_TLB_H_
+
+#define	TLB_DIRECT_ADDRESS_BITS		(43)
+#define	TLB_DIRECT_PAGE_BITS		(PAGE_SHIFT_4M)
+
+#define	TLB_DIRECT_ADDRESS_MASK		((1UL << TLB_DIRECT_ADDRESS_BITS) - 1)
+#define	TLB_DIRECT_PAGE_MASK		((1UL << TLB_DIRECT_PAGE_BITS) - 1)
+
+#define	TLB_PHYS_TO_DIRECT(pa) \
+	((pa) | VM_MIN_DIRECT_ADDRESS)
+#define	TLB_DIRECT_TO_PHYS(va) \
+	((va) & TLB_DIRECT_ADDRESS_MASK)
+#define	TLB_DIRECT_TO_TTE_MASK \
+	(TD_V | TD_4M | (TLB_DIRECT_ADDRESS_MASK - TLB_DIRECT_PAGE_MASK))
+
+#define	TLB_DAR_SLOT_SHIFT		(3)
+#define	TLB_DAR_SLOT(slot)		((slot) << TLB_DAR_SLOT_SHIFT)
+
+#define	TAR_VPN_SHIFT			(13)
+#define	TAR_CTX_MASK			((1 << TAR_VPN_SHIFT) - 1)
+
+#define	TLB_TAR_VA(va)			((va) & ~TAR_CTX_MASK)
+#define	TLB_TAR_CTX(ctx)		((ctx) & TAR_CTX_MASK)
+
+#define	TLB_DEMAP_ID_SHIFT		(4)
+#define	TLB_DEMAP_ID_PRIMARY		(0)
+#define	TLB_DEMAP_ID_SECONDARY		(1)
+#define	TLB_DEMAP_ID_NUCLEUS		(2)
+
+#define	TLB_DEMAP_TYPE_SHIFT		(6)
+#define	TLB_DEMAP_TYPE_PAGE		(0)
+#define	TLB_DEMAP_TYPE_CONTEXT		(1)
+
+#define	TLB_DEMAP_VA(va)		((va) & ~PAGE_MASK)
+#define	TLB_DEMAP_ID(id)		((id) << TLB_DEMAP_ID_SHIFT)
+#define	TLB_DEMAP_TYPE(type)		((type) << TLB_DEMAP_TYPE_SHIFT)
+
+#define	TLB_DEMAP_PAGE			(TLB_DEMAP_TYPE(TLB_DEMAP_TYPE_PAGE))
+#define	TLB_DEMAP_CONTEXT		(TLB_DEMAP_TYPE(TLB_DEMAP_TYPE_CONTEXT))
+
+#define	TLB_DEMAP_PRIMARY		(TLB_DEMAP_ID(TLB_DEMAP_ID_PRIMARY))
+#define	TLB_DEMAP_SECONDARY		(TLB_DEMAP_ID(TLB_DEMAP_ID_SECONDARY))
+#define	TLB_DEMAP_NUCLEUS		(TLB_DEMAP_ID(TLB_DEMAP_ID_NUCLEUS))
+
+#define	TLB_CTX_KERNEL			(0)
+#define	TLB_CTX_USER_MIN		(1)
+#define	TLB_CTX_USER_MAX		(8192)
+
+typedef void tlb_flush_user_t(void);
+
+struct pmap;
+struct tlb_entry;
+
+extern int kernel_tlb_slots;
+extern struct tlb_entry *kernel_tlbs;
+
+void	tlb_context_demap(struct pmap *pm);
+void	tlb_page_demap(struct pmap *pm, vm_offset_t va);
+void	tlb_range_demap(struct pmap *pm, vm_offset_t start, vm_offset_t end);
+
+tlb_flush_user_t cheetah_tlb_flush_user;
+tlb_flush_user_t spitfire_tlb_flush_user;
+
+extern tlb_flush_user_t *tlb_flush_user;
+
+#endif /* !_MACHINE_TLB_H_ */
diff --git a/sys/sun4v/include/trap.h b/sys/sun4v/include/trap.h
new file mode 100644
index 000000000000..16457e06b05d
--- /dev/null
+++ b/sys/sun4v/include/trap.h
@@ -0,0 +1,146 @@
+/*-
+ * Copyright (c) 2001 Jake Burkholder.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef	_MACHINE_TRAP_H_
+#define	_MACHINE_TRAP_H_
+
+#ifdef _KERNEL
+
+
+#define T_DATA_MISS                     0x31
+#define T_ALIGNMENT                     0x34
+#define	T_DATA_PROTECTION		0x6c
+#define T_MEM_ADDRESS_NOT_ALIGNED       T_ALIGNMENT
+
+#define	T_RESERVED			0
+#define	T_INSTRUCTION_EXCEPTION		1
+#define	T_INSTRUCTION_ERROR		2
+#define	T_INSTRUCTION_PROTECTION	3
+#define	T_ILLTRAP_INSTRUCTION		4
+#define	T_ILLEGAL_INSTRUCTION		5
+#define	T_PRIVILEGED_OPCODE		6
+#define	T_FP_DISABLED			7
+#define	T_FP_EXCEPTION_IEEE_754		8
+
+#define	T_INSTRUCTION_MISS		0x09
+#define	T_TAG_OVERFLOW			0x0a
+#define	T_DIVISION_BY_ZERO		0x0b
+#define	T_DATA_EXCEPTION		0x0c
+#define	T_DATA_ERROR			0x0d
+
+
+#define	T_PRIVILEGED_ACTION		0x10
+#define	T_ASYNC_DATA_ERROR		0x11
+#define	T_TRAP_INSTRUCTION_16		0x12
+#define	T_TRAP_INSTRUCTION_17		0x13
+#define	T_TRAP_INSTRUCTION_18		0x14
+#define	T_TRAP_INSTRUCTION_19		0x15
+#define	T_TRAP_INSTRUCTION_20		0x16
+#define	T_TRAP_INSTRUCTION_21		0x17
+#define	T_TRAP_INSTRUCTION_22		0x18
+#define	T_TRAP_INSTRUCTION_23		0x19
+#define	T_TRAP_INSTRUCTION_24		0x1a
+#define	T_TRAP_INSTRUCTION_25		0x1b
+#define	T_TRAP_INSTRUCTION_26		0x1c
+#define	T_TRAP_INSTRUCTION_27		0x1d
+#define	T_TRAP_INSTRUCTION_28		0x1e
+#define	T_TRAP_INSTRUCTION_29		0x1f
+#define	T_TRAP_INSTRUCTION_30		0x20
+#define	T_TRAP_INSTRUCTION_31		0x21
+#define	T_FP_EXCEPTION_OTHER		0x22
+
+
+
+#define	T_INTERRUPT			0x24
+#define	T_PA_WATCHPOINT			0x25
+#define	T_VA_WATCHPOINT			0x26
+#define	T_CORRECTED_ECC_ERROR		0x27
+#define	T_SPILL				0x28
+#define	T_FILL				0x29
+#define	T_FILL_RET			0x2a
+#define	T_BREAKPOINT			0x2b
+#define	T_CLEAN_WINDOW			0x2c
+#define	T_RANGE_CHECK			0x2d
+#define T_FIX_ALIGNMENT                 0x2e
+#define	T_INTEGER_OVERFLOW		0x2f
+#define	T_SYSCALL			0x30
+#define	T_RSTRWP_PHYS			
+#define	T_RSTRWP_VIRT			
+#define	T_KSTACK_FAULT			51
+#define T_RESUMABLE_ERROR               52
+#define T_NONRESUMABLE_ERROR            53
+
+#define	T_MAX				(T_NONRESUMABLE_ERROR + 1)
+
+#define	T_KERNEL			0x100
+#define TRAP_MASK                       ((1<<8)-1)
+#define TRAP_CTX_SHIFT                  10
+
+#define	PTL1_BAD_DEBUG		0
+#define	PTL1_BAD_WTRAP		1
+#define	PTL1_BAD_KMISS		2
+#define	PTL1_BAD_KPROT_FAULT	3
+#define	PTL1_BAD_ISM		4
+#define	PTL1_BAD_MMUTRAP	5
+#define	PTL1_BAD_TRAP		6
+#define	PTL1_BAD_FPTRAP		7
+#define	PTL1_BAD_INTR_REQ	8
+#define	PTL1_BAD_TRACE_PTR	9
+#define	PTL1_BAD_STACK		10
+#define	PTL1_BAD_DTRACE_FLAGS	11
+#define	PTL1_BAD_CTX_STEAL	12
+#define	PTL1_BAD_ECC		13
+#define	PTL1_BAD_HCALL		14
+#define	PTL1_BAD_GL		15
+
+
+#define TL_CPU_MONDO    0x1
+#define TL_DEV_MONDO    0x2
+#define TL_TSB_MISS     0x3
+#define TL_TL0_TRAP     0x4
+#define TL_SET_ACKMASK  0x5
+
+/*
+ * These defines are used by the TL1 tlb miss handlers to calculate
+ * the pc to jump to in the case the entry was not found in the TSB.
+ */
+#define	WTRAP_ALIGN	0x7f	/* window handlers are 128 byte align */
+#define	WTRAP_FAULTOFF	124	/* last instruction in handler */
+
+/* use the following defines to determine if trap was a fill or a spill */
+#define	WTRAP_TTMASK	0x180
+#define	WTRAP_TYPE	0x080
+
+#ifndef LOCORE
+extern const char *trap_msg[];
+void trap_init(void);
+#endif
+
+#endif
+
+#endif /* !_MACHINE_TRAP_H_ */
diff --git a/sys/sun4v/include/tsb.h b/sys/sun4v/include/tsb.h
new file mode 100644
index 000000000000..152e58f53aab
--- /dev/null
+++ b/sys/sun4v/include/tsb.h
@@ -0,0 +1,79 @@
+/*-
+ * Copyright (c) 2006 Kip Macy
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef	_MACHINE_TSB_H_
+#define	_MACHINE_TSB_H_
+
+#define MAX_TSB_INFO                     2
+
+/*
+ * Values for "tsb_ttesz_mask" bitmask.
+ */
+#define	TSB8K	(1 << TTE8K)
+#define	TSB64K  (1 << TTE64K)
+#define	TSB512K (1 << TTE512K)
+#define	TSB4M   (1 << TTE4M)
+#define	TSB32M  (1 << TTE32M)
+#define	TSB256M (1 << TTE256M)
+
+/*
+ * Kernel TSBs
+ */
+#define TSB8K_INDEX           0
+#define TSB4M_INDEX           1
+
+extern hv_tsb_info_t kernel_td[MAX_TSB_INFO];
+
+struct hv_tsb_info;
+
+
+vm_paddr_t tsb_init(struct hv_tsb_info *tsb, uint64_t *scratchval);
+
+void tsb_deinit(struct hv_tsb_info *tsb);
+
+void tsb_assert_invalid(struct hv_tsb_info *tsb, vm_offset_t va);
+
+void tsb_set_tte(struct hv_tsb_info *tsb, vm_offset_t va, tte_t tte_data, uint64_t ctx);
+
+void tsb_set_tte_real(struct hv_tsb_info *tsb, vm_offset_t va, tte_t tte_data, uint64_t ctx);
+
+tte_t tsb_get_tte(struct hv_tsb_info *tsb, vm_offset_t va);
+
+tte_t tsb_lookup_tte(vm_offset_t va, uint64_t context);
+
+void tsb_clear(struct hv_tsb_info *tsb);
+
+void tsb_clear_tte(struct hv_tsb_info *tsb, vm_offset_t va);
+
+void tsb_clear_range(struct hv_tsb_info *tsb, vm_offset_t sva, vm_offset_t eva);
+
+uint64_t tsb_set_scratchpad_kernel(struct hv_tsb_info *tsb);
+
+uint64_t tsb_set_scratchpad_user(struct hv_tsb_info *tsb);
+
+#endif /* !_MACHINE_TSB_H_ */
diff --git a/sys/sun4v/include/tstate.h b/sys/sun4v/include/tstate.h
new file mode 100644
index 000000000000..8bb5b2417727
--- /dev/null
+++ b/sys/sun4v/include/tstate.h
@@ -0,0 +1,84 @@
+/*-
+ * Copyright 2001 by Thomas Moestl <tmm@FreeBSD.org>.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+ * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef	_MACHINE_TSTATE_H_
+#define	_MACHINE_TSTATE_H_
+
+#include <machine/pstate.h>
+#include <machine/ccr.h>
+
+#define	TSTATE_CWP_SHIFT	0
+#define	TSTATE_CWP_BITS		5
+#define	TSTATE_CWP_MASK \
+	(((1 << TSTATE_CWP_BITS) - 1) << TSTATE_CWP_SHIFT)
+#define	TSTATE_CWP(x)		((x & TSTATE_CWP_MASK) >> TSTATE_CWP_SHIFT)
+
+#define	TSTATE_PSTATE_SHIFT	8
+#define	TSTATE_PSTATE_BITS	12
+#define	TSTATE_PSTATE_MASK \
+	(((1 << TSTATE_PSTATE_BITS) - 1) << TSTATE_PSTATE_SHIFT)
+#define	TSTATE_PSTATE(x)	((x) << TSTATE_PSTATE_SHIFT)
+#define	TSTATE_AG		TSTATE_PSTATE(PSTATE_AG)
+#define	TSTATE_IE		TSTATE_PSTATE(PSTATE_IE)
+#define	TSTATE_PRIV		TSTATE_PSTATE(PSTATE_PRIV)
+#define	TSTATE_AM		TSTATE_PSTATE(PSTATE_AM)
+#define	TSTATE_PEF		TSTATE_PSTATE(PSTATE_PEF)
+#define	TSTATE_RED		TSTATE_PSTATE(PSTATE_RED)
+#define	TSTATE_MM_TSO		TSTATE_PSTATE(PSTATE_MM_TSO)
+#define	TSTATE_MM_PSO		TSTATE_PSTATE(PSTATE_MM_PSO)
+#define	TSTATE_MM_RMO		TSTATE_PSTATE(PSTATE_MM_RMO)
+#define	TSTATE_TLE		TSTATE_PSTATE(PSTATE_TLE)
+#define	TSTATE_CLE		TSTATE_PSTATE(PSTATE_CLE)
+#define	TSTATE_MG		TSTATE_PSTATE(PSTATE_MG)
+#define	TSTATE_IG		TSTATE_PSTATE(PSTATE_IG)
+#define TSTATE_KERNEL           TSTATE_PSTATE(PSTATE_KERNEL)
+
+
+#define	TSTATE_ASI_SHIFT	24
+#define	TSTATE_ASI_BITS		8
+#define	TSTATE_ASI_MASK	\
+	(((1 << TSTATE_ASI_BITS) - 1) << TSTATE_ASI_SHIFT)
+#define	TSTATE_ASI(x)		((x & TSTATE_ASI_MASK) >> TSTATE_ASI_SHIFT)
+
+#define	TSTATE_CCR_SHIFT	32
+#define	TSTATE_ICC_SHIFT	(TSTATE_CCR_SHIFT + ICC_SHIFT)
+#define	TSTATE_ICC_MASK		(ICC_MASK << TSTATE_CCR_SHIFT)
+#define	TSTATE_XCC_SHIFT	(TSTATE_CCR_SHIFT + XCC_SHIFT)
+#define	TSTATE_XCC_MASK		(XCC_MASK << TSTATE_CCR_SHIFT)
+#define	TSTATE_CCR(x)		((x) << TSTATE_CCR_SHIFT)
+#define	TSTATE_ICC_C		TSTATE_CCR(ICC_C)
+#define	TSTATE_ICC_V		TSTATE_CCR(ICC_V)
+#define	TSTATE_ICC_Z		TSTATE_CCR(ICC_Z)
+#define	TSTATE_ICC_N		TSTATE_CCR(ICC_N)
+#define	TSTATE_XCC_C		TSTATE_CCR(XCC_C)
+#define	TSTATE_XCC_V		TSTATE_CCR(XCC_V)
+#define	TSTATE_XCC_Z		TSTATE_CCR(XCC_Z)
+#define	TSTATE_XCC_N		TSTATE_CCR(XCC_N)
+
+#define	TSTATE_SECURE(tstate) \
+	PSTATE_SECURE(((tstate) >> TSTATE_PSTATE_SHIFT) & ((1 << TSTATE_PSTATE_BITS) - 1))
+
+#endif /* !_MACHINE_TSTATE_H_ */
diff --git a/sys/sun4v/include/tte.h b/sys/sun4v/include/tte.h
new file mode 100644
index 000000000000..83675757bf12
--- /dev/null
+++ b/sys/sun4v/include/tte.h
@@ -0,0 +1,243 @@
+/*-
+ * Copyright (c) 1997 Berkeley Software Design, Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Berkeley Software Design Inc's name may not be used to endorse or
+ *    promote products derived from this software without specific prior
+ *    written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	from: BSDI: pmap.v9.h,v 1.10.2.6 1999/08/23 22:18:44 cp Exp
+ * $FreeBSD$
+ */
+
+#ifndef	_MACHINE_TTE_H_
+#define	_MACHINE_TTE_H_
+
+#define	TTE_SHIFT	(4)
+
+#define	TD_SOFT2_SHIFT	(50)
+#define	TD_DIAG_SHIFT	(41)
+#define	TD_PA_SHIFT	(13)
+#define	TD_SOFT_SHIFT	(7)
+
+#define	TD_SOFT2_BITS	(9)
+#define	TD_DIAG_BITS	(9)
+#define	TD_PA_BITS	(42)
+#define	TD_SOFT_BITS	(6)
+
+#define	TD_SOFT2_MASK	((1UL << TD_SOFT2_BITS) - 1)
+#define	TD_DIAG_MASK	((1UL << TD_DIAG_BITS) - 1)
+#define	TD_PA_MASK	((1UL << TD_PA_BITS) - 1)
+#define	TD_SOFT_MASK	((1UL << TD_SOFT_BITS) - 1)
+
+#define	TTE8K		(0UL)
+#define	TTE64K		(1UL)
+#define	TTE512K	        (2UL)
+#define	TTE4M		(3UL)
+#define	TTE32M		(4UL)
+#define	TTE256M	        (5UL)
+#define	TTE2G		(6UL)
+#define	TTE16G		(7UL)
+
+#define	TD_PA(pa)	((pa) & (TD_PA_MASK << TD_PA_SHIFT))
+/* NOTE: bit 6 of TD_SOFT will be sign-extended if used as an immediate. */
+#define	TD_FAKE		((1UL << 5) << TD_SOFT_SHIFT)
+#define	TD_EXEC		((1UL << 4) << TD_SOFT_SHIFT)
+#define	TD_REF		((1UL << 3) << TD_SOFT_SHIFT)
+#define	TD_PV		((1UL << 2) << TD_SOFT_SHIFT)
+#define	TD_SW		((1UL << 1) << TD_SOFT_SHIFT)
+#define	TD_WIRED	((1UL << 0) << TD_SOFT_SHIFT)
+#define	TD_L		(1UL << 6)
+#define	TD_CP		(1UL << 5)
+#define	TD_CV		(1UL << 4)
+#define	TD_E		(1UL << 3)
+#define	TD_P		(1UL << 2)
+#define	TD_W		(1UL << 1)
+#define	TD_G		(1UL << 0)
+
+
+#define	TTE_GET_PAGE_SIZE(tp) \
+	(1 << TTE_GET_PAGE_SHIFT(tp))
+#define	TTE_GET_PAGE_MASK(tp) \
+	(TTE_GET_PAGE_SIZE(tp) - 1)
+
+#define	TTE_GET_PA(tte_data) \
+	(tte_data & (TD_PA_MASK << TD_PA_SHIFT))
+#define	TTE_GET_VPN(tp) \
+	((tp)->tte_vpn >> TV_SIZE_BITS)
+#define	TTE_GET_VA(tp) \
+	(TTE_GET_VPN(tp) << TTE_GET_PAGE_SHIFT(tp))
+#define	TTE_GET_PMAP(tp) \
+	(((tp)->tte_data & TD_P) != 0 ? \
+	 (kernel_pmap) : \
+	 (PHYS_TO_VM_PAGE(pmap_kextract((vm_offset_t)(tp)))->md.pmap))
+#define	TTE_ZERO(tp) \
+	memset(tp, 0, sizeof(*tp))
+
+struct pmap;
+#define PTE_SHIFT       (3)
+#define PT_SHIFT        (PAGE_SHIFT - PTE_SHIFT)
+
+#define	VTD_SOFT_SHIFT	(56)
+
+#define	VTD_V		(1UL << 63)
+#define	VTD_NFO		(1UL << 62)
+#define	VTD_PA(pa)	((pa) & (TD_PA_MASK << TD_PA_SHIFT))
+#define	VTD_IE		(1UL << 12)
+#define	VTD_E		(1UL << 11)
+#define	VTD_CP		(1UL << 10)
+#define	VTD_CV		(1UL << 9)
+#define	VTD_P		(1UL << 8)
+#define	VTD_X		(1UL << 7)
+#define	VTD_W		(1UL << 6)
+
+#define	VTD_REF		(1UL << 5)
+#define	VTD_SW_W	(1UL << 4)
+#define	VTD_MANAGED	(1UL << 58)
+#define	VTD_WIRED	(1UL << 57)
+#define	VTD_LOCK	(1UL << 56)
+
+
+
+#define	VTD_8K		TTE8K 
+#define	VTD_64K		TTE64K 
+#define	VTD_512K	TTE512K 
+#define	VTD_4M		TTE4M 
+#define	VTD_32M		TTE32M 
+#define	VTD_256M	TTE256M 
+
+/*
+ * default flags for kernel pages
+ */
+#define TTE_KERNEL            VTD_V | VTD_CP | VTD_CV | VTD_P | VTD_X | VTD_W | VTD_SW_W | VTD_REF | VTD_WIRED
+#define TTE_KERNEL_MINFLAGS   VTD_P
+#define TTE_MINFLAGS          VTD_V | VTD_CP | VTD_CV
+
+#define VTD_SIZE_BITS   (4)
+#define VTD_SIZE_MASK   ((1 << VTD_SIZE_BITS) - 1)
+
+
+#define	TTE_SIZE_SPREAD	(3)
+#define	TTE_PAGE_SHIFT(sz) \
+	(PAGE_SHIFT + ((sz) * TTE_SIZE_SPREAD))
+#define	TTE_GET_SIZE(tte_data) \
+	(tte_data & VTD_SIZE_MASK)
+#define	TTE_GET_PAGE_SHIFT(tte_data) \
+	TTE_PAGE_SHIFT(TTE_GET_SIZE(tte_data))
+
+#ifdef notyet
+typedef union {
+	struct tte {
+		unsigned int	v:1;		/* <63> valid */
+		unsigned int	nfo:1;		/* <62> non-fault only */
+		unsigned int	sw:3;	        /* <61:59> sw */
+		unsigned int    managed:1;      /* <58> managed */
+		unsigned int    wired:1;        /* <57> wired */
+		unsigned int	lock:1;		/* <56> sw - locked */
+		unsigned long	pa:43;	        /* <55:13> pa */
+		unsigned int	ie:1;		/* <12> 1=invert endianness */
+		unsigned int	e:1;		/* <11> side effect */
+		unsigned int	cp:1;		/* <10> physically cache */
+		unsigned int	cv:1;		/* <9> virtually cache */
+		unsigned int	p:1;		/* <8> privilege required */
+		unsigned int	x:1;		/* <7> execute perm */
+		unsigned int	w:1;		/* <6> write perm */
+		unsigned int	ref:1;		/* <5> sw - ref */
+		unsigned int	wr_perm:1;	/* <4> sw - write perm */
+		unsigned int	rsvd:1;		/* <3> reserved */
+		unsigned int	sz:3;		/* <2:0> pagesize */
+	} tte_bit;
+	uint64_t		ll;
+} tte_t;
+#endif
+
+#define	tte_val 	tte_bit.v		/* use < 0 check in asm */
+#define	tte_size	tte_bit.sz
+#define	tte_nfo		tte_bit.nfo
+#define	tte_ie		tte_bit.ie		
+#define	tte_wired       tte_bit.wired
+#define	tte_pa	        tte_bit.pa
+#define	tte_ref		tte_bit.ref
+#define	tte_wr_perm	tte_bit.wr_perm
+#define	tte_exec_perm	tte_bit.x
+#define	tte_lock	tte_bit.lock
+#define	tte_cp		tte_bit.cp
+#define	tte_cv		tte_bit.cv
+#define	tte_se		tte_bit.e
+#define	tte_priv	tte_bit.p
+#define	tte_hwwr	tte_bit.w
+
+#define	TTE_IS_VALID(ttep)	((ttep)->tte_inthi < 0)
+#define	TTE_SET_INVALID(ttep)	((ttep)->tte_val = 0)
+#define	TTE_IS_8K(ttep)		(TTE_CSZ(ttep) == TTE8K)
+#define	TTE_IS_WIRED(ttep)	((ttep)->tte_wired)
+#define	TTE_IS_WRITABLE(ttep)	((ttep)->tte_wr_perm)
+#define	TTE_IS_EXECUTABLE(ttep)	((ttep)->tte_exec_perm)
+#define	TTE_IS_PRIVILEGED(ttep)	((ttep)->tte_priv)
+#define	TTE_IS_NOSYNC(ttep)	((ttep)->tte_no_sync)
+#define	TTE_IS_LOCKED(ttep)	((ttep)->tte_lock)
+#define	TTE_IS_SIDEFFECT(ttep)	((ttep)->tte_se)
+#define	TTE_IS_NFO(ttep)	((ttep)->tte_nfo)
+
+#define	TTE_IS_REF(ttep)	((ttep)->tte_ref)
+#define	TTE_IS_MOD(ttep)	((ttep)->tte_hwwr)
+#define	TTE_IS_IE(ttep)		((ttep)->tte_ie)
+#define	TTE_SET_SUSPEND(ttep)	((ttep)->tte_suspend = 1)
+#define	TTE_CLR_SUSPEND(ttep)	((ttep)->tte_suspend = 0)
+#define	TTE_IS_SUSPEND(ttep)	((ttep)->tte_suspend)
+#define	TTE_SET_REF(ttep)	((ttep)->tte_ref = 1)
+#define	TTE_CLR_REF(ttep)	((ttep)->tte_ref = 0)
+#define	TTE_SET_LOCKED(ttep)	((ttep)->tte_lock = 1)
+#define	TTE_CLR_LOCKED(ttep)	((ttep)->tte_lock = 0)
+#define	TTE_SET_MOD(ttep)	((ttep)->tte_hwwr = 1)
+#define	TTE_CLR_MOD(ttep)	((ttep)->tte_hwwr = 0)
+#define	TTE_SET_RM(ttep)						\
+	(((ttep)->tte_intlo) =						\
+	(ttep)->tte_intlo | TTE_HWWR_INT | TTE_REF_INT)
+#define	TTE_CLR_RM(ttep)						\
+	(((ttep)->tte_intlo) =						\
+	(ttep)->tte_intlo & ~(TTE_HWWR_INT | TTE_REF_INT))
+
+#define	TTE_SET_WRT(ttep)	((ttep)->tte_wr_perm = 1)
+#define	TTE_CLR_WRT(ttep)	((ttep)->tte_wr_perm = 0)
+#define	TTE_SET_EXEC(ttep)	((ttep)->tte_exec_perm = 1)
+#define	TTE_CLR_EXEC(ttep)	((ttep)->tte_exec_perm = 0)
+#define	TTE_SET_PRIV(ttep)	((ttep)->tte_priv = 1)
+#define	TTE_CLR_PRIV(ttep)	((ttep)->tte_priv = 0)
+
+#define	TTE_BSZS_SHIFT(sz)	((sz) * 3)
+
+struct pmap;
+
+void tte_clear_phys_bit(vm_page_t m, uint64_t flags);
+
+void tte_set_phys_bit(vm_page_t m, uint64_t flags);
+
+boolean_t tte_get_phys_bit(vm_page_t m, uint64_t flags);
+
+void tte_clear_virt_bit(struct pmap *pmap, vm_offset_t va, uint64_t flags);
+
+void tte_set_virt_bit(struct pmap *pmap, vm_offset_t va, uint64_t flags);
+
+boolean_t tte_get_virt_bit(struct pmap *pmap, vm_offset_t va, uint64_t flags);
+
+#endif /* !_MACHINE_TTE_H_ */
diff --git a/sys/sun4v/include/tte_hash.h b/sys/sun4v/include/tte_hash.h
new file mode 100644
index 000000000000..cabdbd590323
--- /dev/null
+++ b/sys/sun4v/include/tte_hash.h
@@ -0,0 +1,75 @@
+/*-
+ * Copyright (c) 2006 Kip Macy
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef	_MACHINE_TTE_HASH_H_
+#define	_MACHINE_TTE_HASH_H_
+
+#define HASH_ENTRY_SHIFT   2
+#define HASH_ENTRIES       ((1 << HASH_ENTRY_SHIFT) - 1)
+#define THE_SHIFT          (TTE_SHIFT + HASH_ENTRY_SHIFT)  /* size of TSB entry * #entries */
+#define TH_COLLISION_SHIFT 47                          /* bit 47 will never be set for a valid tag */
+#define TH_COLLISION       (1UL << TH_COLLISION_SHIFT)  
+#define TH_INVALID_SHIFT   46                          /* bit 47 will never be set for a valid tag */
+#define TH_INVALID         (1UL << TH_INVALID_SHIFT)  
+
+
+struct tte_hash;
+typedef struct tte_hash *tte_hash_t;
+
+void tte_hash_init(void);
+
+void tte_hash_clear(tte_hash_t hash);
+
+tte_t tte_hash_clear_bits(tte_hash_t hash, vm_offset_t va, uint64_t flags);
+
+tte_hash_t tte_hash_kernel_create(vm_offset_t, uint64_t, vm_paddr_t);
+
+tte_hash_t tte_hash_create(uint64_t context, uint64_t *scratchval);
+
+void tte_hash_destroy(tte_hash_t th);
+
+tte_t tte_hash_delete(tte_hash_t hash, vm_offset_t va);
+
+void tte_hash_delete_all(tte_hash_t hash);
+
+void tte_hash_insert(tte_hash_t hash, vm_offset_t va, tte_t data);
+
+tte_t tte_hash_lookup(tte_hash_t hash, vm_offset_t va);
+
+tte_t tte_hash_lookup_nolock(tte_hash_t hash, vm_offset_t va);
+
+void tte_hash_reset(tte_hash_t hash);
+
+uint64_t tte_hash_set_scratchpad_kernel(tte_hash_t th);
+
+uint64_t tte_hash_set_scratchpad_user(tte_hash_t th, uint64_t context);
+
+tte_t tte_hash_update(tte_hash_t hash, vm_offset_t va, tte_t tte_data);
+
+
+#endif /* _MACHINE_TTE_HASH_H_ */
diff --git a/sys/sun4v/include/ucontext.h b/sys/sun4v/include/ucontext.h
new file mode 100644
index 000000000000..c05ec26a9053
--- /dev/null
+++ b/sys/sun4v/include/ucontext.h
@@ -0,0 +1,64 @@
+/*-
+ * Copyright (c) 1999 Marcel Moolenaar
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer 
+ *    in this position and unchanged.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The name of the author may not be used to endorse or promote products
+ *    derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ *	from: FreeBSD: src/sys/alpha/include/ucontext.h,v 1.3 1999/10/08
+ * $FreeBSD$
+ */
+
+#ifndef _MACHINE_UCONTEXT_H_
+#define	_MACHINE_UCONTEXT_H_
+
+struct __mcontext {
+	uint64_t mc_global[8];
+	uint64_t mc_out[8];
+	uint64_t mc_local[8];
+	uint64_t mc_in[8];
+	uint32_t mc_fp[64];
+} __aligned(64);
+
+typedef struct __mcontext mcontext_t;
+
+#define	mc_flags	mc_global[0]
+#define	mc_sp           mc_out[6]
+#define	mc_fprs         mc_local[0]
+#define	mc_fsr		mc_local[1]
+#define	mc_gsr		mc_local[2]
+#define	mc_tnpc         mc_in[0]
+#define	mc_tpc          mc_in[1]
+#define	mc_tstate       mc_in[2]
+#define	mc_y            mc_in[4]
+#define	mc_wstate	mc_in[5]
+
+#define	_MC_VERSION_SHIFT	0
+#define	_MC_VERSION_BITS	32
+#define	_MC_VERSION		1L
+
+#define	_MC_FLAGS_SHIFT		32
+#define	_MC_FLAGS_BITS		32
+#define	_MC_VOLUNTARY		((1L << 0) << _MC_FLAGS_SHIFT)
+
+#endif /* !_MACHINE_UCONTEXT_H_ */
diff --git a/sys/sun4v/include/upa.h b/sys/sun4v/include/upa.h
new file mode 100644
index 000000000000..3e56917b6383
--- /dev/null
+++ b/sys/sun4v/include/upa.h
@@ -0,0 +1,50 @@
+/*-
+ * Copyright (c) 2001 by Thomas Moestl <tmm@FreeBSD.org>.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+ * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _MACHINE_UPA_H_
+#define _MACHINE_UPA_H_
+
+#define	UPA_MEMSTART	0x1c000000000UL
+#define	UPA_MEMEND	0x1ffffffffffUL
+
+#define	UPA_CR_MID_SHIFT	(17)
+#define	UPA_CR_MID_SIZE		(5)
+#define	UPA_CR_MID_MASK \
+	(((1 << UPA_CR_MID_SIZE) - 1) << UPA_CR_MID_SHIFT)
+
+#define	UPA_CR_GET_MID(cr)	((cr & UPA_CR_MID_MASK) >> UPA_CR_MID_SHIFT)
+
+#ifdef LOCORE
+
+#define	UPA_GET_MID(r1) \
+	ldxa	[%g0] ASI_UPA_CONFIG_REG, r1 ; \
+	srlx	r1, UPA_CR_MID_SHIFT, r1 ; \
+	and	r1, (1 << UPA_CR_MID_SIZE) - 1, r1
+
+#endif
+
+#endif /* _MACHINE_UPA_H_ */
diff --git a/sys/sun4v/include/utrap.h b/sys/sun4v/include/utrap.h
new file mode 100644
index 000000000000..c8fa7431d95f
--- /dev/null
+++ b/sys/sun4v/include/utrap.h
@@ -0,0 +1,110 @@
+/*-
+ * Copyright (c) 2001 Jake Burkholder.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef	_MACHINE_UTRAP_H_
+#define	_MACHINE_UTRAP_H_
+
+#define	UT_INSTRUCTION_EXCEPTION	1
+#define	UT_INSTRUCTION_ERROR		2
+#define	UT_INSTRUCTION_PROTECTION	3
+#define	UT_ILLTRAP_INSTRUCTION		4
+#define	UT_ILLEGAL_INSTRUCTION		5
+#define	UT_PRIVILEGED_OPCODE		6
+#define	UT_FP_DISABLED			7
+#define	UT_FP_EXCEPTION_IEEE_754	8
+#define	UT_FP_EXCEPTION_OTHER		9
+#define	UT_TAG_OVERFLOW			10
+#define	UT_DIVISION_BY_ZERO		11
+#define	UT_DATA_EXCEPTION		12
+#define	UT_DATA_ERROR			13
+#define	UT_DATA_PROTECTION		14
+#define	UT_MEM_ADDRESS_NOT_ALIGNED	15
+#define	UT_PRIVILEGED_ACTION		16
+#define	UT_ASYNC_DATA_ERROR		17
+#define	UT_TRAP_INSTRUCTION_16		18
+#define	UT_TRAP_INSTRUCTION_17		19
+#define	UT_TRAP_INSTRUCTION_18		20
+#define	UT_TRAP_INSTRUCTION_19		21
+#define	UT_TRAP_INSTRUCTION_20		22
+#define	UT_TRAP_INSTRUCTION_21		23
+#define	UT_TRAP_INSTRUCTION_22		24
+#define	UT_TRAP_INSTRUCTION_23		25
+#define	UT_TRAP_INSTRUCTION_24		26
+#define	UT_TRAP_INSTRUCTION_25		27
+#define	UT_TRAP_INSTRUCTION_26		28
+#define	UT_TRAP_INSTRUCTION_27		29
+#define	UT_TRAP_INSTRUCTION_28		30
+#define	UT_TRAP_INSTRUCTION_29		31
+#define	UT_TRAP_INSTRUCTION_30		32
+#define	UT_TRAP_INSTRUCTION_31		33
+#define	UT_INSTRUCTION_MISS		34
+#define	UT_DATA_MISS			35
+#define	UT_MAX				36
+
+#define	ST_SUNOS_SYSCALL		0
+#define	ST_BREAKPOINT			1
+#define	ST_DIVISION_BY_ZERO		2
+#define	ST_FLUSH_WINDOWS		3	/* XXX implement! */
+#define	ST_CLEAN_WINDOW			4
+#define	ST_RANGE_CHECK			5
+#define	ST_FIX_ALIGNMENT		6
+#define	ST_INTEGER_OVERFLOW		7
+/* 8 is 32-bit ABI syscall (old solaris syscall?) */
+#define	ST_BSD_SYSCALL			9
+#define	ST_FP_RESTORE			10
+/* 11-15 are available */
+/* 16 is linux 32 bit syscall (but supposed to be reserved, grr) */
+/* 17 is old linux 64 bit syscall (but supposed to be reserved, grr) */
+/* 16-31 are reserved for user applications (utraps) */
+#define	ST_GETCC			32	/* XXX implement! */
+#define	ST_SETCC			33	/* XXX implement! */
+#define	ST_GETPSR			34	/* XXX implement! */
+#define	ST_SETPSR			35	/* XXX implement! */
+/* 36-63 are available */
+#define	ST_SOLARIS_SYSCALL		64
+#define	ST_SYSCALL			65
+#define	ST_SYSCALL32			66
+/* 67 is reserved to OS source licensee */
+/* 68 is return from deferred trap (not supported) */
+/* 69-95 are reserved to SPARC international */
+/* 96-108 are available */
+/* 109 is linux 64 bit syscall */
+/* 110 is linux 64 bit getcontext (?) */
+/* 111 is linux 64 bit setcontext (?) */
+/* 112-255 are available */
+
+#define	UTH_NOCHANGE			(-1)
+
+#ifndef __ASM__
+
+typedef	int utrap_entry_t;
+typedef void *utrap_handler_t;
+
+#endif
+
+#endif
diff --git a/sys/sun4v/include/varargs.h b/sys/sun4v/include/varargs.h
new file mode 100644
index 000000000000..a168c6e33c32
--- /dev/null
+++ b/sys/sun4v/include/varargs.h
@@ -0,0 +1,91 @@
+/*-
+ * Copyright (c) 2002 David E. O'Brien.  All rights reserved.
+ * Copyright (c) 1992, 1993
+ *	The Regents of the University of California.  All rights reserved.
+ * (c) UNIX System Laboratories, Inc.
+ * All or some portions of this file are derived from material licensed
+ * to the University of California by American Telephone and Telegraph
+ * Co. or Unix System Laboratories, Inc. and are reproduced herein with
+ * the permission of UNIX System Laboratories, Inc.
+ *
+ * This software was developed by the Computer Systems Engineering group
+ * at Lawrence Berkeley Laboratory under DARPA contract BG 91-66 and
+ * contributed to Berkeley.
+ *
+ * All advertising materials mentioning features or use of this software
+ * must display the following acknowledgement:
+ *	This product includes software developed by the University of
+ *	California, Lawrence Berkeley Laboratory.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ *    must display the following acknowledgement:
+ *	This product includes software developed by the University of
+ *	California, Berkeley and its contributors.
+ * 4. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	@(#)varargs.h	8.3 (Berkeley) 3/22/94
+ * $FreeBSD$
+ */
+
+#ifndef _MACHINE_VARARGS_H_
+#define	_MACHINE_VARARGS_H_
+
+#ifndef _SYS_CDEFS_H_
+#error this file needs sys/cdefs.h as a prerequisite
+#endif
+
+#ifdef __GNUCLIKE_BUILTIN_VARARGS
+
+#include <sys/_types.h>
+
+#ifndef _VA_LIST_DECLARED
+#define	_VA_LIST_DECLARED
+typedef	__va_list	va_list;
+#endif
+
+typedef int __builtin_va_alist_t __attribute__((__mode__(__word__)));
+
+#define	va_alist		__builtin_va_alist
+#define	va_dcl			__builtin_va_alist_t __builtin_va_alist; ...
+#define	va_start(ap)		__builtin_varargs_start(ap)
+#define	va_arg(ap, type)	__builtin_va_arg((ap), type)
+#define	va_end(ap)		__builtin_va_end(ap)
+
+#else	/* !__GNUCLIKE_BUILTIN_VARARGS */
+
+#include <machine/stdarg.h>
+
+#define	__va_ellipsis	...
+
+#define	va_alist	__builtin_va_alist
+#define	va_dcl		long __builtin_va_alist; __va_ellipsis
+
+#undef va_start
+#define	va_start(ap) \
+	((ap) = (va_list)__builtin_saveregs())
+
+#endif /* __GNUCLIKE_BUILTIN_VARARGS */
+
+#endif /* !_MACHINE_VARARGS_H_ */
diff --git a/sys/sun4v/include/ver.h b/sys/sun4v/include/ver.h
new file mode 100644
index 000000000000..fec56ce04e9c
--- /dev/null
+++ b/sys/sun4v/include/ver.h
@@ -0,0 +1,74 @@
+/*-
+ * Copyright (c) 2001 Jake Burkholder.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef	_MACHINE_VER_H_
+#define	_MACHINE_VER_H_
+
+#define	VER_MANUF_SHIFT		(48)
+#define	VER_IMPL_SHIFT		(32)
+#define	VER_MASK_SHIFT		(24)
+#define	VER_MAXTL_SHIFT		(8)
+#define	VER_MAXWIN_SHIFT	(0)
+
+#define	VER_MANUF_SIZE		(16)
+#define	VER_IMPL_SIZE		(16)
+#define	VER_MASK_SIZE		(8)
+#define	VER_MAXTL_SIZE		(8)
+#define	VER_MAXWIN_SIZE		(5)
+
+#define	VER_MANUF_MASK		(((1L<<VER_MANUF_SIZE)-1)<<VER_MANUF_SHIFT)
+#define	VER_IMPL_MASK		(((1L<<VER_IMPL_SIZE)-1)<<VER_IMPL_SHIFT)
+#define	VER_MASK_MASK		(((1L<<VER_MASK_SIZE)-1)<<VER_MASK_SHIFT)
+#define	VER_MAXTL_MASK		(((1L<<VER_MAXTL_SIZE)-1)<<VER_MAXTL_SHIFT)
+#define	VER_MAXWIN_MASK		(((1L<<VER_MAXWIN_SIZE)-1)<<VER_MAXWIN_SHIFT)
+
+#define	VER_MANUF(ver) \
+	(((ver) & VER_MANUF_MASK) >> VER_MANUF_SHIFT)
+#define	VER_IMPL(ver) \
+	(((ver) & VER_IMPL_MASK) >> VER_IMPL_SHIFT)
+#define	VER_MASK(ver) \
+	(((ver) & VER_MASK_MASK) >> VER_MASK_SHIFT)
+#define	VER_MAXTL(ver) \
+	(((ver) & VER_MAXTL_MASK) >> VER_MAXTL_SHIFT)
+#define	VER_MAXWIN(ver) \
+	(((ver) & VER_MAXWIN_MASK) >> VER_MAXWIN_SHIFT)
+
+extern int cpu_impl;
+extern char sparc64_model[];
+
+/* Known implementations. */
+#define	CPU_IMPL_SPARC64	0x01
+#define	CPU_IMPL_ULTRASPARCI	0x10
+#define	CPU_IMPL_ULTRASPARCII	0x11
+#define	CPU_IMPL_ULTRASPARCIIi	0x12
+#define	CPU_IMPL_ULTRASPARCIIe	0x13
+#define	CPU_IMPL_ULTRASPARCIII	0x14
+#define	CPU_IMPL_ULTRASPARCIIIp	0x15
+#define	CPU_IMPL_ULTRASPARCIIIi	0x16
+
+#endif /* !_MACHINE_VER_H_ */
diff --git a/sys/sun4v/include/vmparam.h b/sys/sun4v/include/vmparam.h
new file mode 100644
index 000000000000..1ee39bc2b261
--- /dev/null
+++ b/sys/sun4v/include/vmparam.h
@@ -0,0 +1,182 @@
+/*-
+ * Copyright (c) 1990 The Regents of the University of California.
+ * All rights reserved.
+ * Copyright (c) 1994 John S. Dyson
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to Berkeley by
+ * William Jolitz.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ *    must display the following acknowledgement:
+ *      This product includes software developed by the University of
+ *      California, Berkeley and its contributors.
+ * 4. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	from: @(#)vmparam.h     5.9 (Berkeley) 5/12/91
+ *	from: FreeBSD: src/sys/i386/include/vmparam.h,v 1.33 2000/03/30
+ * $FreeBSD$
+ */
+
+
+#ifndef	_MACHINE_VMPARAM_H_
+#define	_MACHINE_VMPARAM_H_
+
+/*
+ * Virtual memory related constants, all in bytes
+ */
+#ifndef MAXTSIZ
+#define	MAXTSIZ		(1*1024*1024*1024)	/* max text size */
+#endif
+#ifndef DFLDSIZ
+#define	DFLDSIZ		(128*1024*1024)		/* initial data size limit */
+#endif
+#ifndef MAXDSIZ
+#define	MAXDSIZ		(1*1024*1024*1024)	/* max data size */
+#endif
+#ifndef	DFLSSIZ
+#define	DFLSSIZ		(128*1024*1024)		/* initial stack size limit */
+#endif
+#ifndef	MAXSSIZ
+#define	MAXSSIZ		(1*1024*1024*1024)	/* max stack size */
+#endif
+#ifndef	SGROWSIZ
+#define	SGROWSIZ	(128*1024)		/* amount to grow stack */
+#endif
+
+/*
+ * The time for a process to be blocked before being very swappable.
+ * This is a number of seconds which the system takes as being a non-trivial
+ * amount of real time.  You probably shouldn't change this;
+ * it is used in subtle ways (fractions and multiples of it are, that is, like
+ * half of a ``long time'', almost a long time, etc.)
+ * It is related to human patience and other factors which don't really
+ * change over time.
+ */
+#define	MAXSLP			20
+
+/*
+ * Address space layout.
+ *
+ * UltraSPARC I and II implement a 44 bit virtual address space.  The address
+ * space is split into 2 regions at each end of the 64 bit address space, with
+ * an out of range "hole" in the middle.  UltraSPARC III implements the full
+ * 64 bit virtual address space, but we don't really have any use for it and
+ * 43 bits of user address space is considered to be "enough", so we ignore it.
+ *
+ * Upper region:	0xffffffffffffffff
+ * 			0xfffff80000000000
+ * 
+ * Hole:		0xfffff7ffffffffff
+ * 			0x0000080000000000
+ *
+ * Lower region:	0x000007ffffffffff
+ * 			0x0000000000000000
+ *
+ * In general we ignore the upper region, and use the lower region as mappable
+ * space.
+ *
+ * We define some interesting address constants:
+ *
+ * VM_MIN_ADDRESS and VM_MAX_ADDRESS define the start and of the entire 64 bit
+ * address space, mostly just for convenience.
+ *
+ * VM_MIN_DIRECT_ADDRESS and VM_MAX_DIRECT_ADDRESS define the start and end
+ * of the direct mapped region.  This maps virtual addresses to physical
+ * addresses directly using 4mb tlb entries, with the physical address encoded
+ * in the lower 43 bits of virtual address.  These mappings are convenient
+ * because they do not require page tables, and because they never change they
+ * do not require tlb flushes.  However, since these mappings are cacheable,
+ * we must ensure that all pages accessed this way are either not double
+ * mapped, or that all other mappings have virtual color equal to physical
+ * color, in order to avoid creating illegal aliases in the data cache.
+ *
+ * VM_MIN_KERNEL_ADDRESS and VM_MAX_KERNEL_ADDRESS define the start and end of
+ * mappable kernel virtual address space.  VM_MIN_KERNEL_ADDRESS is basically
+ * arbitrary, a convenient address is chosen which allows both the kernel text
+ * and data and the prom's address space to be mapped with 1 4mb tsb page.
+ * VM_MAX_KERNEL_ADDRESS is variable, computed at startup time based on the
+ * amount of physical memory available.  Each 4mb tsb page provides 1g of
+ * virtual address space, with the only practical limit being available
+ * phsyical memory.
+ *
+ * VM_MIN_PROM_ADDRESS and VM_MAX_PROM_ADDRESS define the start and end of the
+ * prom address space.  On startup the prom's mappings are duplicated in the
+ * kernel tsb, to allow prom memory to be accessed normally by the kernel.
+ *
+ * VM_MIN_USER_ADDRESS and VM_MAX_USER_ADDRESS define the start and end of the
+ * user address space.  There are some hardware errata about using addresses
+ * at the boundary of the va hole, so we allow just under 43 bits of user
+ * address space.  Note that the kernel and user address spaces overlap, but
+ * this doesn't matter because they use different tlb contexts, and because
+ * the kernel address space is not mapped into each process' address space.
+ */
+#define	VM_MIN_ADDRESS		(0x0000000000000000UL)
+#define	VM_MAX_ADDRESS		(0xffffffffffffffffUL)
+
+#define	VM_MIN_DIRECT_ADDRESS	(0xfffff80000000000UL)
+#define	VM_MAX_DIRECT_ADDRESS	(VM_MAX_ADDRESS)
+
+#define	VM_MIN_KERNEL_ADDRESS	(0x00000000c0000000UL)
+#define	VM_MAX_KERNEL_ADDRESS	(vm_max_kernel_address)
+
+#define	VM_MIN_PROM_ADDRESS	(0x00000000f0000000UL)
+#define	VM_MAX_PROM_ADDRESS	(0x00000000ffffffffUL)
+
+#define	VM_MIN_USER_ADDRESS	(0x0000000000002000UL)
+#define	VM_MAX_USER_ADDRESS	(0x000007fe00000000UL)
+
+#define	VM_MINUSER_ADDRESS	(VM_MIN_USER_ADDRESS)
+#define	VM_MAXUSER_ADDRESS	(VM_MAX_USER_ADDRESS)
+
+#define	KERNBASE		(VM_MIN_KERNEL_ADDRESS)
+#define	USRSTACK		(VM_MAX_USER_ADDRESS)
+
+/*
+ * Virtual size (bytes) for various kernel submaps.
+ */
+#ifndef	VM_KMEM_SIZE
+#define	VM_KMEM_SIZE		(16*1024*1024)
+#endif
+
+/*
+ * How many physical pages per KVA page allocated.
+ * min(max(VM_KMEM_SIZE, Physical memory/VM_KMEM_SIZE_SCALE), VM_KMEM_SIZE_MAX)
+ * is the total KVA space allocated for kmem_map.
+ */
+#ifndef VM_KMEM_SIZE_SCALE
+#define	VM_KMEM_SIZE_SCALE	(3)
+#endif
+
+/*
+ * Initial pagein size of beginning of executable file.
+ */
+#ifndef	VM_INITIAL_PAGEIN
+#define	VM_INITIAL_PAGEIN	16
+#endif
+#define	UMA_MD_SMALL_ALLOC
+extern vm_offset_t vm_max_kernel_address;
+
+#endif /* !_MACHINE_VMPARAM_H_ */
diff --git a/sys/sun4v/include/watch.h b/sys/sun4v/include/watch.h
new file mode 100644
index 000000000000..bb658f9efb53
--- /dev/null
+++ b/sys/sun4v/include/watch.h
@@ -0,0 +1,41 @@
+/*-
+ * Copyright 2001 by Thomas Moestl <tmm@FreeBSD.org>.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+ * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef	_MACHINE_WATCH_H_
+#define	_MACHINE_WATCH_H_
+
+int	watch_phys_set_mask(vm_paddr_t pa, u_long mask);
+int	watch_phys_set(vm_paddr_t pa, int sz);
+vm_paddr_t	watch_phys_get(int *bm);
+void	watch_phys_clear(void);
+int	watch_phys_active(void);
+int	watch_virt_set_mask(vm_offset_t va, u_long mask);
+int	watch_virt_set(vm_offset_t va, int sz);
+vm_offset_t	watch_virt_get(int *bm);
+void	watch_virt_clear(void);
+int	watch_virt_active(void);
+
+#endif	/* _MACHINE_WATCH_H_ */
diff --git a/sys/sun4v/include/wstate.h b/sys/sun4v/include/wstate.h
new file mode 100644
index 000000000000..3d42bf9e326b
--- /dev/null
+++ b/sys/sun4v/include/wstate.h
@@ -0,0 +1,59 @@
+/*-
+ * Copyright (c) 2006 Kip Macy
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef	_MACHINE_WSTATE_H_
+#define	_MACHINE_WSTATE_H_
+
+/*
+ * Window State Register (WSTATE)
+ *
+ *   |------------|
+ *   |OTHER|NORMAL|
+ *   |-----|------|
+ *    5	  3 2    0
+ */
+
+#define	WSTATE_BAD	0	/* unused */
+#define	WSTATE_U32	1	/* 32b stack */
+#define	WSTATE_U64	2	/* 64b stack */
+#define	WSTATE_CLEAN32	3	/* cleanwin workaround, 32b stack */
+#define	WSTATE_CLEAN64	4	/* cleanwin workaround, 64b stack */
+#define	WSTATE_K32	5	/* priv 32b stack */
+#define	WSTATE_K64	6	/* priv 64b stack */
+#define	WSTATE_KMIX	7	/* priv mixed stack */
+
+#define	WSTATE_CLEAN_OFFSET	2
+#define	WSTATE_SHIFT	3	/* normal-to-other shift */
+#define	WSTATE_MASK	7	/* mask for each set */
+#define	WSTATE(o, n)	(((o) << WSTATE_SHIFT) | (n))
+
+#define	WSTATE_USER32	WSTATE(WSTATE_BAD, WSTATE_U32)
+#define	WSTATE_USER64	WSTATE(WSTATE_BAD, WSTATE_U64)
+#define	WSTATE_KERN	WSTATE(WSTATE_U32, WSTATE_K64)
+
+#endif /* !_MACHINE_WSTATE_H_ */
diff --git a/sys/sun4v/mdesc/mdesc_bus_if.m b/sys/sun4v/mdesc/mdesc_bus_if.m
new file mode 100644
index 000000000000..9f5df3d95a89
--- /dev/null
+++ b/sys/sun4v/mdesc/mdesc_bus_if.m
@@ -0,0 +1,127 @@
+#-
+# Copyright (c) 2001, 2003 by Thomas Moestl <tmm@FreeBSD.org>
+# Copyright (c) 2004, 2005 by Marius Strobl <marius@FreeBSD.org>
+# Copyright (c) 2006 by Kip Macy <kmacy@FreeBSD.org>
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+# IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# $FreeBSD$
+
+# Interface for retrieving the package handle and a subset, namely
+# 'compatible', 'device_type', 'model' and 'name', of the standard
+# properties of a device on an Open Firmware assisted bus for use
+# in device drivers. The rest of the standard properties, 'address',
+# 'interrupts', 'reg' and 'status', are not covered by this interface
+# as they are expected to be only of interest in the respective bus
+# driver.
+
+#include <sys/bus.h>
+
+#include <machine/cddl/mdesc.h>
+
+INTERFACE mdesc_bus;
+
+HEADER {
+	struct mdesc_bus_devinfo {
+		char		*mbd_compat;
+		char		*mbd_name;
+		char		*mbd_type;
+		uint64_t         mbd_handle;
+	};
+};
+
+CODE {
+	static mdesc_bus_get_devinfo_t mdesc_bus_default_get_devinfo;
+	static mdesc_bus_get_compat_t mdesc_bus_default_get_compat;
+	static mdesc_bus_get_name_t mdesc_bus_default_get_name;
+	static mdesc_bus_get_type_t mdesc_bus_default_get_type;
+
+	static const struct mdesc_bus_devinfo *
+	mdesc_bus_default_get_devinfo(device_t bus, device_t dev)
+	{
+
+		return (NULL);
+	}
+
+	static const char *
+	mdesc_bus_default_get_compat(device_t bus, device_t dev)
+	{
+
+		return (NULL);
+	}
+
+	static const char *
+	mdesc_bus_default_get_name(device_t bus, device_t dev)
+	{
+
+		return (NULL);
+	}
+
+	static const char *
+	mdesc_bus_default_get_type(device_t bus, device_t dev)
+	{
+
+		return (NULL);
+	}
+
+	static uint64_t
+	mdesc_bus_default_get_handle(device_t bus, device_t dev)
+	{
+
+		return (0);
+	}
+};
+
+# Get the mdesc_bus_devinfo struct for the device dev on the bus. Used for bus
+# drivers which use the generic methods in mdesc_bus_subr.c to implement the
+# reset of this interface. The default method will return NULL, which means
+# there is no such struct associated with the device.
+METHOD const struct mdesc_bus_devinfo * get_devinfo {
+	device_t bus;
+	device_t dev;
+} DEFAULT mdesc_bus_default_get_devinfo;
+
+# Get the alternate firmware name for the device dev on the bus. The default
+# method will return NULL, which means the device doesn't have such a property.
+METHOD const char * get_compat {
+	device_t bus;
+	device_t dev;
+} DEFAULT mdesc_bus_default_get_compat;
+
+# Get the firmware name for the device dev on the bus. The default method will
+# return NULL, which means the device doesn't have such a property.
+METHOD const char * get_name {
+	device_t bus;
+	device_t dev;
+} DEFAULT mdesc_bus_default_get_name;
+
+# Get the firmware device type for the device dev on the bus. The default
+# method will return NULL, which means the device doesn't have such a property.
+METHOD const char * get_type {
+	device_t bus;
+	device_t dev;
+} DEFAULT mdesc_bus_default_get_type;
+
+METHOD uint64_t get_handle {
+	device_t bus;
+	device_t dev;
+} DEFAULT mdesc_bus_default_get_handle;
diff --git a/sys/sun4v/mdesc/mdesc_bus_subr.c b/sys/sun4v/mdesc/mdesc_bus_subr.c
new file mode 100644
index 000000000000..0399168a2a1c
--- /dev/null
+++ b/sys/sun4v/mdesc/mdesc_bus_subr.c
@@ -0,0 +1,133 @@
+/*-
+ * Copyright (c) 2006 Kip Macy
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/types.h>
+#include <sys/param.h>
+#include <sys/kernel.h>
+#include <sys/systm.h>
+#include <sys/bus.h>
+#include <sys/errno.h>
+#include <sys/malloc.h>
+
+#include <machine/mdesc_bus_subr.h>
+#include <machine/cddl/mdesc.h>
+#include <machine/cddl/mdesc_impl.h>
+
+#include "mdesc_bus_if.h"
+
+MALLOC_DEFINE(M_MDPROP, "mdesc", "machine description");
+
+int
+mdesc_bus_gen_setup_devinfo(struct mdesc_bus_devinfo *mbd, mde_cookie_t node)
+{
+	md_t *mdp;
+	
+        if (mbd == NULL)
+                return (ENOMEM);
+	
+	mdp = md_get();
+	
+        /* The 'name' property is considered mandatory. */
+        if ((md_get_prop_alloc(mdp, node, "name", MDET_PROP_STR, (uint8_t **)&mbd->mbd_name)) == -1)
+                return (EINVAL);
+        md_get_prop_alloc(mdp, node, "compatible", MDET_PROP_DAT, (uint8_t **)&mbd->mbd_compat);
+        md_get_prop_alloc(mdp, node, "device-type", MDET_PROP_STR, (uint8_t **)&mbd->mbd_type);
+        md_get_prop_val(mdp, node, "cfg-handle", &mbd->mbd_handle);
+
+	md_put(mdp);
+        return (0);
+}
+
+void
+mdesc_bus_gen_destroy_devinfo(struct mdesc_bus_devinfo *mbd)
+{
+
+        if (mbd == NULL)
+                return;
+        if (mbd->mbd_compat != NULL)
+                free(mbd->mbd_compat, M_MDPROP);
+        if (mbd->mbd_name != NULL)
+                free(mbd->mbd_name, M_MDPROP);
+        if (mbd->mbd_type != NULL)
+                free(mbd->mbd_type, M_MDPROP);
+}
+
+const char *
+mdesc_bus_gen_get_compat(device_t bus, device_t dev)
+{
+        const struct mdesc_bus_devinfo *mbd;
+
+        mbd = MDESC_BUS_GET_DEVINFO(bus, dev);
+        if (mbd == NULL)
+                return (NULL);
+        return (mbd->mbd_compat);
+}
+
+const char *
+mdesc_bus_gen_get_name(device_t bus, device_t dev)
+{
+        const struct mdesc_bus_devinfo *mbd;
+
+        mbd = MDESC_BUS_GET_DEVINFO(bus, dev);
+        if (mbd == NULL)
+                return (NULL);
+        return (mbd->mbd_name);
+}
+
+const char *
+mdesc_bus_gen_get_type(device_t bus, device_t dev)
+{
+        const struct mdesc_bus_devinfo *mbd;
+
+        mbd = MDESC_BUS_GET_DEVINFO(bus, dev);
+        if (mbd == NULL)
+                return (NULL);
+        return (mbd->mbd_type);
+}
+
+uint64_t
+mdesc_bus_gen_get_handle(device_t bus, device_t dev)
+{
+        const struct mdesc_bus_devinfo *mbd;
+
+        mbd = MDESC_BUS_GET_DEVINFO(bus, dev);
+        if (mbd == NULL)
+                return (0);
+        return (mbd->mbd_handle);
+}
+
+
+
+
+
+
+
+
+
diff --git a/sys/sun4v/mdesc/mdesc_diff.c b/sys/sun4v/mdesc/mdesc_diff.c
new file mode 100644
index 000000000000..115016e9ab1a
--- /dev/null
+++ b/sys/sun4v/mdesc/mdesc_diff.c
@@ -0,0 +1,603 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ * $FreeBSD$ 
+ */
+
+/*
+ * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#pragma ident	"@(#)mdesc_diff.c	1.1	06/05/16 SMI"
+
+#include <sys/types.h>
+#ifdef _KERNEL
+#include <sys/systm.h>
+#else /* _KERNEL */
+#include <string.h>
+#include <strings.h>
+#endif /* _KERNEL */
+#include <sys/note.h>
+#include <sys/mdesc.h>
+#include <sys/mdesc_impl.h>
+
+#define	MDD_FREE_CHECK(mdp, ptr, sz)		\
+	do {					\
+		if (ptr) mdp->freep(ptr, sz);	\
+	_NOTE(CONSTCOND) } while (0)
+
+#define	MD_DIFF_MAGIC			0x4D445F4449464621ull	/* 'MD_DIFF!' */
+#define	MD_DIFF_NOMATCH			(-1)
+#define	MD_DIFF_MATCH			(1)
+
+typedef struct {
+	mde_cookie_t	*mdep;
+	uint_t		nelem;
+} md_diff_t;
+
+typedef struct {
+	uint64_t	mdd_magic;
+	md_diff_t	added;
+	md_diff_t	removed;
+	md_diff_t	match1;
+	md_diff_t	match2;
+	void 		*(*allocp)(size_t);
+	void		(*freep)(void *, size_t);
+} md_diff_impl_t;
+
+/*
+ * Internal utility functions
+ */
+static int mdd_scan_for_nodes(md_t *mdp, mde_cookie_t start,
+    char *compnodep, int *countp, mde_cookie_t **nodespp);
+
+static boolean_t mdd_any_dup_nodes(md_impl_t *mdp, md_prop_match_t *pmp,
+    int count, mde_cookie_t *nodesp);
+
+static int mdd_node_list_match(md_impl_t *md1, md_impl_t *md2,
+    md_element_t *match_nodep, mde_cookie_t *match_listp,
+    uint8_t *match_seenp, int start, int end, md_prop_match_t *match_elemsp);
+
+static int mdd_node_compare(md_impl_t *mdap, md_impl_t *mdbp,
+    md_element_t *nodeap, md_element_t *nodebp, md_prop_match_t *match_elemsp);
+
+/*
+ * Given two DAGs and information about how to uniquely identify
+ * the nodes of interest, determine which nodes have been added
+ * to the second MD, removed from the first MD, or exist in both
+ * MDs. This information is recorded and can be accessed using the
+ * opaque cookie returned to the caller.
+ */
+md_diff_cookie_t
+md_diff_init(md_t *md1p, mde_cookie_t start1, md_t *md2p, mde_cookie_t start2,
+    char *compnodep, md_prop_match_t *match_fieldsp)
+{
+	int		idx;
+	md_impl_t	*md1 = (md_impl_t *)md1p;
+	md_impl_t	*md2 = (md_impl_t *)md2p;
+	mde_cookie_t	*md1nodesp = NULL;
+	mde_cookie_t	*md2nodesp = NULL;
+	int		md1count = 0;
+	int		md2count = 0;
+	uint8_t		*seenp = NULL;
+
+	/* variables used to gather results */
+	md_diff_impl_t	*diff_res;
+	mde_cookie_t	*mde_add_scr;
+	mde_cookie_t	*mde_rem_scr;
+	mde_cookie_t	*mde_match1_scr;
+	mde_cookie_t	*mde_match2_scr;
+	int		nadd = 0;
+	int		nrem = 0;
+	int		nmatch = 0;
+
+	/* sanity check params */
+	if ((md1p == NULL) || (md2p == NULL))
+		return (MD_INVAL_DIFF_COOKIE);
+
+	if ((start1 == MDE_INVAL_ELEM_COOKIE) ||
+	    (start2 == MDE_INVAL_ELEM_COOKIE))
+		return (MD_INVAL_DIFF_COOKIE);
+
+	if ((compnodep == NULL) || (match_fieldsp == NULL))
+		return (MD_INVAL_DIFF_COOKIE);
+
+	/*
+	 * Prepare an array of the matching nodes from the first MD.
+	 */
+	if (mdd_scan_for_nodes(md1p,
+	    start1, compnodep, &md1count, &md1nodesp) == -1)
+		return (MD_INVAL_DIFF_COOKIE);
+
+	/* sanity check that all nodes are unique */
+	if (md1nodesp &&
+	    mdd_any_dup_nodes(md1, match_fieldsp, md1count, md1nodesp)) {
+		MDD_FREE_CHECK(md1, md1nodesp, sizeof (mde_cookie_t) *
+		    md1count);
+		return (MD_INVAL_DIFF_COOKIE);
+	}
+
+
+	/*
+	 * Prepare an array of the matching nodes from the second MD.
+	 */
+	if (mdd_scan_for_nodes(md2p,
+	    start2, compnodep, &md2count, &md2nodesp) == -1)
+		return (MD_INVAL_DIFF_COOKIE);
+
+	/* sanity check that all nodes are unique */
+	if (md2nodesp &&
+	    mdd_any_dup_nodes(md2, match_fieldsp, md2count, md2nodesp)) {
+		MDD_FREE_CHECK(md1, md1nodesp, sizeof (mde_cookie_t) *
+		    md1count);
+		MDD_FREE_CHECK(md2, md2nodesp, sizeof (mde_cookie_t) *
+		    md2count);
+		return (MD_INVAL_DIFF_COOKIE);
+	}
+
+	/* setup our result structure */
+	diff_res = md1->allocp(sizeof (md_diff_impl_t));
+	bzero(diff_res, sizeof (md_diff_impl_t));
+	diff_res->allocp = md1->allocp;
+	diff_res->freep = md1->freep;
+	diff_res->mdd_magic = MD_DIFF_MAGIC;
+
+	/*
+	 * Special cases for empty lists
+	 */
+	if ((md1count == 0) && (md2count != 0)) {
+		/* all the nodes found were added */
+		diff_res->added.mdep = md2nodesp;
+		diff_res->added.nelem = md2count;
+		return ((mde_cookie_t)diff_res);
+	}
+
+	if ((md1count != 0) && (md2count == 0)) {
+		/* all the nodes found were removed */
+		diff_res->removed.mdep = md1nodesp;
+		diff_res->removed.nelem = md1count;
+		return ((mde_cookie_t)diff_res);
+	}
+
+	if ((md1count == 0) && (md2count == 0))
+		/* no nodes found */
+		return ((mde_cookie_t)diff_res);
+
+	/*
+	 * Both lists have some elements. Allocate some scratch
+	 * buffers to sort them into our three categories, added,
+	 * removed, and matched pairs.
+	 */
+	mde_add_scr = diff_res->allocp(sizeof (mde_cookie_t) * md2count);
+	mde_rem_scr = diff_res->allocp(sizeof (mde_cookie_t) * md1count);
+	mde_match1_scr = diff_res->allocp(sizeof (mde_cookie_t) * md1count);
+	mde_match2_scr = diff_res->allocp(sizeof (mde_cookie_t) * md2count);
+
+	/* array of seen flags only needed for md2 */
+	seenp = (uint8_t *)diff_res->allocp(sizeof (uint8_t) * md2count);
+	bzero(seenp, sizeof (uint8_t) * md2count);
+
+	/*
+	 * Make a pass through the md1 node array. Make note of
+	 * any nodes not in the md2 array, indicating that they
+	 * have been removed. Also keep track of the nodes that
+	 * are present in both arrays for the matched pair results.
+	 */
+	for (idx = 0; idx < md1count; idx++) {
+
+		md_element_t *elem = &(md1->mdep[md1nodesp[idx]]);
+
+		int match = mdd_node_list_match(md1, md2, elem, md2nodesp,
+		    seenp, 0, md2count - 1, match_fieldsp);
+
+		if (match == MD_DIFF_NOMATCH)
+			/* record deleted node */
+			mde_rem_scr[nrem++] = md1nodesp[idx];
+		else {
+			/* record matched node pair */
+			mde_match1_scr[nmatch] = md1nodesp[idx];
+			mde_match2_scr[nmatch] = md2nodesp[match];
+			nmatch++;
+
+			/* mark that this match has been recorded */
+			seenp[match] = 1;
+		}
+	}
+
+	/*
+	 * Make a pass through the md2 array. Any nodes that have
+	 * not been marked as seen have been added.
+	 */
+	for (idx = 0; idx < md2count; idx++) {
+		if (!seenp[idx])
+			/* record added node */
+			mde_add_scr[nadd++] = md2nodesp[idx];
+	}
+
+	/* fill in the added node list */
+	if (nadd) {
+		int addsz = sizeof (mde_cookie_t) * nadd;
+		diff_res->added.mdep = (mde_cookie_t *)diff_res->allocp(addsz);
+
+		bcopy(mde_add_scr, diff_res->added.mdep, addsz);
+
+		diff_res->added.nelem = nadd;
+	}
+
+	/* fill in the removed node list */
+	if (nrem) {
+		int remsz = sizeof (mde_cookie_t) * nrem;
+		diff_res->removed.mdep =
+		    (mde_cookie_t *)diff_res->allocp(remsz);
+
+		bcopy(mde_rem_scr, diff_res->removed.mdep, remsz);
+		diff_res->removed.nelem = nrem;
+	}
+
+	/* fill in the matching node lists */
+	if (nmatch) {
+		int matchsz = sizeof (mde_cookie_t) * nmatch;
+		diff_res->match1.mdep =
+		    (mde_cookie_t *)diff_res->allocp(matchsz);
+		diff_res->match2.mdep =
+		    (mde_cookie_t *)diff_res->allocp(matchsz);
+
+		bcopy(mde_match1_scr, diff_res->match1.mdep, matchsz);
+		bcopy(mde_match2_scr, diff_res->match2.mdep, matchsz);
+		diff_res->match1.nelem = nmatch;
+		diff_res->match2.nelem = nmatch;
+	}
+
+	/* clean up */
+	md1->freep(md1nodesp, sizeof (mde_cookie_t) * md1count);
+	md2->freep(md2nodesp, sizeof (mde_cookie_t) * md2count);
+
+	diff_res->freep(mde_add_scr, sizeof (mde_cookie_t) * md2count);
+	diff_res->freep(mde_rem_scr, sizeof (mde_cookie_t) * md1count);
+	diff_res->freep(mde_match1_scr, sizeof (mde_cookie_t) * md1count);
+	diff_res->freep(mde_match2_scr, sizeof (mde_cookie_t) * md2count);
+
+	diff_res->freep(seenp, sizeof (uint8_t) * md2count);
+
+	return ((md_diff_cookie_t)diff_res);
+}
+
+/*
+ * Returns an array of the nodes added to the second MD in a
+ * previous md_diff_init() call. Returns the number of elements
+ * in the returned array. If the value is zero, the pointer
+ * passed back will be NULL.
+ */
+int
+md_diff_added(md_diff_cookie_t mdd, mde_cookie_t **mde_addedp)
+{
+	md_diff_impl_t	*mddp = (md_diff_impl_t *)mdd;
+
+	if ((mddp == NULL) || (mddp->mdd_magic != MD_DIFF_MAGIC))
+		return (-1);
+
+	*mde_addedp = mddp->added.mdep;
+
+	return (mddp->added.nelem);
+}
+
+/*
+ * Returns an array of the nodes removed from the first MD in a
+ * previous md_diff_init() call. Returns the number of elements
+ * in the returned array. If the value is zero, the pointer
+ * passed back will be NULL.
+ */
+int
+md_diff_removed(md_diff_cookie_t mdd, mde_cookie_t **mde_removedp)
+{
+	md_diff_impl_t	*mddp = (md_diff_impl_t *)mdd;
+
+	if ((mddp == NULL) || (mddp->mdd_magic != MD_DIFF_MAGIC))
+		return (-1);
+
+	*mde_removedp = mddp->removed.mdep;
+
+	return (mddp->removed.nelem);
+}
+
+/*
+ * Returns a pair of parallel arrays that contain nodes that were
+ * considered matching based on the match criteria passed in to
+ * a previous md_diff_init() call. Returns the number of elements
+ * in the arrays. If the value is zero, both pointers passed back
+ * will be NULL.
+ */
+int
+md_diff_matched(md_diff_cookie_t mdd, mde_cookie_t **mde_match1p,
+    mde_cookie_t **mde_match2p)
+{
+	md_diff_impl_t	*mddp = (md_diff_impl_t *)mdd;
+
+	if ((mddp == NULL) || (mddp->mdd_magic != MD_DIFF_MAGIC))
+		return (-1);
+
+	*mde_match1p = mddp->match1.mdep;
+	*mde_match2p = mddp->match2.mdep;
+
+	return (mddp->match1.nelem);
+}
+
+/*
+ * Deallocate any storage used to store results of a previous
+ * md_diff_init() call. Returns 0 on success and -1 on failure.
+ */
+int
+md_diff_fini(md_diff_cookie_t mdd)
+{
+	md_diff_impl_t	*mddp = (md_diff_impl_t *)mdd;
+
+	if ((mddp == NULL) || (mddp->mdd_magic != MD_DIFF_MAGIC))
+		return (-1);
+
+	mddp->mdd_magic = 0;
+
+	MDD_FREE_CHECK(mddp, mddp->added.mdep, mddp->added.nelem *
+	    sizeof (mde_cookie_t));
+
+	MDD_FREE_CHECK(mddp, mddp->removed.mdep, mddp->removed.nelem *
+	    sizeof (mde_cookie_t));
+
+	MDD_FREE_CHECK(mddp, mddp->match1.mdep, mddp->match1.nelem *
+	    sizeof (mde_cookie_t));
+
+	MDD_FREE_CHECK(mddp, mddp->match2.mdep, mddp->match2.nelem *
+	    sizeof (mde_cookie_t));
+
+	mddp->freep(mddp, sizeof (md_diff_impl_t));
+
+	return (0);
+}
+
+/*
+ * Walk the "fwd" DAG in an MD and return an array of nodes that are
+ * of the specified type. The start param is used to start the walk
+ * from an arbitrary location in the DAG. Returns an array of nodes
+ * as well as a count of the number of nodes in the array.  If the
+ * count is zero, the node pointer will be passed back as NULL.
+ *
+ * Returns: 0 success; -1 failure
+ */
+static int
+mdd_scan_for_nodes(md_t *mdp,
+    mde_cookie_t start, char *compnodep, int *countp, mde_cookie_t **nodespp)
+{
+	mde_str_cookie_t	cname;
+	mde_str_cookie_t	aname;
+	md_impl_t		*mdip = (md_impl_t *)mdp;
+
+	if (mdip == NULL)
+		return (-1);
+
+	cname = md_find_name(mdp, compnodep);
+	aname = md_find_name(mdp, "fwd");
+
+	/* get the number of nodes of interest in the DAG */
+	*countp = md_scan_dag(mdp, start, cname, aname, NULL);
+	if (*countp == 0) {
+		*nodespp = NULL;
+		return (0);
+	}
+
+	/* allocate the storage */
+	*nodespp = mdip->allocp(sizeof (mde_cookie_t) * (*countp));
+
+	/* populate our array with the matching nodes */
+	(void) md_scan_dag(mdp, start, cname, aname, *nodespp);
+
+	return (0);
+}
+
+/*
+ * Walk an array of nodes and check if there are any duplicate
+ * nodes. A duplicate is determined based on the specified match
+ * criteria. Returns B_TRUE if there are any duplicates and B_FALSE
+ * otherwise.
+ */
+static boolean_t
+mdd_any_dup_nodes(md_impl_t *mdp, md_prop_match_t *pmp, int count,
+    mde_cookie_t *nodesp)
+{
+	int		idx;
+	int		match;
+	md_element_t	*elem;
+
+	ASSERT(count > 0 || nodesp == NULL);
+
+	for (idx = 0; idx < count; idx++) {
+		elem = &(mdp->mdep[nodesp[idx]]);
+
+		match = mdd_node_list_match(mdp, mdp, elem, nodesp, NULL,
+		    idx + 1, count - 1, pmp);
+
+		if (match != MD_DIFF_NOMATCH)
+			return (B_TRUE);
+	}
+
+	return (B_FALSE);
+}
+
+/*
+ * Given a node and a array of nodes, compare the node to all elements
+ * in the specified start-end range of the array. If the node matches
+ * one of the nodes in the array, return the index of that node. Otherwise
+ * return MD_DIFF_NOMATCH.
+ *
+ * The optional seen array parameter can be used to optimize repeated
+ * calls to this function. If the seen array indicates that an element
+ * has already been matched, the full comparison is not necessary.
+ */
+static int
+mdd_node_list_match(md_impl_t *md1, md_impl_t *md2, md_element_t *match_nodep,
+    mde_cookie_t *match_listp, uint8_t *match_seenp, int start, int end,
+    md_prop_match_t *match_elemsp)
+{
+	int		match;
+	int		idx;
+	md_element_t	*elem;
+
+	for (idx = start; idx <= end; idx++) {
+
+		if ((match_seenp != NULL) && (match_seenp[idx]))
+			continue;
+
+		elem = &(md2->mdep[match_listp[idx]]);
+
+		match = mdd_node_compare(md1, md2, match_nodep, elem,
+		    match_elemsp);
+		if (match == MD_DIFF_MATCH)
+			return (idx);
+	}
+
+	return (MD_DIFF_NOMATCH);
+}
+
+/*
+ * Given two nodes and a list of properties, compare the nodes.
+ * A match is concluded if both nodes have all of the specified
+ * properties and all the values of those properties are the
+ * same. Returns MD_DIFF_NOMATCH if the nodes do not match and
+ * MD_DIFF_MATCH otherwise.
+ */
+static int
+mdd_node_compare(md_impl_t *mdap, md_impl_t *mdbp, md_element_t *nodeap,
+    md_element_t *nodebp, md_prop_match_t *match_elemsp)
+{
+	md_element_t	*ap;
+	md_element_t	*bp;
+	boolean_t	nodea_interest;
+	boolean_t	nodeb_interest;
+	int		idx;
+
+	/* make sure we are starting at the beginning of the nodes */
+	if ((MDE_TAG(nodeap) != MDET_NODE) || (MDE_TAG(nodebp) != MDET_NODE))
+		return (MD_DIFF_NOMATCH);
+
+	for (idx = 0; match_elemsp[idx].type != MDET_LIST_END; idx++) {
+
+		int type;
+
+		nodea_interest = B_FALSE;
+		nodeb_interest = B_FALSE;
+
+		type = match_elemsp[idx].type;
+
+		/*
+		 * Check node A for the property of interest
+		 */
+		for (ap = nodeap; MDE_TAG(ap) != MDET_NODE_END; ap++) {
+			char *elemname;
+
+			if (MDE_TAG(ap) != type)
+				continue;
+
+			elemname = mdap->namep + MDE_NAME(ap);
+
+			if (strcmp(elemname, match_elemsp[idx].namep) == 0) {
+				/* found the property of interest */
+				nodea_interest = B_TRUE;
+				break;
+			}
+		}
+
+		/* node A is not of interest */
+		if (!nodea_interest)
+			return (MD_DIFF_NOMATCH);
+
+		/*
+		 * Check node B for the property of interest
+		 */
+		for (bp = nodebp; MDE_TAG(bp) != MDET_NODE_END; bp++) {
+			char *elemname;
+
+			if (MDE_TAG(bp) != type)
+				continue;
+
+			elemname = mdbp->namep + MDE_NAME(bp);
+
+			if (strcmp(elemname, match_elemsp[idx].namep) == 0) {
+				nodeb_interest = B_TRUE;
+				break;
+			}
+		}
+
+		/* node B is not of interest */
+		if (!nodeb_interest)
+			return (MD_DIFF_NOMATCH);
+
+		/*
+		 * Both nodes have the property of interest. The
+		 * nodes are not a match unless the value of that
+		 * property match
+		 */
+		switch (type) {
+		case MDET_PROP_VAL:
+			if (MDE_PROP_VALUE(ap) != MDE_PROP_VALUE(bp))
+				return (MD_DIFF_NOMATCH);
+			break;
+
+		case MDET_PROP_STR: {
+			char *stra = (char *)(mdap->datap +
+			    MDE_PROP_DATA_OFFSET(ap));
+			char *strb = (char *)(mdbp->datap +
+			    MDE_PROP_DATA_OFFSET(bp));
+
+			if (strcmp(stra, strb) != 0)
+				return (MD_DIFF_NOMATCH);
+			break;
+		}
+
+		case MDET_PROP_DAT: {
+
+			caddr_t dataa;
+			caddr_t datab;
+
+			if (MDE_PROP_DATA_LEN(ap) != MDE_PROP_DATA_LEN(bp))
+				return (MD_DIFF_NOMATCH);
+
+			dataa = (caddr_t)(mdap->datap +
+			    MDE_PROP_DATA_OFFSET(ap));
+			datab = (caddr_t)(mdbp->datap +
+			    MDE_PROP_DATA_OFFSET(bp));
+
+			if (memcmp(dataa, datab, MDE_PROP_DATA_LEN(ap)) != 0)
+				return (MD_DIFF_NOMATCH);
+
+			break;
+		}
+
+		default:
+			/* unsupported prop type */
+			return (MD_DIFF_NOMATCH);
+		}
+	}
+
+	/*
+	 * All the specified properties exist in both
+	 * nodes and have the same value. The two nodes
+	 * match.
+	 */
+
+	return (MD_DIFF_MATCH);
+}
diff --git a/sys/sun4v/mdesc/mdesc_findname.c b/sys/sun4v/mdesc/mdesc_findname.c
new file mode 100644
index 000000000000..c9616192a113
--- /dev/null
+++ b/sys/sun4v/mdesc/mdesc_findname.c
@@ -0,0 +1,67 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License").  You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ * $FreeBSD$
+ */
+/*
+ * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+
+
+#include <sys/types.h>
+
+#ifdef _KERNEL
+#include <sys/systm.h>
+#else
+#include <strings.h>
+#endif
+
+#include <machine/cddl/mdesc.h>
+#include <machine/cddl/mdesc_impl.h>
+
+mde_str_cookie_t
+md_find_name(md_t *ptr, char *namep)
+{
+	int idx, len;
+	md_impl_t *mdp;
+
+	mdp = (md_impl_t *)ptr;
+
+	/*
+	 * At some point init should build a local hash table to
+	 * speed these name searches ... for now use brute force
+	 * because the machien descriptions are so small anyway.
+	 */
+
+	for (idx = 0; idx < mdp->name_blk_size; idx += len) {
+		char *p;
+
+		p = &(mdp->namep[idx]);
+
+		len = strlen(p)+1;
+
+		if (strcmp(p, namep) == 0)
+			return ((mde_str_cookie_t)idx);
+	}
+
+	return (MDE_INVAL_STR_COOKIE);
+}
diff --git a/sys/sun4v/mdesc/mdesc_findnodeprop.c b/sys/sun4v/mdesc/mdesc_findnodeprop.c
new file mode 100644
index 000000000000..558b97631b17
--- /dev/null
+++ b/sys/sun4v/mdesc/mdesc_findnodeprop.c
@@ -0,0 +1,76 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License").  You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ * $FreeBSD$
+ */
+/*
+ * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+
+#include <sys/types.h>
+#include <sys/param.h>
+#include <machine/cddl/mdesc.h>
+#include <machine/cddl/mdesc_impl.h>
+
+mde_cookie_t
+md_find_node_prop(md_impl_t *mdp,
+	mde_cookie_t node,
+	mde_str_cookie_t prop_name,
+	int tag_type)
+{
+	md_element_t *mdep;
+	int idx;
+
+	if (node == MDE_INVAL_ELEM_COOKIE ||
+	    prop_name == MDE_INVAL_STR_COOKIE) {
+		return (MDE_INVAL_ELEM_COOKIE);
+	}
+
+	idx = (int)node;
+	mdep = &(mdp->mdep[idx]);
+
+		/* Skip over any empty elements */
+	while (MDE_TAG(mdep) == MDET_NULL) {
+		idx++;
+		mdep++;
+	}
+
+		/* see if cookie is infact a node */
+	if (MDE_TAG(mdep) != MDET_NODE) {
+		return (MDE_INVAL_ELEM_COOKIE);
+	}
+
+	/*
+	 * Simply walk the elements in the node
+	 * looking for a property with a matching name.
+	 */
+
+	for (idx++, mdep++; MDE_TAG(mdep) != MDET_NODE_END; idx++, mdep++) {
+		if (MDE_TAG(mdep) == tag_type) {
+			if (MDE_NAME(mdep) == prop_name) {
+				return ((mde_cookie_t)idx);
+			}
+		}
+	}
+
+	return (MDE_INVAL_ELEM_COOKIE);	/* no such property name */
+}
diff --git a/sys/sun4v/mdesc/mdesc_fini.c b/sys/sun4v/mdesc/mdesc_fini.c
new file mode 100644
index 000000000000..ff32ce8c8b50
--- /dev/null
+++ b/sys/sun4v/mdesc/mdesc_fini.c
@@ -0,0 +1,48 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ * $FreeBSD$
+ */
+
+/*
+ * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#pragma ident	"@(#)mdesc_fini.c	1.4	06/05/16 SMI"
+
+#include <sys/types.h>
+#include <sys/mdesc.h>
+#include <sys/mdesc_impl.h>
+
+/*
+ * Cleanup the internal MD structure. Does not
+ * deallocate the buffer holding the MD.
+ */
+int
+md_fini(md_t *ptr)
+{
+	md_impl_t *mdp;
+
+	mdp = (md_impl_t *)ptr;
+
+	mdp->freep(mdp, sizeof (md_impl_t));
+
+	return (0);
+}
diff --git a/sys/sun4v/mdesc/mdesc_getbinsize.c b/sys/sun4v/mdesc/mdesc_getbinsize.c
new file mode 100644
index 000000000000..c3e2386c86e7
--- /dev/null
+++ b/sys/sun4v/mdesc/mdesc_getbinsize.c
@@ -0,0 +1,46 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ * $FreeBSD$
+ */
+
+/*
+ * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#pragma ident	"@(#)mdesc_getbinsize.c	1.1	06/05/16 SMI"
+
+#include <sys/types.h>
+#include <sys/param.h>
+#include <sys/mdesc.h>
+#include <sys/mdesc_impl.h>
+
+size_t
+md_get_bin_size(md_t *ptr)
+{
+	md_impl_t *mdp;
+
+	mdp = (md_impl_t *)ptr;
+
+	if (mdp == NULL)
+		return (0);
+
+	return (mdp->size);
+}
diff --git a/sys/sun4v/mdesc/mdesc_getgen.c b/sys/sun4v/mdesc/mdesc_getgen.c
new file mode 100644
index 000000000000..7d1ad5198b46
--- /dev/null
+++ b/sys/sun4v/mdesc/mdesc_getgen.c
@@ -0,0 +1,46 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ * $FreeBSD$
+ */
+
+/*
+ * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#pragma ident	"@(#)mdesc_getgen.c	1.1	06/05/16 SMI"
+
+#include <sys/types.h>
+#include <sys/param.h>
+#include <sys/mdesc.h>
+#include <sys/mdesc_impl.h>
+
+uint64_t
+md_get_gen(md_t *ptr)
+{
+	md_impl_t *mdp;
+
+	mdp = (md_impl_t *)ptr;
+
+	if (mdp == NULL)
+		return (MDESC_INVAL_GEN);
+
+	return (mdp->gen);
+}
diff --git a/sys/sun4v/mdesc/mdesc_getpropdata.c b/sys/sun4v/mdesc/mdesc_getpropdata.c
new file mode 100644
index 000000000000..199a4362e498
--- /dev/null
+++ b/sys/sun4v/mdesc/mdesc_getpropdata.c
@@ -0,0 +1,65 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License").  You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ * $FreeBSD$
+ */
+/*
+ * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#pragma ident	"@(#)mdesc_getpropdata.c	1.2	05/06/08 SMI"
+
+#include <sys/types.h>
+#include <sys/mdesc.h>
+#include <sys/mdesc_impl.h>
+
+int
+md_get_prop_data(md_t *ptr, mde_cookie_t node, char *namep,
+	uint8_t **datap, int *lenp)
+{
+	mde_str_cookie_t prop_name;
+	md_impl_t *mdp;
+	mde_cookie_t	elem;
+
+	mdp = (md_impl_t *)ptr;
+
+	if (node == MDE_INVAL_ELEM_COOKIE) {
+		return (-1);
+	}
+
+	prop_name = md_find_name(ptr, namep);
+	if (prop_name == MDE_INVAL_STR_COOKIE) {
+		return (-1);
+	}
+
+	elem = md_find_node_prop(mdp, node, prop_name, MDET_PROP_DAT);
+
+	if (elem != MDE_INVAL_ELEM_COOKIE) {
+		md_element_t *mdep;
+		mdep = &(mdp->mdep[(int)elem]);
+
+		*lenp = (int)MDE_PROP_DATA_LEN(mdep);
+		*datap = mdp->datap+(int)MDE_PROP_DATA_OFFSET(mdep);
+		return (0);
+	}
+
+	return (-1);	/* no such property name */
+}
diff --git a/sys/sun4v/mdesc/mdesc_getpropstr.c b/sys/sun4v/mdesc/mdesc_getpropstr.c
new file mode 100644
index 000000000000..61f9c307884f
--- /dev/null
+++ b/sys/sun4v/mdesc/mdesc_getpropstr.c
@@ -0,0 +1,62 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License").  You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ * $FreeBSD$
+ */
+/*
+ * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#include <sys/types.h>
+#include <machine/cddl/mdesc.h>
+#include <machine/cddl/mdesc_impl.h>
+
+int
+md_get_prop_str(md_t *ptr, mde_cookie_t node, char *namep, char **strp)
+{
+	mde_str_cookie_t prop_name;
+	md_impl_t	*mdp;
+	mde_cookie_t	elem;
+
+	mdp = (md_impl_t *)ptr;
+
+	if (node == MDE_INVAL_ELEM_COOKIE) {
+		return (-1);
+	}
+
+	prop_name = md_find_name(ptr, namep);
+	if (prop_name == MDE_INVAL_STR_COOKIE) {
+		return (-1);
+	}
+
+	elem = md_find_node_prop(mdp, node, prop_name, MDET_PROP_STR);
+
+	if (elem != MDE_INVAL_ELEM_COOKIE) {
+		md_element_t *mdep;
+		mdep = &(mdp->mdep[(int)elem]);
+
+		*strp = (char *)(mdp->datap+
+			MDE_PROP_DATA_OFFSET(mdep));
+		return (0);
+	}
+
+	return (-1);	/* no such property name */
+}
diff --git a/sys/sun4v/mdesc/mdesc_getpropval.c b/sys/sun4v/mdesc/mdesc_getpropval.c
new file mode 100644
index 000000000000..3a17d5c7b01d
--- /dev/null
+++ b/sys/sun4v/mdesc/mdesc_getpropval.c
@@ -0,0 +1,68 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License").  You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ * $FreeBSD$
+ */
+/*
+ * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+
+#include <sys/types.h>
+#ifdef _KERNEL
+#include <sys/systm.h>
+#else
+#include <strings.h>
+#endif
+
+#include <machine/cddl/mdesc.h>
+#include <machine/cddl/mdesc_impl.h>
+
+int
+md_get_prop_val(md_t *ptr, mde_cookie_t node, char *namep, uint64_t *valp)
+{
+	mde_str_cookie_t prop_name;
+	md_impl_t	*mdp;
+	mde_cookie_t	elem;
+
+	mdp = (md_impl_t *)ptr;
+
+	if (node == MDE_INVAL_ELEM_COOKIE) {
+		return (-1);
+	}
+
+	prop_name = md_find_name(ptr, namep);
+	if (prop_name == MDE_INVAL_STR_COOKIE) {
+		return (-1);
+	}
+
+	elem = md_find_node_prop(mdp, node, prop_name, MDET_PROP_VAL);
+
+	if (elem != MDE_INVAL_ELEM_COOKIE) {
+		md_element_t *mdep;
+		mdep = &(mdp->mdep[(int)elem]);
+
+		*valp = MDE_PROP_VALUE(mdep);
+		return (0);
+	}
+
+	return (-1);	/* no such property name */
+}
diff --git a/sys/sun4v/mdesc/mdesc_init.c b/sys/sun4v/mdesc/mdesc_init.c
new file mode 100644
index 000000000000..4625cd6a05dc
--- /dev/null
+++ b/sys/sun4v/mdesc/mdesc_init.c
@@ -0,0 +1,303 @@
+/*-
+ * Copyright (c) 2006 Kip Macy
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/types.h>
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/bus.h>
+#include <sys/kernel.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
+#include <vm/vm.h>
+#include <vm/pmap.h>
+
+#include <machine/hypervisor_api.h>
+#include <machine/cddl/mdesc.h>
+#include <machine/cddl/mdesc_impl.h>
+
+
+static void mdesc_postvm_init(void *);
+/*
+ * It doesn't really matter when this happens right now
+ * it just needs to happen before device initialization 
+ * and after VM initialization. Later on we will end up 
+ * doing this statically from pmap_bootstrap so that we 
+ * can kill off all calls to OBP. OBP removal is not in
+ * the critical path for sun4v at this time.
+ */
+SYSINIT(mdesc_init, SI_SUB_CPU, SI_ORDER_FIRST, mdesc_postvm_init, NULL);
+
+
+#define UNIMPLEMENTED panic("%s not implemented.", __FUNCTION__)
+
+typedef struct mdesc_memops_ {
+        void            *(*mm_buf_alloc)(size_t size, size_t align);
+        void            (*mm_buf_free)(void *, size_t size);
+        void            *(*mm_meta_alloc)(size_t size);
+        void            (*mm_meta_free)(void *, size_t size);
+} mdesc_memops_t;
+
+typedef struct mdesc_ {
+        uint64_t        md_gen;         /* md-generation# */
+        struct mtx      md_lock;       
+        void           *md_addr;        /* address of raw MD */
+        uint64_t        md_size;        /* size of raw MD */
+        uint64_t        md_buf_size;    /* size of buffer allocated for MD */
+        int             md_refcount;    /* reference count */
+        mdesc_memops_t *md_memops;      /* Memory operations for this MD */
+} mdesc_t;
+
+
+
+static mdesc_t        *curr_mdesc = NULL;
+static struct mtx      curr_mdesc_lock;
+static mdesc_memops_t *mdesc_memops;
+
+static void *mdesc_boot_buf_alloc(size_t size, size_t align);
+static void mdesc_boot_buf_free(void *buf, size_t align);
+static void *mdesc_boot_meta_alloc(size_t size);
+static void mdesc_boot_meta_free(void *buf, size_t size);
+
+static void *mdesc_buf_alloc(size_t size, size_t align);
+static void mdesc_buf_free(void *buf, size_t align);
+static void *mdesc_meta_alloc(size_t size);
+static void mdesc_meta_free(void *buf, size_t size);
+
+static mdesc_memops_t mdesc_boot_memops = {
+	mdesc_boot_buf_alloc,
+	mdesc_boot_buf_free,
+	mdesc_boot_meta_alloc,
+	mdesc_boot_meta_free,
+};
+
+static mdesc_memops_t mdesc_generic_memops = {
+	mdesc_buf_alloc,
+	mdesc_buf_free,
+	mdesc_meta_alloc,
+	mdesc_meta_free,
+};
+
+static void *
+mdesc_boot_buf_alloc(size_t size, size_t align)
+{
+	UNIMPLEMENTED;
+}
+
+static void 
+mdesc_boot_buf_free(void *buf, size_t align)
+{
+	UNIMPLEMENTED;
+}
+
+static void *
+mdesc_boot_meta_alloc(size_t size)
+{
+	UNIMPLEMENTED;
+}
+
+static void 
+mdesc_boot_meta_free(void *buf, size_t size)
+{
+	UNIMPLEMENTED;
+}
+
+static void *
+mdesc_buf_alloc(size_t size, size_t align)
+{
+	return contigmalloc(size, M_MDPROP, M_WAITOK, phys_avail[0],
+			    phys_avail[1], align, (1UL<<34));
+}
+
+static void 
+mdesc_buf_free(void *buf, size_t align)
+{
+	contigfree(buf, PAGE_SIZE /*fix me*/, M_MDPROP);
+}
+
+static void *
+mdesc_meta_alloc(size_t size)
+{
+	return malloc(size, M_MDPROP, M_WAITOK);
+}
+
+static void 
+mdesc_meta_free(void *buf, size_t size)
+{
+	free(buf, M_MDPROP);
+}
+
+void 
+mdesc_init(void)
+{
+
+	mtx_init(&curr_mdesc_lock, "current machine description lock", NULL, MTX_DEF);
+	mdesc_memops = &mdesc_boot_memops;
+}
+
+static void 
+mdesc_postvm_init(void *unused)
+{
+	mdesc_memops = &mdesc_generic_memops;
+}
+
+static mdesc_t *
+mdesc_alloc(void)
+{
+	mdesc_t *mdp;
+	
+	mdp = (mdesc_t *)(mdesc_memops->mm_meta_alloc)(sizeof(mdesc_t));
+	if (mdp != NULL) {
+		bzero(mdp, sizeof(*mdp));
+		mdp->md_memops = mdesc_memops;
+		mtx_init(&mdp->md_lock, "machine descriptor lock", NULL, MTX_DEF);
+	}
+	
+	return (mdp);
+}
+
+int 
+mdesc_update(void)
+{
+	uint64_t rv;
+	uint64_t mdesc_size, mdesc_buf_size = 0;
+	void    *buf = NULL;
+#ifdef notyet
+	uint64_t gen;
+#endif
+	do {
+		if (buf != NULL)
+			(mdesc_memops->mm_buf_free)(buf, mdesc_buf_size);
+
+		mdesc_size = 0LL;
+		hv_mach_desc((uint64_t)0, &mdesc_size);
+
+		mdesc_size = mdesc_buf_size = round_page(mdesc_size);
+		
+		if ((buf = (*mdesc_memops->mm_buf_alloc)(mdesc_buf_size, PAGE_SIZE)) == NULL) {
+			rv = -1;
+			goto done;
+		}
+		
+		rv = hv_mach_desc(vtophys(buf), &mdesc_size);
+
+		if (rv != H_EOK && rv != H_EINVAL) {
+			rv = -1;
+			goto done;
+		}
+	} while (mdesc_size > mdesc_buf_size);
+	
+	KASSERT(rv == H_EOK, ("unexpected return from hv_mach_desc"));
+	
+	/* XXX we ignore the generation count... not all versions may 
+	 * support it  
+	 */
+
+	if (curr_mdesc->md_refcount == 0) {
+		(*mdesc_memops->mm_buf_free) (curr_mdesc->md_addr, curr_mdesc->md_size);
+	} else {
+		panic("out of date machine description list not implemented");
+	}
+
+	mtx_lock(&curr_mdesc_lock);
+#ifdef notyet
+	curr_mdesc->md_gen = gen;
+#endif
+	curr_mdesc->md_addr = buf;
+	curr_mdesc->md_size = mdesc_size;
+	curr_mdesc->md_buf_size = mdesc_buf_size;
+	mtx_unlock(&curr_mdesc_lock);
+	
+	return (rv);
+
+ done:
+	if (buf != NULL)
+		(*mdesc_memops->mm_buf_free)(buf, mdesc_buf_size);
+	return (rv);
+}
+
+md_t *
+md_get(void)
+{
+	md_t *mdp;
+	
+
+	/*
+	 * XXX This should actually happen in init
+	 */
+	if (curr_mdesc == NULL) {
+		if ((curr_mdesc = mdesc_alloc()) == NULL)
+			panic("machine description allocation failed");
+		if (mdesc_update())
+			panic("machine description update failed");
+	}
+		
+	mtx_lock(&curr_mdesc_lock);
+	curr_mdesc->md_refcount++;
+	mdp = md_init_intern(curr_mdesc->md_addr, 
+			     curr_mdesc->md_memops->mm_meta_alloc,
+			     curr_mdesc->md_memops->mm_meta_free);
+	mtx_unlock(&curr_mdesc_lock);
+
+	return (mdp);
+}
+
+void
+md_put(md_t *ptr)
+{
+	md_impl_t *mdp;
+	
+	mdp = (md_impl_t *)ptr;
+
+#ifdef notyet
+	mtx_lock(&curr_mdesc_lock)
+
+	/* XXX check against generation */
+	if (curr_mdesc->md_gen == mdp->md_gen) {
+		curr_mdesc->md_refcount--;
+		mtx_unlock(&curr_mdesc_lock)
+		goto done;
+	}
+	mtx_unlock(&curr_mdesc_lock);
+	/*
+	 * MD is on the out of date list 
+	 */
+
+ done:
+#endif
+	/*
+	 * We don't keep an out of date list currently
+	 */
+	mtx_lock(&curr_mdesc_lock);
+	curr_mdesc->md_refcount--;
+	mtx_unlock(&curr_mdesc_lock);
+	mdp->freep(mdp, sizeof(md_impl_t));
+}
+
+
diff --git a/sys/sun4v/mdesc/mdesc_init_intern.c b/sys/sun4v/mdesc/mdesc_init_intern.c
new file mode 100644
index 000000000000..dd5973e067f3
--- /dev/null
+++ b/sys/sun4v/mdesc/mdesc_init_intern.c
@@ -0,0 +1,178 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ * $FreeBSD$
+ */
+
+/*
+ * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#include <sys/types.h>
+#include <sys/param.h>
+#include <machine/cddl/mdesc.h>
+#include <machine/cddl/mdesc_impl.h>
+
+md_t *
+md_init_intern(uint64_t *ptr,  void *(*allocp)(size_t),
+	    void (*freep)(void *, size_t))
+{
+	md_impl_t	*mdp;
+	int		idx;
+	int		count;
+	int		done;
+	uint64_t	gen;
+	mde_str_cookie_t root_name;
+
+	/*
+	 * Very basic checkup for alignment to avoid
+	 * bus error issues.
+	 */
+	if ((((uintptr_t)ptr) & 7) != 0)
+		return (NULL);
+
+	mdp = (md_impl_t *)allocp(sizeof (md_impl_t));
+
+	if (mdp == NULL)
+		return (NULL);
+
+	mdp->allocp = allocp;
+	mdp->freep = freep;
+
+	mdp->caddr = (char *)ptr;
+
+	/*
+	 * setup internal structures
+	 */
+
+	mdp->headerp = (md_header_t *)mdp->caddr;
+
+	if (mdtoh32(mdp->headerp->transport_version) != MD_TRANSPORT_VERSION) {
+		goto cleanup_nohash;
+	}
+
+	mdp->node_blk_size = mdtoh32(mdp->headerp->node_blk_sz);
+	mdp->name_blk_size = mdtoh32(mdp->headerp->name_blk_sz);
+	mdp->data_blk_size = mdtoh32(mdp->headerp->data_blk_sz);
+
+	mdp->size = MD_HEADER_SIZE + mdp->node_blk_size +
+	    mdp->name_blk_size + mdp->data_blk_size;
+
+	mdp->mdep = (md_element_t *)(mdp->caddr + MD_HEADER_SIZE);
+	mdp->namep = (char *)(mdp->caddr + MD_HEADER_SIZE + mdp->node_blk_size);
+	mdp->datap = (uint8_t *)(mdp->caddr + MD_HEADER_SIZE +
+	    mdp->name_blk_size + mdp->node_blk_size);
+
+	mdp->root_node = MDE_INVAL_ELEM_COOKIE;
+
+
+	/*
+	 * Should do a lot more sanity checking here.
+	 */
+
+	/*
+	 * Should initialize a name hash here if we intend to use one
+	 */
+
+	/*
+	 * Setup to find the root node
+	 */
+	root_name = md_find_name((md_t *)mdp, "root");
+	if (root_name == MDE_INVAL_STR_COOKIE) {
+		goto cleanup;
+	}
+
+	/*
+	 * One more property we need is the count of nodes in the
+	 * DAG, not just the number of elements.
+	 *
+	 * We try and pickup the root node along the way here.
+	 */
+
+	for (done = 0, idx = 0, count = 0; !done; ) {
+		md_element_t *np;
+
+		np = &(mdp->mdep[idx]);
+
+		switch (MDE_TAG(np)) {
+		case MDET_LIST_END:
+			done = 1;
+			break;
+
+		case MDET_NODE:
+			if (root_name == MDE_NAME(np)) {
+				if (mdp->root_node != MDE_INVAL_ELEM_COOKIE) {
+					/* Gah .. more than one root */
+					goto cleanup;
+				}
+				mdp->root_node = (mde_cookie_t)idx;
+			}
+			idx = MDE_PROP_INDEX(np);
+			count++;
+			break;
+
+		default:
+			idx++;	/* ignore */
+		}
+	}
+
+	/*
+	 * Ensure there is a root node
+	 */
+	if (mdp->root_node == MDE_INVAL_ELEM_COOKIE) {
+		goto cleanup;
+	}
+
+	/*
+	 * Register the counts
+	 */
+
+	mdp->element_count = idx + 1;	/* include LIST_END */
+	mdp->node_count = count;
+
+	/*
+	 * Final sanity check that everything adds up
+	 */
+	if (mdp->element_count != (mdp->node_blk_size / MD_ELEMENT_SIZE))
+		goto cleanup;
+
+	mdp->md_magic = LIBMD_MAGIC;
+
+	/*
+	 * Setup MD generation
+	 */
+	if (md_get_prop_val((md_t *)mdp, mdp->root_node,
+	    "md-generation#", &gen) != 0)
+		mdp->gen = MDESC_INVAL_GEN;
+	else
+		mdp->gen = gen;
+
+	return ((md_t *)mdp);
+
+cleanup:
+	/*
+	 * Clean up here - including a name hash if
+	 * we build one.
+	 */
+
+cleanup_nohash:
+	mdp->freep(mdp, sizeof (md_impl_t));
+	return (NULL);
+}
diff --git a/sys/sun4v/mdesc/mdesc_nodecount.c b/sys/sun4v/mdesc/mdesc_nodecount.c
new file mode 100644
index 000000000000..1c0376d5138e
--- /dev/null
+++ b/sys/sun4v/mdesc/mdesc_nodecount.c
@@ -0,0 +1,44 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License").  You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ * $FreeBSD$
+ */
+/*
+ * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+
+#include <sys/types.h>
+#include <machine/cddl/mdesc.h>
+#include <machine/cddl/mdesc_impl.h>
+
+int
+md_node_count(md_t *ptr)
+{
+	md_impl_t *mdp;
+
+	mdp = (md_impl_t *)ptr;
+
+	if (mdp->md_magic != LIBMD_MAGIC)
+		return (-1);
+
+	return (mdp->node_count);
+}
diff --git a/sys/sun4v/mdesc/mdesc_rootnode.c b/sys/sun4v/mdesc/mdesc_rootnode.c
new file mode 100644
index 000000000000..2e4437aa131d
--- /dev/null
+++ b/sys/sun4v/mdesc/mdesc_rootnode.c
@@ -0,0 +1,44 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ * $FreeBSD$
+ */
+
+/*
+ * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+
+#include <sys/types.h>
+#include <machine/cddl/mdesc.h>
+#include <machine/cddl/mdesc_impl.h>
+
+mde_cookie_t
+md_root_node(md_t *ptr)
+{
+	md_impl_t *mdp;
+
+	mdp = (md_impl_t *)ptr;
+
+	if (mdp->md_magic != LIBMD_MAGIC)
+		return (MDE_INVAL_ELEM_COOKIE);
+
+	return (mdp->root_node);
+}
diff --git a/sys/sun4v/mdesc/mdesc_scandag.c b/sys/sun4v/mdesc/mdesc_scandag.c
new file mode 100644
index 000000000000..4d7716a14cae
--- /dev/null
+++ b/sys/sun4v/mdesc/mdesc_scandag.c
@@ -0,0 +1,191 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ * $FreeBSD$
+ */
+
+/*
+ * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#include <sys/types.h>
+#include <sys/param.h>
+#ifdef _KERNEL
+#include <sys/systm.h>
+#else
+#include <string.h>
+#include <strings.h>
+#endif
+
+#include <machine/cddl/mdesc.h>
+#include <machine/cddl/mdesc_impl.h>
+
+static int
+mdl_scan_dag(md_impl_t *mdp,
+	int nodeidx,
+	mde_str_cookie_t node_cookie,
+	mde_str_cookie_t arc_cookie,
+	uint8_t *dseenp,
+	int *idxp,
+	mde_cookie_t *stashp,
+	int level);
+
+
+int
+md_scan_dag(md_t *ptr,
+	mde_cookie_t startnode,
+	mde_str_cookie_t node_name_cookie,
+	mde_str_cookie_t arc_name_cookie,
+	mde_cookie_t *stashp)
+{
+	int	res;
+	int	idx;
+	uint8_t *seenp;
+	md_impl_t *mdp;
+	int	start;
+
+	mdp = (md_impl_t *)ptr;
+
+	/*
+	 * Possible the caller was lazy and didn't check the
+	 * validitiy of either the node name or the arc name
+	 * on calling ... in which case fail to find any
+	 * nodes.
+	 * This is distinct, from a fail (-1) since we return
+	 * that nothing was found.
+	 */
+
+	if (node_name_cookie == MDE_INVAL_STR_COOKIE ||
+		arc_name_cookie == MDE_INVAL_STR_COOKIE) return 0;
+
+	/*
+	 * if we want to start at the top, start at index 0
+	 */
+
+	start = (int)startnode;
+	if (start == MDE_INVAL_ELEM_COOKIE) start = 0;
+
+	/*
+	 * Scan from the start point until the first node.
+	 */
+	while (MDE_TAG(&mdp->mdep[start]) == MDET_NULL) start++;
+
+	/*
+	 * This was a bogus start point if no node found
+	 */
+	if (MDE_TAG(&mdp->mdep[start]) != MDET_NODE) {
+		return (-1);	/* illegal start node specified */
+	}
+
+	/*
+	 * Allocate a recursion mask on the local stack fail
+	 * if we can't allocate the recursion detection.
+	 */
+	seenp = (uint8_t *)mdp->allocp(mdp->element_count);
+	if (seenp == NULL)
+		return (-1);
+	(void) memset(seenp, 0, mdp->element_count);
+
+	/*
+	 * Now build the list of requested nodes.
+	 */
+	idx = 0;
+	res = mdl_scan_dag(mdp, start,
+		node_name_cookie, arc_name_cookie,
+		seenp, &idx, stashp, 0);
+
+	mdp->freep(seenp, mdp->element_count);
+
+	return (res >= 0 ? idx : res);
+}
+
+
+
+
+
+static int
+mdl_scan_dag(md_impl_t *mdp,
+	int nodeidx,
+	mde_str_cookie_t node_name_cookie,
+	mde_str_cookie_t arc_name_cookie,
+	uint8_t *seenp,
+	int *idxp,
+	mde_cookie_t *stashp,
+	int level)
+{
+	md_element_t *mdep;
+
+	mdep = &(mdp->mdep[nodeidx]);
+
+	/* see if cookie is infact a node */
+	if (MDE_TAG(mdep) != MDET_NODE)
+		return (-1);
+
+	/* have we been here before ? */
+	if (seenp[nodeidx])
+		return (0);
+	seenp[nodeidx] = 1;
+
+	/* is this node of the type we seek ? */
+
+#ifdef	DEBUG_LIBMDESC
+	{
+	int x;
+	for (x = 0; x < level; x++)
+		printf("-");
+	printf("%d (%s)\n", nodeidx, (char *)(mdp->datap + MDE_NAME(mdep)));
+	}
+#endif
+
+	if (MDE_NAME(mdep) == node_name_cookie) {
+		/* record the node in the list and keep searching */
+		if (stashp != NULL) {
+			stashp[*idxp] = (mde_cookie_t)nodeidx;
+		}
+		(*idxp)++;
+#ifdef	DEBUG_LIBMDESC
+		printf("\t* %d\n", *idxp);
+#endif
+	}
+
+	/*
+	 * Simply walk the elements in the node.
+	 * if we find a matching arc, then recursively call
+	 * the subordinate looking for a match
+	 */
+
+	for (mdep++; MDE_TAG(mdep) != MDET_NODE_END; mdep++) {
+		if (MDE_TAG(mdep) == MDET_PROP_ARC &&
+			MDE_NAME(mdep) == arc_name_cookie) {
+			int res;
+
+			res = mdl_scan_dag(mdp,
+			    (int)mdep->d.prop_idx,
+			    node_name_cookie,
+			    arc_name_cookie,
+			    seenp, idxp, stashp, level+1);
+
+			if (res == -1)
+				return (res);
+		}
+	}
+
+	return (0);
+}
diff --git a/sys/sun4v/mdesc/mdesc_subr.c b/sys/sun4v/mdesc/mdesc_subr.c
new file mode 100644
index 000000000000..4840708abddb
--- /dev/null
+++ b/sys/sun4v/mdesc/mdesc_subr.c
@@ -0,0 +1,76 @@
+/*-
+ * Copyright (c) 2006 Kip Macy
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/types.h>
+#include <sys/systm.h>
+#include <sys/malloc.h>
+
+
+#include <machine/cddl/mdesc.h>
+#include <machine/cddl/mdesc_impl.h>
+
+
+int
+md_get_prop_alloc(md_t *ptr, mde_cookie_t node, char *namep, int tag_type,
+		 uint8_t **datap)
+{
+        mde_str_cookie_t prop_name;
+        md_impl_t       *mdp;
+        mde_cookie_t     elem;
+	int              len;
+
+        mdp = (md_impl_t *)ptr;
+
+        if (node == MDE_INVAL_ELEM_COOKIE) {
+                return (-1);
+        }
+
+        prop_name = md_find_name(ptr, namep);
+        if (prop_name == MDE_INVAL_STR_COOKIE) {
+                return (-1);
+        }
+
+        elem = md_find_node_prop(mdp, node, prop_name, tag_type);
+
+        if (elem != MDE_INVAL_ELEM_COOKIE) {
+                md_element_t *mdep;
+                mdep = &(mdp->mdep[(int)elem]);
+
+                len = (int)MDE_PROP_DATA_LEN(mdep);
+		KASSERT(len > 0, ("invalid length"));
+		*datap = malloc(len, M_MDPROP, M_WAITOK);
+		bcopy(mdp->datap + (int)MDE_PROP_DATA_OFFSET(mdep), *datap, len);
+                return (0);
+        }
+
+        return (-1);    /* no such property name */
+}
+
+
diff --git a/sys/sun4v/mdesc/mdesc_vdevfindval.c b/sys/sun4v/mdesc/mdesc_vdevfindval.c
new file mode 100644
index 000000000000..8216996a1078
--- /dev/null
+++ b/sys/sun4v/mdesc/mdesc_vdevfindval.c
@@ -0,0 +1,98 @@
+/*-
+ * Copyright (c) 2006 Kip Macy
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/endian.h>
+#include <sys/systm.h>
+#include <sys/sockio.h>
+#include <sys/mbuf.h>
+#include <sys/malloc.h>
+#include <sys/kernel.h>
+#include <sys/module.h>
+#include <sys/socket.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
+#include <sys/taskqueue.h>
+
+#include <machine/bus.h>
+#include <machine/resource.h>
+#include <sys/bus.h>
+#include <sys/rman.h>
+
+#include <sys/types.h>
+#include <sys/param.h>
+#include <machine/mdesc_bus.h>
+#include <machine/cddl/mdesc.h>
+
+
+int
+md_vdev_find_val(device_t dev, char *namep, uint64_t *valp)
+{
+	uint64_t              cfg_handle;
+	mde_cookie_t          rootnode, node, *listp = NULL;
+	int                   i, listsz, num_nodes, num_devices, error;
+	md_t                 *mdp;
+
+	cfg_handle = mdesc_bus_get_handle(dev);
+
+	if ((mdp = md_get()) == NULL) 
+		return (ENXIO);
+
+	num_nodes = md_node_count(mdp);
+	listsz = num_nodes * sizeof(mde_cookie_t);
+	listp = (mde_cookie_t *)malloc(listsz, M_DEVBUF, M_WAITOK);
+	rootnode = md_root_node(mdp);
+	node = error = 0;
+
+	num_devices = md_scan_dag(mdp, rootnode, 
+				  md_find_name(mdp, "virtual-device"),
+				  md_find_name(mdp, "fwd"), listp);
+	
+	if (num_devices == 0) {
+		error = ENOENT;
+		goto done;
+	}
+		
+	for (i = 0; i < num_devices; i++) {
+		uint64_t thandle;
+
+		node = listp[i];
+		md_get_prop_val(mdp, node, "cfg-handle", &thandle);
+		if (thandle == cfg_handle) 
+			break;
+	}
+
+	md_get_prop_val(mdp, node, namep, valp);
+done:
+	md_put(mdp);
+
+	return (error);
+}
+
diff --git a/sys/sun4v/sun4v/autoconf.c b/sys/sun4v/sun4v/autoconf.c
new file mode 100644
index 000000000000..b30fbdd2cece
--- /dev/null
+++ b/sys/sun4v/sun4v/autoconf.c
@@ -0,0 +1,83 @@
+/*-
+ * Copyright (c) 2001 Jake Burkholder.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include "opt_isa.h"
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/bus.h>
+#include <sys/cons.h>
+#include <sys/kernel.h>
+
+#ifdef DEV_ISA
+#include <isa/isavar.h>
+extern device_t isa_bus_device;
+#endif
+
+static device_t nexusdev;
+
+static void	configure_first(void *);
+static void	configure(void *);
+static void	configure_final(void *);
+
+SYSINIT(configure1, SI_SUB_CONFIGURE, SI_ORDER_FIRST, configure_first, NULL);
+/* SI_ORDER_SECOND is hookable */
+SYSINIT(configure2, SI_SUB_CONFIGURE, SI_ORDER_THIRD, configure, NULL);
+/* SI_ORDER_MIDDLE is hookable */
+SYSINIT(configure3, SI_SUB_CONFIGURE, SI_ORDER_ANY, configure_final, NULL);
+
+/*
+ * Determine i/o configuration for a machine.
+ */
+static void
+configure_first(void *dummy)
+{
+
+	nexusdev = device_add_child(root_bus, "nexus", 0);
+}
+
+static void
+configure(void *dummy)
+{
+	
+	intr_restore_all(0x16);
+
+	root_bus_configure();
+#ifdef DEV_ISA
+	if (isa_bus_device != NULL)
+		isa_probe_children(isa_bus_device);
+#endif
+}
+
+static void
+configure_final(void *dummy)
+{
+
+	cninit_finish();
+	cold = 0;
+}
diff --git a/sys/sun4v/sun4v/bus_machdep.c b/sys/sun4v/sun4v/bus_machdep.c
new file mode 100644
index 000000000000..c05086db5da2
--- /dev/null
+++ b/sys/sun4v/sun4v/bus_machdep.c
@@ -0,0 +1,844 @@
+/*-
+ * Copyright (c) 1996, 1997, 1998 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The NetBSD Foundation
+ * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
+ * NASA Ames Research Center.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ *    must display the following acknowledgement:
+ *	This product includes software developed by the NetBSD
+ *	Foundation, Inc. and its contributors.
+ * 4. Neither the name of The NetBSD Foundation nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+/*-
+ * Copyright (c) 1992, 1993
+ *	The Regents of the University of California.  All rights reserved.
+ *
+ * This software was developed by the Computer Systems Engineering group
+ * at Lawrence Berkeley Laboratory under DARPA contract BG 91-66 and
+ * contributed to Berkeley.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 4. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+/*-
+ * Copyright (c) 1997, 1998 Justin T. Gibbs.
+ * All rights reserved.
+ * Copyright 2001 by Thomas Moestl <tmm@FreeBSD.org>.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions, and the following disclaimer,
+ *    without modification, immediately at the beginning of the file.
+ * 2. The name of the author may not be used to endorse or promote products
+ *    derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	from: @(#)machdep.c	8.6 (Berkeley) 1/14/94
+ *	from: NetBSD: machdep.c,v 1.111 2001/09/15 07:13:40 eeh Exp
+ *	and
+ * 	from: FreeBSD: src/sys/i386/i386/busdma_machdep.c,v 1.24 2001/08/15
+ *
+ * $FreeBSD$
+ */
+
+#include <sys/param.h>
+#include <sys/bus.h>
+#include <sys/lock.h>
+#include <sys/malloc.h>
+#include <sys/mbuf.h>
+#include <sys/mutex.h>
+#include <sys/proc.h>
+#include <sys/smp.h>
+#include <sys/systm.h>
+#include <sys/uio.h>
+
+#include <vm/vm.h>
+#include <vm/vm_extern.h>
+#include <vm/vm_kern.h>
+#include <vm/vm_page.h>
+#include <vm/vm_param.h>
+#include <vm/vm_map.h>
+
+#include <machine/asi.h>
+#include <machine/atomic.h>
+#include <machine/bus.h>
+#include <machine/bus_private.h>
+#include <machine/cache.h>
+#include <machine/smp.h>
+#include <machine/tlb.h>
+
+static void nexus_bus_barrier(bus_space_tag_t, bus_space_handle_t,
+    bus_size_t, bus_size_t, int);
+
+/* ASI's for bus access. */
+int bus_type_asi[] = {
+	ASI_REAL_IO,		/* UPA */
+	ASI_REAL_IO,		/* SBUS */
+	ASI_REAL_IO_L,		/* PCI configuration space */
+	ASI_REAL_IO_L,		/* PCI memory space */
+	ASI_REAL_IO_L,		/* PCI I/O space */
+	0
+};
+
+int bus_stream_asi[] = {
+	ASI_REAL_IO,		/* UPA */
+	ASI_REAL_IO,		/* SBUS */
+	ASI_REAL_IO,		/* PCI configuration space */
+	ASI_REAL_IO,		/* PCI memory space */
+	ASI_REAL_IO,		/* PCI I/O space */
+	0
+};
+
+/*
+ * Convenience function for manipulating driver locks from busdma (during
+ * busdma_swi, for example).  Drivers that don't provide their own locks
+ * should specify &Giant to dmat->lockfuncarg.  Drivers that use their own
+ * non-mutex locking scheme don't have to use this at all.
+ */
+void
+busdma_lock_mutex(void *arg, bus_dma_lock_op_t op)
+{
+	struct mtx *dmtx;
+
+	dmtx = (struct mtx *)arg;
+	switch (op) {
+	case BUS_DMA_LOCK:
+		mtx_lock(dmtx);
+		break;
+	case BUS_DMA_UNLOCK:
+		mtx_unlock(dmtx);
+		break;
+	default:
+		panic("Unknown operation 0x%x for busdma_lock_mutex!", op);
+	}
+}
+
+/*
+ * dflt_lock should never get called.  It gets put into the dma tag when
+ * lockfunc == NULL, which is only valid if the maps that are associated
+ * with the tag are meant to never be defered.
+ * XXX Should have a way to identify which driver is responsible here.
+ */
+static void
+dflt_lock(void *arg, bus_dma_lock_op_t op)
+{
+#ifdef INVARIANTS
+	panic("driver error: busdma dflt_lock called");
+#else
+	printf("DRIVER_ERROR: busdma dflt_lock called\n");
+#endif
+}
+
+/*
+ * Since there is no way for a device to obtain a dma tag from its parent
+ * we use this kluge to handle different the different supported bus systems.
+ * The sparc64_root_dma_tag is used as parent for tags that have none, so that
+ * the correct methods will be used.
+ */
+bus_dma_tag_t sparc64_root_dma_tag;
+
+/*
+ * Allocate a device specific dma_tag.
+ */
+int
+bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment,
+    bus_size_t boundary, bus_addr_t lowaddr, bus_addr_t highaddr,
+    bus_dma_filter_t *filter, void *filterarg, bus_size_t maxsize,
+    int nsegments, bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc,
+    void *lockfuncarg, bus_dma_tag_t *dmat)
+{
+	bus_dma_tag_t impptag;
+	bus_dma_tag_t newtag;
+
+	/* Return a NULL tag on failure */
+	*dmat = NULL;
+
+	newtag = (bus_dma_tag_t)malloc(sizeof(*newtag), M_DEVBUF, M_NOWAIT);
+	if (newtag == NULL)
+		return (ENOMEM);
+
+	impptag = parent != NULL ? parent : sparc64_root_dma_tag;
+	/*
+	 * The method table pointer and the cookie need to be taken over from
+	 * the parent or the root tag.
+	 */
+	newtag->dt_cookie = impptag->dt_cookie;
+	newtag->dt_mt = impptag->dt_mt;
+
+	newtag->dt_parent = parent;
+	newtag->dt_alignment = alignment;
+	newtag->dt_boundary = boundary;
+	newtag->dt_lowaddr = trunc_page((vm_offset_t)lowaddr) + (PAGE_SIZE - 1);
+	newtag->dt_highaddr = trunc_page((vm_offset_t)highaddr) +
+	    (PAGE_SIZE - 1);
+	newtag->dt_filter = filter;
+	newtag->dt_filterarg = filterarg;
+	newtag->dt_maxsize = maxsize;
+	newtag->dt_nsegments = nsegments;
+	newtag->dt_maxsegsz = maxsegsz;
+	newtag->dt_flags = flags;
+	newtag->dt_ref_count = 1; /* Count ourselves */
+	newtag->dt_map_count = 0;
+
+	if (lockfunc != NULL) {
+		newtag->dt_lockfunc = lockfunc;
+		newtag->dt_lockfuncarg = lockfuncarg;
+	} else {
+		newtag->dt_lockfunc = dflt_lock;
+		newtag->dt_lockfuncarg = NULL;
+	}
+
+	newtag->dt_segments = NULL;
+
+	/* Take into account any restrictions imposed by our parent tag */
+	if (parent != NULL) {
+		newtag->dt_lowaddr = ulmin(parent->dt_lowaddr,
+		    newtag->dt_lowaddr);
+		newtag->dt_highaddr = ulmax(parent->dt_highaddr,
+		    newtag->dt_highaddr);
+                if (newtag->dt_boundary == 0)
+                        newtag->dt_boundary = parent->dt_boundary;
+                else if (parent->dt_boundary != 0)            
+			newtag->dt_boundary = ulmin(parent->dt_boundary,
+			    newtag->dt_boundary);
+		atomic_add_int(&parent->dt_ref_count, 1);
+	}
+
+	if (newtag->dt_boundary > 0)
+		newtag->dt_maxsegsz = ulmin(newtag->dt_maxsegsz,
+		    newtag->dt_boundary);
+
+	*dmat = newtag;
+	return (0);
+}
+
+int
+bus_dma_tag_destroy(bus_dma_tag_t dmat)
+{
+	bus_dma_tag_t parent;
+
+	if (dmat != NULL) {
+		if (dmat->dt_map_count != 0)
+			return (EBUSY);
+		while (dmat != NULL) {
+			parent = dmat->dt_parent;
+			atomic_subtract_int(&dmat->dt_ref_count, 1);
+			if (dmat->dt_ref_count == 0) {
+				if (dmat->dt_segments != NULL)
+					free(dmat->dt_segments, M_DEVBUF);
+				free(dmat, M_DEVBUF);
+				/*
+				 * Last reference count, so
+				 * release our reference
+				 * count on our parent.
+				 */
+				dmat = parent;
+			} else
+				dmat = NULL;
+		}
+	}
+	return (0);
+}
+
+/* Allocate/free a tag, and do the necessary management work. */
+int
+sparc64_dma_alloc_map(bus_dma_tag_t dmat, bus_dmamap_t *mapp)
+{
+
+	if (dmat->dt_segments == NULL) {
+		dmat->dt_segments = (bus_dma_segment_t *)malloc(
+		    sizeof(bus_dma_segment_t) * dmat->dt_nsegments, M_DEVBUF,
+		    M_NOWAIT);
+		if (dmat->dt_segments == NULL)
+			return (ENOMEM);
+	}
+	*mapp = malloc(sizeof(**mapp), M_DEVBUF, M_NOWAIT | M_ZERO);
+	if (*mapp == NULL)
+		return (ENOMEM);
+
+	SLIST_INIT(&(*mapp)->dm_reslist);
+	dmat->dt_map_count++;
+	return (0);
+}
+
+void
+sparc64_dma_free_map(bus_dma_tag_t dmat, bus_dmamap_t map)
+{
+
+	free(map, M_DEVBUF);
+	dmat->dt_map_count--;
+}
+
+static int
+nexus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp)
+{
+
+	return (sparc64_dma_alloc_map(dmat, mapp));
+}
+
+static int
+nexus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map)
+{
+
+	sparc64_dma_free_map(dmat, map);
+	return (0);
+}
+
+/*
+ * Utility function to load a linear buffer.  lastaddrp holds state
+ * between invocations (for multiple-buffer loads).  segp contains
+ * the starting segment on entrace, and the ending segment on exit.
+ * first indicates if this is the first invocation of this function.
+ */
+static int
+_nexus_dmamap_load_buffer(bus_dma_tag_t dmat, void *buf, bus_size_t buflen,
+    struct thread *td, int flags, bus_addr_t *lastaddrp,
+    bus_dma_segment_t *segs, int *segp, int first)
+{
+	bus_size_t sgsize;
+	bus_addr_t curaddr, lastaddr, baddr, bmask;
+	vm_offset_t vaddr = (vm_offset_t)buf;
+	int seg;
+	pmap_t pmap;
+
+	if (td != NULL)
+		pmap = vmspace_pmap(td->td_proc->p_vmspace);
+	else
+		pmap = NULL;
+
+	lastaddr = *lastaddrp;
+	bmask  = ~(dmat->dt_boundary - 1);
+
+	for (seg = *segp; buflen > 0 ; ) {
+		/*
+		 * Get the physical address for this segment.
+		 */
+		if (pmap)
+			curaddr = pmap_extract(pmap, vaddr);
+		else
+			curaddr = pmap_kextract(vaddr);
+
+		/*
+		 * Compute the segment size, and adjust counts.
+		 */
+		sgsize = PAGE_SIZE - ((u_long)curaddr & PAGE_MASK);
+		if (buflen < sgsize)
+			sgsize = buflen;
+
+		/*
+		 * Make sure we don't cross any boundaries.
+		 */
+		if (dmat->dt_boundary > 0) {
+			baddr = (curaddr + dmat->dt_boundary) & bmask;
+			if (sgsize > (baddr - curaddr))
+				sgsize = (baddr - curaddr);
+		}
+
+		/*
+		 * Insert chunk into a segment, coalescing with
+		 * previous segment if possible.
+		 */
+		if (first) {
+			segs[seg].ds_addr = curaddr;
+			segs[seg].ds_len = sgsize;
+			first = 0;
+		} else {
+			if (curaddr == lastaddr &&
+			    (segs[seg].ds_len + sgsize) <= dmat->dt_maxsegsz &&
+			    (dmat->dt_boundary == 0 ||
+			     (segs[seg].ds_addr & bmask) == (curaddr & bmask)))
+				segs[seg].ds_len += sgsize;
+			else {
+				if (++seg >= dmat->dt_nsegments)
+					break;
+				segs[seg].ds_addr = curaddr;
+				segs[seg].ds_len = sgsize;
+			}
+		}
+
+		lastaddr = curaddr + sgsize;
+		vaddr += sgsize;
+		buflen -= sgsize;
+	}
+
+	*segp = seg;
+	*lastaddrp = lastaddr;
+
+	/*
+	 * Did we fit?
+	 */
+	return (buflen != 0 ? EFBIG : 0); /* XXX better return value here? */
+}
+
+/*
+ * Common function for loading a DMA map with a linear buffer.  May
+ * be called by bus-specific DMA map load functions.
+ *
+ * Most SPARCs have IOMMUs in the bus controllers.  In those cases
+ * they only need one segment and will use virtual addresses for DVMA.
+ * Those bus controllers should intercept these vectors and should
+ * *NEVER* call nexus_dmamap_load() which is used only by devices that
+ * bypass DVMA.
+ */
+static int
+nexus_dmamap_load(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf,
+    bus_size_t buflen, bus_dmamap_callback_t *callback, void *callback_arg,
+    int flags)
+{
+	bus_addr_t lastaddr;
+	int error, nsegs;
+
+	error = _nexus_dmamap_load_buffer(dmat, buf, buflen, NULL, flags,
+	    &lastaddr, dmat->dt_segments, &nsegs, 1);
+
+	if (error == 0) {
+		(*callback)(callback_arg, dmat->dt_segments, nsegs + 1, 0);
+		map->dm_flags |= DMF_LOADED;
+	} else
+		(*callback)(callback_arg, NULL, 0, error);
+
+	return (0);
+}
+
+/*
+ * Like nexus_dmamap_load(), but for mbufs.
+ */
+static int
+nexus_dmamap_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map, struct mbuf *m0,
+    bus_dmamap_callback2_t *callback, void *callback_arg, int flags)
+{
+	int nsegs, error;
+
+	M_ASSERTPKTHDR(m0);
+
+	nsegs = 0;
+	error = 0;
+	if (m0->m_pkthdr.len <= dmat->dt_maxsize) {
+		int first = 1;
+		bus_addr_t lastaddr = 0;
+		struct mbuf *m;
+
+		for (m = m0; m != NULL && error == 0; m = m->m_next) {
+			if (m->m_len > 0) {
+				error = _nexus_dmamap_load_buffer(dmat,
+				    m->m_data, m->m_len,NULL, flags, &lastaddr,
+				    dmat->dt_segments, &nsegs, first);
+				first = 0;
+			}
+		}
+	} else {
+		error = EINVAL;
+	}
+
+	if (error) {
+		/* force "no valid mappings" in callback */
+		(*callback)(callback_arg, dmat->dt_segments, 0, 0, error);
+	} else {
+		map->dm_flags |= DMF_LOADED;
+		(*callback)(callback_arg, dmat->dt_segments, nsegs + 1,
+		    m0->m_pkthdr.len, error);
+	}
+	return (error);
+}
+
+static int
+nexus_dmamap_load_mbuf_sg(bus_dma_tag_t dmat, bus_dmamap_t map, struct mbuf *m0,
+    bus_dma_segment_t *segs, int *nsegs, int flags)
+{
+	int error;
+
+	M_ASSERTPKTHDR(m0);
+
+	*nsegs = 0;
+	error = 0;
+	if (m0->m_pkthdr.len <= dmat->dt_maxsize) {
+		int first = 1;
+		bus_addr_t lastaddr = 0;
+		struct mbuf *m;
+
+		for (m = m0; m != NULL && error == 0; m = m->m_next) {
+			if (m->m_len > 0) {
+				error = _nexus_dmamap_load_buffer(dmat,
+				    m->m_data, m->m_len,NULL, flags, &lastaddr,
+				    segs, nsegs, first);
+				first = 0;
+			}
+		}
+	} else {
+		error = EINVAL;
+	}
+
+	++*nsegs;
+	return (error);
+}
+
+/*
+ * Like nexus_dmamap_load(), but for uios.
+ */
+static int
+nexus_dmamap_load_uio(bus_dma_tag_t dmat, bus_dmamap_t map, struct uio *uio,
+    bus_dmamap_callback2_t *callback, void *callback_arg, int flags)
+{
+	bus_addr_t lastaddr;
+	int nsegs, error, first, i;
+	bus_size_t resid;
+	struct iovec *iov;
+	struct thread *td = NULL;
+
+	resid = uio->uio_resid;
+	iov = uio->uio_iov;
+
+	if (uio->uio_segflg == UIO_USERSPACE) {
+		td = uio->uio_td;
+		KASSERT(td != NULL,
+			("nexus_dmamap_load_uio: USERSPACE but no proc"));
+	}
+
+	nsegs = 0;
+	error = 0;
+	first = 1;
+	for (i = 0; i < uio->uio_iovcnt && resid != 0 && !error; i++) {
+		/*
+		 * Now at the first iovec to load.  Load each iovec
+		 * until we have exhausted the residual count.
+		 */
+		bus_size_t minlen =
+			resid < iov[i].iov_len ? resid : iov[i].iov_len;
+		caddr_t addr = (caddr_t) iov[i].iov_base;
+
+		if (minlen > 0) {
+			error = _nexus_dmamap_load_buffer(dmat, addr, minlen,
+			    td, flags, &lastaddr, dmat->dt_segments, &nsegs,
+			    first);
+			first = 0;
+
+			resid -= minlen;
+		}
+	}
+
+	if (error) {
+		/* force "no valid mappings" in callback */
+		(*callback)(callback_arg, dmat->dt_segments, 0, 0, error);
+	} else {
+		map->dm_flags |= DMF_LOADED;
+		(*callback)(callback_arg, dmat->dt_segments, nsegs + 1,
+		    uio->uio_resid, error);
+	}
+	return (error);
+}
+
+/*
+ * Common function for unloading a DMA map.  May be called by
+ * bus-specific DMA map unload functions.
+ */
+static void
+nexus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map)
+{
+
+	map->dm_flags &= ~DMF_LOADED;
+}
+
+/*
+ * Common function for DMA map synchronization.  May be called
+ * by bus-specific DMA map synchronization functions.
+ */
+static void
+nexus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op)
+{
+
+	/*
+	 * We sync out our caches, but the bus must do the same.
+	 *
+	 * Actually a #Sync is expensive.  We should optimize.
+	 */
+	if ((op & BUS_DMASYNC_PREREAD) || (op & BUS_DMASYNC_PREWRITE)) {
+		/* 
+		 * Don't really need to do anything, but flush any pending
+		 * writes anyway. 
+		 */
+		membar(Sync);
+	}
+#if 0
+	/* Should not be needed. */
+	if (op & BUS_DMASYNC_POSTREAD) {
+		ecache_flush((vm_offset_t)map->buf,
+		    (vm_offset_t)map->buf + map->buflen - 1);
+	}
+#endif
+	if (op & BUS_DMASYNC_POSTWRITE) {
+		/* Nothing to do.  Handled by the bus controller. */
+	}
+}
+
+/*
+ * Common function for DMA-safe memory allocation.  May be called
+ * by bus-specific DMA memory allocation functions.
+ */
+static int
+nexus_dmamem_alloc(bus_dma_tag_t dmat, void **vaddr, int flags,
+    bus_dmamap_t *mapp)
+{
+	int mflags;
+
+	if (flags & BUS_DMA_NOWAIT)
+		mflags = M_NOWAIT;
+	else
+		mflags = M_WAITOK;
+	if (flags & BUS_DMA_ZERO)
+		mflags |= M_ZERO;
+
+	if ((dmat->dt_maxsize <= PAGE_SIZE)) {
+		*vaddr = malloc(dmat->dt_maxsize, M_DEVBUF, mflags);
+	} else {
+		/*
+		 * XXX: Use contigmalloc until it is merged into this facility
+		 * and handles multi-seg allocations.  Nobody is doing multi-seg
+		 * allocations yet though.
+		 */
+		*vaddr = contigmalloc(dmat->dt_maxsize, M_DEVBUF, mflags,
+		    0ul, dmat->dt_lowaddr,
+		    dmat->dt_alignment ? dmat->dt_alignment : 1UL,
+		    dmat->dt_boundary);
+	}
+	if (*vaddr == NULL)
+		return (ENOMEM);
+	return (0);
+}
+
+/*
+ * Common function for freeing DMA-safe memory.  May be called by
+ * bus-specific DMA memory free functions.
+ */
+static void
+nexus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map)
+{
+
+	if ((dmat->dt_maxsize <= PAGE_SIZE))
+		free(vaddr, M_DEVBUF);
+	else {
+		contigfree(vaddr, dmat->dt_maxsize, M_DEVBUF);
+	}
+}
+
+struct bus_dma_methods nexus_dma_methods = {
+	nexus_dmamap_create,
+	nexus_dmamap_destroy,
+	nexus_dmamap_load,
+	nexus_dmamap_load_mbuf,
+	nexus_dmamap_load_mbuf_sg,
+	nexus_dmamap_load_uio,
+	nexus_dmamap_unload,
+	nexus_dmamap_sync,
+	nexus_dmamem_alloc,
+	nexus_dmamem_free,
+};
+
+struct bus_dma_tag nexus_dmatag = {
+	NULL,
+	NULL,
+	8,
+	0,
+	0,
+	0x3ffffffff,
+	NULL,		/* XXX */
+	NULL,
+	0x3ffffffff,	/* XXX */
+	0xff,		/* XXX */
+	0xffffffff,	/* XXX */
+	0,
+	0,
+	0,
+	NULL,
+	NULL,
+	NULL,
+	&nexus_dma_methods,
+};
+
+/*
+ * Helpers to map/unmap bus memory
+ */
+int
+sparc64_bus_mem_map(bus_space_tag_t tag, bus_space_handle_t handle,
+    bus_size_t size, int flags, vm_offset_t vaddr, void **hp)
+{
+#if 0
+	vm_offset_t addr;
+	vm_offset_t sva;
+	vm_offset_t va;
+	vm_paddr_t pa;
+	vm_size_t vsz;
+	u_long pm_flags;
+
+	addr = (vm_offset_t)handle;
+	size = round_page(size);
+	if (size == 0) {
+		printf("sparc64_bus_map: zero size\n");
+		return (EINVAL);
+	}
+	switch (tag->bst_type) {
+	case PCI_CONFIG_BUS_SPACE:
+	case PCI_IO_BUS_SPACE:
+	case PCI_MEMORY_BUS_SPACE:
+		pm_flags = TD_IE;
+		break;
+	default:
+		pm_flags = 0;
+		break;
+	}
+
+	if (!(flags & BUS_SPACE_MAP_CACHEABLE))
+		pm_flags |= TD_E;
+
+	if (vaddr != 0L)
+		sva = trunc_page(vaddr);
+	else {
+		if ((sva = kmem_alloc_nofault(kernel_map, size)) == 0)
+			panic("sparc64_bus_map: cannot allocate virtual "
+			    "memory");
+	}
+
+	/* Preserve page offset. */
+	*hp = (void *)(sva | ((u_long)addr & PAGE_MASK));
+
+	pa = trunc_page(addr);
+	if ((flags & BUS_SPACE_MAP_READONLY) == 0)
+		pm_flags |= TD_W;
+
+	va = sva;
+	vsz = size;
+	do {
+		pmap_kenter_flags(va, pa, pm_flags);
+		va += PAGE_SIZE;
+		pa += PAGE_SIZE;
+	} while ((vsz -= PAGE_SIZE) > 0);
+	tlb_range_demap(kernel_pmap, sva, sva + size - 1);
+#endif
+	return (0);
+}
+
+int
+sparc64_bus_mem_unmap(void *bh, bus_size_t size)
+{
+#if 0
+	vm_offset_t sva;
+	vm_offset_t va;
+	vm_offset_t endva;
+
+	sva = trunc_page((vm_offset_t)bh);
+	endva = sva + round_page(size);
+	for (va = sva; va < endva; va += PAGE_SIZE)
+		pmap_kremove_flags(va);
+	tlb_range_demap(kernel_pmap, sva, sva + size - 1);
+	kmem_free(kernel_map, sva, size);
+#endif
+	return (0);
+}
+
+/*
+ * Fake up a bus tag, for use by console drivers in early boot when the regular
+ * means to allocate resources are not yet available.
+ * Addr is the physical address of the desired start of the handle.
+ */
+bus_space_handle_t
+sparc64_fake_bustag(int space, bus_addr_t addr, struct bus_space_tag *ptag)
+{
+
+	ptag->bst_cookie = NULL;
+	ptag->bst_parent = NULL;
+	ptag->bst_type = space;
+	ptag->bst_bus_barrier = nexus_bus_barrier;
+	return (addr);
+}
+
+/*
+ * Base bus space handlers.
+ */
+
+static void
+nexus_bus_barrier(bus_space_tag_t t, bus_space_handle_t h, bus_size_t offset,
+    bus_size_t size, int flags)
+{
+
+	/* 
+	 * We have lots of alternatives depending on whether we're
+	 * synchronizing loads with loads, loads with stores, stores
+	 * with loads, or stores with stores.  The only ones that seem
+	 * generic are #Sync and #MemIssue.  I'll use #Sync for safety.
+	 */
+	switch(flags) {
+	case BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE:
+	case BUS_SPACE_BARRIER_READ:
+	case BUS_SPACE_BARRIER_WRITE:
+		membar(Sync);
+		break;
+	default:
+		panic("sparc64_bus_barrier: unknown flags");
+	}
+	return;
+}
+
+struct bus_space_tag nexus_bustag = {
+	NULL,				/* cookie */
+	NULL,				/* parent bus tag */
+	UPA_BUS_SPACE,			/* type */
+	nexus_bus_barrier,		/* bus_space_barrier */
+};
diff --git a/sys/sun4v/sun4v/clock.c b/sys/sun4v/sun4v/clock.c
new file mode 100644
index 000000000000..37341fd3ba43
--- /dev/null
+++ b/sys/sun4v/sun4v/clock.c
@@ -0,0 +1,67 @@
+/*-
+ * Copyright (c) 2001 Jake Burkholder.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <machine/clock.h>
+
+u_long tick_increment;
+u_long tick_freq;
+u_long tick_MHz;
+
+void
+DELAY(int n)
+{
+	u_long start, end;
+
+	start = rd(tick);
+	if (n < 0)
+		return;
+	end = start + (u_long)n * tick_MHz;
+	while (rd(tick) < end)
+		;
+}
+
+void
+cpu_startprofclock(void)
+{
+}
+
+void
+cpu_stopprofclock(void)
+{
+}
+
+int
+sysbeep(int pitch, int period)
+{
+	/*
+	 * XXX: function exists to enable RAID drivers to compile at the moment.
+	 */
+	return (0);
+}
diff --git a/sys/sun4v/sun4v/counter.c b/sys/sun4v/sun4v/counter.c
new file mode 100644
index 000000000000..3e934659f9b0
--- /dev/null
+++ b/sys/sun4v/sun4v/counter.c
@@ -0,0 +1,110 @@
+/*-
+ * Copyright (c) 2002 by Thomas Moestl <tmm@FreeBSD.org>.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+ * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <sys/param.h>
+#include <sys/bus.h>
+#include <sys/malloc.h>
+#include <sys/systm.h>
+#include <sys/time.h>
+#include <sys/timetc.h>
+
+#include <machine/bus.h>
+#include <machine/bus_common.h>
+
+#define	COUNTER_MASK	((1 << 29) - 1)
+#define	COUNTER_FREQ	1000000
+#define	COUNTER_QUALITY	100
+
+/* Bits in the limit register. */
+#define	CTLR_INTEN	(1U << 31)	/* Enable timer interrupts */
+#define	CTLR_RELOAD	(1U << 30)	/* Zero counter on write to limit reg */
+#define	CTLR_PERIODIC	(1U << 29)	/* Wrap to 0 if limit is reached */
+
+/* Offsets of the registers for the two counters. */
+#define	CTR_CT0		0x00
+#define	CTR_CT1		0x10
+
+/* Register offsets from the base address. */
+#define	CTR_COUNT	0x00
+#define	CTR_LIMIT	0x08
+
+
+static unsigned int counter_get_timecount(struct timecounter *tc);
+
+struct ct_softc {
+	bus_space_tag_t		sc_tag;
+	bus_space_handle_t	sc_handle;
+	bus_addr_t		sc_offset;
+};
+
+
+/*
+ * This is called from the psycho and sbus drivers. It does not directly attach
+ * to the nexus because it shares register space with the bridge in question.
+ */
+void
+sparc64_counter_init(bus_space_tag_t tag, bus_space_handle_t handle,
+    bus_addr_t offset)
+{
+	struct timecounter *tc;
+	struct ct_softc *sc;
+
+	printf("initializing counter-timer\n");
+	/*
+	 * Turn off interrupts from both counters. Set the limit to the maximum
+	 * value (although that should not change anything with CTLR_INTEN and
+	 * CTLR_PERIODIC off).
+	 */
+	bus_space_write_8(tag, handle, offset + CTR_CT0 + CTR_LIMIT,
+	    COUNTER_MASK);
+	bus_space_write_8(tag, handle, offset + CTR_CT1 + CTR_LIMIT,
+	    COUNTER_MASK);
+	/* Register as a time counter. */
+	tc = malloc(sizeof(*tc), M_DEVBUF, M_WAITOK);
+	sc = malloc(sizeof(*sc), M_DEVBUF, M_WAITOK);
+	sc->sc_tag = tag;
+	sc->sc_handle = handle;
+	sc->sc_offset = offset + CTR_CT0;
+	tc->tc_get_timecount = counter_get_timecount;
+	tc->tc_poll_pps = NULL;
+	tc->tc_counter_mask = COUNTER_MASK;
+	tc->tc_frequency = COUNTER_FREQ;
+	tc->tc_name = "counter-timer";
+	tc->tc_priv = sc;
+	tc->tc_quality = COUNTER_QUALITY;
+	tc_init(tc);
+}
+
+static unsigned int
+counter_get_timecount(struct timecounter *tc)
+{
+	struct ct_softc *sc;
+
+	sc = tc->tc_priv;
+	return (bus_space_read_8(sc->sc_tag, sc->sc_handle, sc->sc_offset) &
+	    COUNTER_MASK);
+}
diff --git a/sys/sun4v/sun4v/db_disasm.c b/sys/sun4v/sun4v/db_disasm.c
new file mode 100644
index 000000000000..32bdcca9069b
--- /dev/null
+++ b/sys/sun4v/sun4v/db_disasm.c
@@ -0,0 +1,1031 @@
+/*-
+ * Copyright (c) 1994 David S. Miller, davem@nadzieja.rutgers.edu
+ * Copyright (c) 1995 Paul Kranenburg
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ *    must display the following acknowledgement:
+ *      This product includes software developed by David Miller.
+ * 4. The name of the author may not be used to endorse or promote products
+ *    derived from this software without specific prior written permission
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *	from: NetBSD: db_disasm.c,v 1.9 2000/08/16 11:29:42 pk Exp
+ * $FreeBSD$
+ */
+
+#include <sys/param.h>
+#include <sys/systm.h>
+
+#include <ddb/ddb.h>
+#include <ddb/db_access.h>
+#include <ddb/db_sym.h>
+
+#include <machine/db_machdep.h>
+#include <machine/instr.h>
+
+#define SIGN(v)			(((v)<0)?"-":"")
+
+/*
+ * All Sparc instructions are 32-bits, with the one exception being
+ * the set instruction which is actually a macro which expands into
+ * two instructions...
+ *
+ * There are 5 different fields that can be used to identify which
+ * operation is encoded into a particular 32-bit insn. There are 3
+ * formats for instuctions, which one being used is determined by
+ * bits 30-31 of the insn. Here are the bit fields and their names:
+ *
+ * 1100 0000 0000 0000 0000 0000 0000 0000 op field, determines format
+ * 0000 0001 1100 0000 0000 0000 0000 0000 op2 field, format 2 only
+ * 0000 0001 1111 1000 0000 0000 0000 0000 op3 field, format 3 only
+ * 0000 0000 0000 0000 0010 0000 0000 0000 f3i bit, format 3 only
+ * 0000 0000 0000 0000 0001 0000 0000 0000 X bit, format 3 only
+ */
+
+/* FORMAT macros used in sparc_i table to decode each opcode */
+#define FORMAT1(a)	(EIF_OP(a))
+#define FORMAT2(a,b)	(EIF_OP(a) | EIF_F2_OP2(b))
+/* For formats 3 and 4 */
+#define FORMAT3(a,b,c)	(EIF_OP(a) | EIF_F3_OP3(b) | EIF_F3_I(c))
+#define FORMAT3F(a,b,c)	(EIF_OP(a) | EIF_F3_OP3(b) | EIF_F3_OPF(c))
+
+/* Helper macros to construct OP3 & OPF */
+#define OP3_X(x,y)	((((x) & 3) << 4) | ((y) & 0xf))
+#define OPF_X(x,y)	((((x) & 0x1f) << 4) | ((y) & 0xf))
+
+/* COND condition codes field... */
+#define COND2(y,x)	(((((y)<<4) & 1)|((x) & 0xf)) << 14)
+
+struct sparc_insn {
+	  unsigned int match;
+	  char* name;
+	  char* format;
+};
+
+char* regs[] = {
+	"g0", "g1", "g2", "g3", "g4", "g5", "g6", "g7",
+	"o0", "o1", "o2", "o3", "o4", "o5", "sp", "o7",
+	"l0", "l1", "l2", "l3", "l4", "l5", "l6", "l7",
+	"i0", "i1", "i2", "i3", "i4", "i5", "fp", "i7"
+};
+
+char* priv_regs[] = {
+	"tpc", "tnpc", "tstate", "tt", "tick", "tba", "pstate", "tl",
+	"pil", "cwp", "cansave", "canrestore", "cleanwin", "otherwin",
+	"wstate", "fq",
+	"", "", "", "", "", "", "", "",
+	"", "", "", "", "", "", "", "ver"
+};
+
+char* state_regs[] = {
+	"y", "", "ccr", "asi", "tick", "pc", "fprs", "asr",
+	"", "", "", "", "", "", "", "",
+	"pcr", "pic", "dcr", "gsr", "set_softint", "clr_softint", "softint", "tick_cmpr", "",
+	"", "", "", "", "", "", "", ""
+};
+
+char* ccodes[] = {
+	"fcc0", "fcc1", "fcc2", "fcc3", "icc", "", "xcc", ""
+};
+
+char* prefetch[] = {
+	"n_reads", "one_read", "n_writes", "one_write", "page"
+};
+
+
+/* The sparc instruction table has a format field which tells what
+   the operand structure for this instruction is. Here are the codes:
+
+Modifiers (nust be first):
+	a -- opcode has annul bit
+	p -- opcode has branch prediction bit
+
+Codes:
+        1 -- source register operand stored in rs1
+	2 -- source register operand stored in rs2
+	d -- destination register operand stored in rd
+	3 -- floating source register in rs1
+	4 -- floating source register in rs2
+	e -- floating destination register in rd
+	i -- 13-bit immediate value stored in simm13
+	j -- 11-bit immediate value stored in simm11
+	l -- displacement using d16lo and d16hi
+	m -- 22-bit fcc displacement value
+	n -- 30-bit displacement used in call insns
+	o -- %fcc number specified in cc1 and cc0 fields
+	p -- address computed by the contents of rs1+rs2
+	q -- address computed by the contents of rs1+simm13
+	r -- prefetch
+	s -- %asi is implicit in the insn, rs1 value not used
+	t -- immediate 8-bit asi value
+	u -- 19-bit fcc displacement value
+	5 -- hard register, %fsr lower-half
+	6 -- hard register, %fsr all
+	7 -- [reg_addr rs1+rs2] imm_asi
+	8 -- [reg_addr rs1+simm13] %asi
+	9 -- logical or of the cmask and mmask fields (membar insn)
+	0 -- icc or xcc condition codes register
+	. -- %fcc, %icc, or %xcc in opf_cc field
+	r -- prefection function stored in fcn field
+	A -- privileged register encoded in rs1
+	B -- state register encoded in rs1
+	C -- %hi(value) where value is stored in imm22 field
+	D -- 32-bit shift count in shcnt32
+	E -- 64-bit shift count in shcnt64
+	F -- software trap number stored in sw_trap
+	G -- privileged register encoded in rd
+	H -- state register encoded in rd
+
+V8 only:
+	Y -- write y register
+	P -- write psr register
+	T -- write tbr register
+	W -- write wim register
+*/
+
+
+struct sparc_insn sparc_i[] = {
+
+	/*
+	 * Format 1: Call
+	 */
+	{(FORMAT1(1)), "call", "n"},
+
+	/*
+	 * Format 0: Sethi & Branches
+	 */
+	/* Illegal Instruction Trap */
+	{(FORMAT2(0, 0)), "illtrap", "m"},
+
+	/* Note: if imm22 is zero then this is actually a "nop" grrr... */
+	{(FORMAT2(0, 0x4)), "sethi", "Cd"},
+
+	/* Branch on Integer Co`ndition Codes "Bicc" */
+	{(FORMAT2(0, 2) | EIF_F2_COND(8)), "ba", "a,m"},
+	{(FORMAT2(0, 2) | EIF_F2_COND(0)), "bn", "a,m"},
+	{(FORMAT2(0, 2) | EIF_F2_COND(9)), "bne", "a,m"},
+	{(FORMAT2(0, 2) | EIF_F2_COND(1)), "be", "a,m"},
+	{(FORMAT2(0, 2) | EIF_F2_COND(10)), "bg", "a,m"},
+	{(FORMAT2(0, 2) | EIF_F2_COND(2)), "ble", "a,m"},
+	{(FORMAT2(0, 2) | EIF_F2_COND(11)), "bge", "a,m"},
+	{(FORMAT2(0, 2) | EIF_F2_COND(3)), "bl", "a,m"},
+	{(FORMAT2(0, 2) | EIF_F2_COND(12)), "bgu", "a,m"},
+	{(FORMAT2(0, 2) | EIF_F2_COND(4)), "bleu", "a,m"},
+	{(FORMAT2(0, 2) | EIF_F2_COND(13)), "bcc", "a,m"},
+	{(FORMAT2(0, 2) | EIF_F2_COND(5)), "bcs", "a,m"},
+	{(FORMAT2(0, 2) | EIF_F2_COND(14)), "bpos", "a,m"},
+	{(FORMAT2(0, 2) | EIF_F2_COND(6)), "bneg", "a,m"},
+	{(FORMAT2(0, 2) | EIF_F2_COND(15)), "bvc", "a,m"},
+	{(FORMAT2(0, 2) | EIF_F2_COND(7)), "bvs", "a,m"},
+
+	/* Branch on Integer Condition Codes with Prediction "BPcc" */
+	{(FORMAT2(0, 1) | EIF_F2_COND(8)), "ba", "ap,u"},
+	{(FORMAT2(0, 1) | EIF_F2_COND(0)), "bn", "ap,u"},
+	{(FORMAT2(0, 1) | EIF_F2_COND(9)), "bne", "ap,u"},
+	{(FORMAT2(0, 1) | EIF_F2_COND(1)), "be", "ap,u"},
+	{(FORMAT2(0, 1) | EIF_F2_COND(10)), "bg", "ap,u"},
+	{(FORMAT2(0, 1) | EIF_F2_COND(2)), "ble", "ap,u"},
+	{(FORMAT2(0, 1) | EIF_F2_COND(11)), "bge", "ap,u"},
+	{(FORMAT2(0, 1) | EIF_F2_COND(3)), "bl", "ap,u"},
+	{(FORMAT2(0, 1) | EIF_F2_COND(12)), "bgu", "ap,u"},
+	{(FORMAT2(0, 1) | EIF_F2_COND(4)), "bleu", "ap,u"},
+	{(FORMAT2(0, 1) | EIF_F2_COND(13)), "bcc", "ap,u"},
+	{(FORMAT2(0, 1) | EIF_F2_COND(5)), "bcs", "ap,u"},
+	{(FORMAT2(0, 1) | EIF_F2_COND(14)), "bpos", "ap,u"},
+	{(FORMAT2(0, 1) | EIF_F2_COND(6)), "bneg", "ap,u"},
+	{(FORMAT2(0, 1) | EIF_F2_COND(15)), "bvc", "ap,u"},
+	{(FORMAT2(0, 1) | EIF_F2_COND(7)), "bvs", "ap,u"},
+
+	/* Branch on Integer Register with Prediction "BPr" */
+	{(FORMAT2(0, 3) | EIF_F2_RCOND(1)), "brz", "ap,1l"},
+	{(FORMAT2(0, 3) | EIF_F2_A(1) | EIF_F2_P(1) |
+	    EIF_F2_RCOND(2)), "brlex", "ap,1l"},
+	{(FORMAT2(0, 3) | EIF_F2_RCOND(3)), "brlz", "ap,1l"},
+	{(FORMAT2(0, 3) | EIF_F2_RCOND(5)), "brnz", "ap,1l"},
+	{(FORMAT2(0, 3) | EIF_F2_RCOND(6)), "brgz", "ap,1l"},
+	{(FORMAT2(0, 3) | EIF_F2_RCOND(7)), "brgez", "ap,1l"},
+
+	/* Branch on Floating-Point Condition Codes with Prediction "FBPfcc" */
+	{(FORMAT2(0, 5) | EIF_F2_COND(8)), "fba", "ap,m"},
+	{(FORMAT2(0, 5) | EIF_F2_COND(0)), "fbn", "ap,m"},
+	{(FORMAT2(0, 5) | EIF_F2_COND(7)), "fbu", "ap,m"},
+	{(FORMAT2(0, 5) | EIF_F2_COND(6)), "fbg", "ap,m"},
+	{(FORMAT2(0, 5) | EIF_F2_COND(5)), "fbug", "ap,m"},
+	{(FORMAT2(0, 5) | EIF_F2_COND(4)), "fbl", "ap,m"},
+	{(FORMAT2(0, 5) | EIF_F2_COND(3)), "fbul", "ap,m"},
+	{(FORMAT2(0, 5) | EIF_F2_COND(2)), "fblg", "ap,m"},
+	{(FORMAT2(0, 5) | EIF_F2_COND(1)), "fbne", "ap,m"},
+	{(FORMAT2(0, 5) | EIF_F2_COND(9)), "fbe", "ap,m"},
+	{(FORMAT2(0, 5) | EIF_F2_COND(10)), "fbue", "ap,m"},
+	{(FORMAT2(0, 5) | EIF_F2_COND(11)), "fbge", "ap,m"},
+	{(FORMAT2(0, 5) | EIF_F2_COND(12)), "fbuge", "ap,m"},
+	{(FORMAT2(0, 5) | EIF_F2_COND(13)), "fble", "ap,m"},
+	{(FORMAT2(0, 5) | EIF_F2_COND(14)), "fbule", "ap,m"},
+	{(FORMAT2(0, 5) | EIF_F2_COND(15)), "fbo", "ap,m"},
+
+	/* Branch on Floating-Point Condition Codes "FBfcc" */
+	{(FORMAT2(0, 6) | EIF_F2_COND(8)), "fba", "a,m"},
+	{(FORMAT2(0, 6) | EIF_F2_COND(0)), "fbn", "a,m"},
+	{(FORMAT2(0, 6) | EIF_F2_COND(7)), "fbu", "a,m"},
+	{(FORMAT2(0, 6) | EIF_F2_COND(6)), "fbg", "a,m"},
+	{(FORMAT2(0, 6) | EIF_F2_COND(5)), "fbug", "a,m"},
+	{(FORMAT2(0, 6) | EIF_F2_COND(4)), "fbl", "a,m"},
+	{(FORMAT2(0, 6) | EIF_F2_COND(3)), "fbul", "a,m"},
+	{(FORMAT2(0, 6) | EIF_F2_COND(2)), "fblg", "a,m"},
+	{(FORMAT2(0, 6) | EIF_F2_COND(1)), "fbne", "a,m"},
+	{(FORMAT2(0, 6) | EIF_F2_COND(9)), "fbe", "a,m"},
+	{(FORMAT2(0, 6) | EIF_F2_COND(10)), "fbue", "a,m"},
+	{(FORMAT2(0, 6) | EIF_F2_COND(11)), "fbge", "a,m"},
+	{(FORMAT2(0, 6) | EIF_F2_COND(12)), "fbuge", "a,m"},
+	{(FORMAT2(0, 6) | EIF_F2_COND(13)), "fble", "a,m"},
+	{(FORMAT2(0, 6) | EIF_F2_COND(14)), "fbule", "a,m"},
+	{(FORMAT2(0, 6) | EIF_F2_COND(15)), "fbo", "a,m"},
+
+
+
+	/*
+	 * Format 3/2: Arithmetic & misc (table 32, appendix E)
+	 */
+	{FORMAT3(2, OP3_X(0,0), 0), "add", "12d"},
+	{FORMAT3(2, OP3_X(0,0), 1), "add", "1id"},
+	{FORMAT3(2, OP3_X(1,0), 0), "addcc", "12d"},
+	{FORMAT3(2, OP3_X(1,0), 1), "addcc", "1id"},
+	{FORMAT3(2, OP3_X(2,0), 0), "taddcc", "12d"},
+	{FORMAT3(2, OP3_X(2,0), 1), "taddcc", "1id"},
+	{(FORMAT3(2, 0x30, 1) | EIF_F3_RD(0xf)), "sir", "i"},
+	{FORMAT3(2, OP3_X(3,0), 0), "wr", "12H"},
+	{FORMAT3(2, OP3_X(3,0), 1), "wr", "1iH"},
+
+	{FORMAT3(2, OP3_X(0,1), 0), "and", "12d"},
+	{FORMAT3(2, OP3_X(0,1), 1), "and", "1id"},
+	{FORMAT3(2, OP3_X(1,1), 0), "andcc", "12d"},
+	{FORMAT3(2, OP3_X(1,1), 1), "andcc", "1id"},
+	{FORMAT3(2, OP3_X(2,1), 0), "tsubcc", "12d"},
+	{FORMAT3(2, OP3_X(2,1), 1), "tsubcc", "1id"},
+	{FORMAT3(2, OP3_X(3,1), 0), "saved", ""},
+	{FORMAT3(2, OP3_X(3,1), 0) | EIF_F3_FCN(1), "restored", ""},
+
+	{FORMAT3(2, OP3_X(0,2), 0), "or", "12d"},
+	{FORMAT3(2, OP3_X(0,2), 1), "or", "1id"},
+	{FORMAT3(2, OP3_X(1,2), 0), "orcc", "12d"},
+	{FORMAT3(2, OP3_X(1,2), 1), "orcc", "1id"},
+	{FORMAT3(2, OP3_X(2,2), 0), "taddcctv", "12d"},
+	{FORMAT3(2, OP3_X(2,2), 1), "taddcctv", "1id"},
+	{FORMAT3(2, OP3_X(3,2), 0), "wrpr", "12G"},
+	{FORMAT3(2, OP3_X(3,2), 1), "wrpr", "1iG"},
+
+	{FORMAT3(2, OP3_X(0,3), 0), "xor", "12d"},
+	{FORMAT3(2, OP3_X(0,3), 1), "xor", "1id"},
+	{FORMAT3(2, OP3_X(1,3), 0), "xorcc", "12d"},
+	{FORMAT3(2, OP3_X(1,3), 1), "xorcc", "1id"},
+	{FORMAT3(2, OP3_X(2,3), 0), "tsubcctv", "12d"},
+	{FORMAT3(2, OP3_X(2,3), 1), "tsubcctv", "1id"},
+	{FORMAT3(2, OP3_X(3,3), 0), "UNDEFINED", ""},
+
+	{FORMAT3(2, OP3_X(0,4), 0), "sub", "12d"},
+	{FORMAT3(2, OP3_X(0,4), 1), "sub", "1id"},
+	{FORMAT3(2, OP3_X(1,4), 0), "subcc", "12d"},
+	{FORMAT3(2, OP3_X(1,4), 1), "subcc", "1id"},
+	{FORMAT3(2, OP3_X(2,4), 0), "mulscc", "12d"},
+	{FORMAT3(2, OP3_X(2,4), 1), "mulscc", "1id"},
+	{FORMAT3(2, OP3_X(3,4), 1), "FPop1", ""},	/* see below */
+
+	{FORMAT3(2, OP3_X(0,5), 0), "andn", "12d"},
+	{FORMAT3(2, OP3_X(0,5), 1), "andn", "1id"},
+	{FORMAT3(2, OP3_X(1,5), 0), "andncc", "12d"},
+	{FORMAT3(2, OP3_X(1,5), 1), "andncc", "1id"},
+	{FORMAT3(2, OP3_X(2,5), 0), "sll", "12d"},
+	{FORMAT3(2, OP3_X(2,5), 1), "sll", "1Dd"},
+	{FORMAT3(2, OP3_X(2,5), 0) | EIF_F3_X(1), "sllx", "12d"},
+	{FORMAT3(2, OP3_X(2,5), 1) | EIF_F3_X(1), "sllx", "1Ed"},
+	{FORMAT3(2, OP3_X(3,5), 1), "FPop2", ""},	/* see below */
+
+	{FORMAT3(2, OP3_X(0,6), 0), "orn", "12d"},
+	{FORMAT3(2, OP3_X(0,6), 1), "orn", "1id"},
+	{FORMAT3(2, OP3_X(1,6), 0), "orncc", "12d"},
+	{FORMAT3(2, OP3_X(1,6), 1), "orncc", "1id"},
+	{FORMAT3(2, OP3_X(2,6), 0), "srl", "12d"},
+	{FORMAT3(2, OP3_X(2,6), 1), "srl", "1Dd"},
+	{FORMAT3(2, OP3_X(2,6), 0) | EIF_F3_X(1), "srlx", "12d"},
+	{FORMAT3(2, OP3_X(2,6), 1) | EIF_F3_X(1), "srlx", "1Ed"},
+	{FORMAT3(2, OP3_X(3,6), 1), "impdep1", ""},
+
+	{FORMAT3(2, OP3_X(0,7), 0), "xorn", "12d"},
+	{FORMAT3(2, OP3_X(0,7), 1), "xorn", "1id"},
+	{FORMAT3(2, OP3_X(1,7), 0), "xorncc", "12d"},
+	{FORMAT3(2, OP3_X(1,7), 1), "xorncc", "1id"},
+	{FORMAT3(2, OP3_X(2,7), 0), "sra", "12d"},
+	{FORMAT3(2, OP3_X(2,7), 1), "sra", "1Dd"},
+	{FORMAT3(2, OP3_X(2,7), 0) | EIF_F3_X(1), "srax", "12d"},
+	{FORMAT3(2, OP3_X(2,7), 1) | EIF_F3_X(1), "srax", "1Ed"},
+	{FORMAT3(2, OP3_X(3,7), 1), "impdep2", ""},
+
+	{FORMAT3(2, OP3_X(0,8), 0), "addc", "12d"},
+	{FORMAT3(2, OP3_X(0,8), 1), "addc", "1id"},
+	{FORMAT3(2, OP3_X(1,8), 0), "addccc", "12d"},
+	{FORMAT3(2, OP3_X(1,8), 1), "addccc", "1id"},
+	{(FORMAT3(2, 0x28, 1) | EIF_F3_RS1(15)), "membar", "9"},
+	{(FORMAT3(2, 0x28, 0) | EIF_F3_RS1(15)), "stbar", ""},
+	{FORMAT3(2, OP3_X(2,8), 0), "rd", "Bd"},
+
+	{FORMAT3(2, OP3_X(3,8), 0), "jmpl", "pd"},
+	{FORMAT3(2, OP3_X(3,8), 1), "jmpl", "qd"},
+
+	{FORMAT3(2, OP3_X(0,9), 0), "mulx", "12d"},
+	{FORMAT3(2, OP3_X(0,9), 1), "mulx", "1id"},
+	{FORMAT3(2, OP3_X(1,9), 0), "UNDEFINED", ""},
+	{FORMAT3(2, OP3_X(2,9), 0), "UNDEFINED", ""},
+	{FORMAT3(2, OP3_X(3,9), 0), "return", "p"},
+	{FORMAT3(2, OP3_X(3,9), 1), "return", "q"},
+
+	{FORMAT3(2, OP3_X(0,10), 0), "umul", "12d"},
+	{FORMAT3(2, OP3_X(0,10), 1), "umul", "1id"},
+	{FORMAT3(2, OP3_X(1,10), 0), "umulcc", "12d"},
+	{FORMAT3(2, OP3_X(1,10), 1), "umulcc", "1id"},
+	{FORMAT3(2, OP3_X(2,10), 0), "rdpr", "Ad"},
+		/*
+		 * OP3 = (3,10): TCC: Trap on Integer Condition Codes
+		 */
+		{(FORMAT3(2, OP3_X(3,10), 0) | EIF_F4_TCOND(0x8)), "ta", "12F"},
+		{(FORMAT3(2, OP3_X(3,10), 1) | EIF_F4_TCOND(0x8)), "ta", "0F"},
+		{(FORMAT3(2, OP3_X(3,10), 0) | EIF_F4_TCOND(0x0)), "tn", "12F"},
+		{(FORMAT3(2, OP3_X(3,10), 1) | EIF_F4_TCOND(0x0)), "tn", "0F"},
+		{(FORMAT3(2, OP3_X(3,10), 0) | EIF_F4_TCOND(0x9)), "tne", "12F"},
+		{(FORMAT3(2, OP3_X(3,10), 1) | EIF_F4_TCOND(0x9)), "tne", "0F"},
+		{(FORMAT3(2, OP3_X(3,10), 0) | EIF_F4_TCOND(0x1)), "te", "12F"},
+		{(FORMAT3(2, OP3_X(3,10), 1) | EIF_F4_TCOND(0x1)), "te", "0F"},
+		{(FORMAT3(2, OP3_X(3,10), 0) | EIF_F4_TCOND(0xa)), "tg", "12F"},
+		{(FORMAT3(2, OP3_X(3,10), 1) | EIF_F4_TCOND(0xa)), "tg", "0F"},
+		{(FORMAT3(2, OP3_X(3,10), 0) | EIF_F4_TCOND(0x2)), "tle", "12F"},
+		{(FORMAT3(2, OP3_X(3,10), 1) | EIF_F4_TCOND(0x2)), "tle", "0F"},
+		{(FORMAT3(2, OP3_X(3,10), 0) | EIF_F4_TCOND(0xb)), "tge", "12F"},
+		{(FORMAT3(2, OP3_X(3,10), 1) | EIF_F4_TCOND(0xb)), "tge", "0F"},
+		{(FORMAT3(2, OP3_X(3,10), 0) | EIF_F4_TCOND(0x3)), "tl", "12F"},
+		{(FORMAT3(2, OP3_X(3,10), 1) | EIF_F4_TCOND(0x3)), "tl", "0F"},
+		{(FORMAT3(2, OP3_X(3,10), 0) | EIF_F4_TCOND(0xc)), "tgu", "12F"},
+		{(FORMAT3(2, OP3_X(3,10), 1) | EIF_F4_TCOND(0xc)), "tgu", "0F"},
+		{(FORMAT3(2, OP3_X(3,10), 0) | EIF_F4_TCOND(0x4)), "tleu", "12F"},
+		{(FORMAT3(2, OP3_X(3,10), 1) | EIF_F4_TCOND(0x4)), "tleu", "0F"},
+		{(FORMAT3(2, OP3_X(3,10), 0) | EIF_F4_TCOND(0xd)), "tcc", "12F"},
+		{(FORMAT3(2, OP3_X(3,10), 1) | EIF_F4_TCOND(0xd)), "tcc", "0F"},
+		{(FORMAT3(2, OP3_X(3,10), 0) | EIF_F4_TCOND(0x5)), "tcs", "12F"},
+		{(FORMAT3(2, OP3_X(3,10), 1) | EIF_F4_TCOND(0x5)), "tcs", "0F"},
+		{(FORMAT3(2, OP3_X(3,10), 0) | EIF_F4_TCOND(0xe)), "tpos", "12F"},
+		{(FORMAT3(2, OP3_X(3,10), 1) | EIF_F4_TCOND(0xe)), "tpos", "0F"},
+		{(FORMAT3(2, OP3_X(3,10), 0) | EIF_F4_TCOND(0x6)), "tneg", "12F"},
+		{(FORMAT3(2, OP3_X(3,10), 1) | EIF_F4_TCOND(0x6)), "tneg", "0F"},
+		{(FORMAT3(2, OP3_X(3,10), 0) | EIF_F4_TCOND(0xf)), "tvc", "12F"},
+		{(FORMAT3(2, OP3_X(3,10), 1) | EIF_F4_TCOND(0xf)), "tvc", "0F"},
+		{(FORMAT3(2, OP3_X(3,10), 0) | EIF_F4_TCOND(0x7)), "tvs", "12F"},
+		{(FORMAT3(2, OP3_X(3,10), 1) | EIF_F4_TCOND(0x7)), "tvs", "0F"},
+
+	{FORMAT3(2, OP3_X(0,11), 0), "smul", "12d"},
+	{FORMAT3(2, OP3_X(0,11), 1), "smul", "1id"},
+	{FORMAT3(2, OP3_X(1,11), 0), "smulcc", "12d"},
+	{FORMAT3(2, OP3_X(1,11), 1), "smulcc", "1id"},
+	{FORMAT3(2, OP3_X(2,11), 0), "flushw", ""},
+	{FORMAT3(2, OP3_X(3,11), 0), "flush", "p"},
+	{FORMAT3(2, OP3_X(3,11), 1), "flush", "q"},
+
+	{FORMAT3(2, OP3_X(0,12), 0), "subc", "12d"},
+	{FORMAT3(2, OP3_X(0,12), 1), "subc", "1id"},
+	{FORMAT3(2, OP3_X(1,12), 0), "subccc", "12d"},
+	{FORMAT3(2, OP3_X(1,12), 1), "subccc", "1id"},
+		/*
+		 * OP3 = (2,12): MOVcc, Move Integer Register on Condition
+		 */
+		/* For Integer Condition Codes */
+		{(FORMAT3(2, OP3_X(2,12), 1) | COND2(1,8)), "mova", "0jd"},
+		{(FORMAT3(2, OP3_X(2,12), 0) | COND2(1,8)), "mova", "02d"},
+		{(FORMAT3(2, OP3_X(2,12), 1) | COND2(1,0)), "movn", "0jd"},
+		{(FORMAT3(2, OP3_X(2,12), 0) | COND2(1,0)), "movn", "02d"},
+		{(FORMAT3(2, OP3_X(2,12), 1) | COND2(1,9)), "movne", "0jd"},
+		{(FORMAT3(2, OP3_X(2,12), 0) | COND2(1,9)), "movne", "02d"},
+		{(FORMAT3(2, OP3_X(2,12), 1) | COND2(1,1)), "move", "0jd"},
+		{(FORMAT3(2, OP3_X(2,12), 0) | COND2(1,1)), "move", "02d"},
+		{(FORMAT3(2, OP3_X(2,12), 1) | COND2(1,10)), "movg", "0jd"},
+		{(FORMAT3(2, OP3_X(2,12), 0) | COND2(1,10)), "movg", "02d"},
+		{(FORMAT3(2, OP3_X(2,12), 1) | COND2(1,2)), "movle", "0jd"},
+		{(FORMAT3(2, OP3_X(2,12), 0) | COND2(1,2)), "movle", "02d"},
+		{(FORMAT3(2, OP3_X(2,12), 1) | COND2(1,11)), "movge", "0jd"},
+		{(FORMAT3(2, OP3_X(2,12), 0) | COND2(1,11)), "movge", "02d"},
+		{(FORMAT3(2, OP3_X(2,12), 1) | COND2(1,3)), "movl", "0jd"},
+		{(FORMAT3(2, OP3_X(2,12), 0) | COND2(1,3)), "movl", "02d"},
+		{(FORMAT3(2, OP3_X(2,12), 1) | COND2(1,12)), "movgu", "0jd"},
+		{(FORMAT3(2, OP3_X(2,12), 0) | COND2(1,12)), "movgu", "02d"},
+		{(FORMAT3(2, OP3_X(2,12), 1) | COND2(1,4)), "movleu", "0jd"},
+		{(FORMAT3(2, OP3_X(2,12), 0) | COND2(1,4)), "movleu", "02d"},
+		{(FORMAT3(2, OP3_X(2,12), 1) | COND2(1,13)), "movcc", "0jd"},
+		{(FORMAT3(2, OP3_X(2,12), 0) | COND2(1,13)), "movcc", "02d"},
+		{(FORMAT3(2, OP3_X(2,12), 1) | COND2(1,5)), "movcs", "0jd"},
+		{(FORMAT3(2, OP3_X(2,12), 0) | COND2(1,5)), "movcs", "02d"},
+		{(FORMAT3(2, OP3_X(2,12), 1) | COND2(1,14)), "movpos", "0jd"},
+		{(FORMAT3(2, OP3_X(2,12), 0) | COND2(1,14)), "movpos", "02d"},
+		{(FORMAT3(2, OP3_X(2,12), 1) | COND2(1,6)), "movneg", "0jd"},
+		{(FORMAT3(2, OP3_X(2,12), 0) | COND2(1,6)), "movneg", "02d"},
+		{(FORMAT3(2, OP3_X(2,12), 1) | COND2(1,15)), "movvc", "0jd"},
+		{(FORMAT3(2, OP3_X(2,12), 0) | COND2(1,15)), "movvc", "02d"},
+		{(FORMAT3(2, OP3_X(2,12), 1) | COND2(1,7)), "movvs", "0jd"},
+		{(FORMAT3(2, OP3_X(2,12), 0) | COND2(1,7)), "movvs", "02d"},
+
+		/* For Floating-Point Condition Codes */
+		{(FORMAT3(2, OP3_X(2,12), 1) | COND2(0,8)), "mova", "ojd"},
+		{(FORMAT3(2, OP3_X(2,12), 0) | COND2(0,8)), "mova", "o2d"},
+		{(FORMAT3(2, OP3_X(2,12), 1) | COND2(0,0)), "movn", "ojd"},
+		{(FORMAT3(2, OP3_X(2,12), 0) | COND2(0,0)), "movn", "o2d"},
+		{(FORMAT3(2, OP3_X(2,12), 1) | COND2(0,7)), "movu", "ojd"},
+		{(FORMAT3(2, OP3_X(2,12), 0) | COND2(0,7)), "movu", "o2d"},
+		{(FORMAT3(2, OP3_X(2,12), 1) | COND2(0,6)), "movg", "ojd"},
+		{(FORMAT3(2, OP3_X(2,12), 0) | COND2(0,6)), "movg", "o2d"},
+		{(FORMAT3(2, OP3_X(2,12), 1) | COND2(0,5)), "movug", "ojd"},
+		{(FORMAT3(2, OP3_X(2,12), 0) | COND2(0,5)), "movug", "o2d"},
+		{(FORMAT3(2, OP3_X(2,12), 1) | COND2(0,4)), "movl", "ojd"},
+		{(FORMAT3(2, OP3_X(2,12), 0) | COND2(0,4)), "movl", "o2d"},
+		{(FORMAT3(2, OP3_X(2,12), 1) | COND2(0,3)), "movul", "ojd"},
+		{(FORMAT3(2, OP3_X(2,12), 0) | COND2(0,3)), "movul", "o2d"},
+		{(FORMAT3(2, OP3_X(2,12), 1) | COND2(0,2)), "movlg", "ojd"},
+		{(FORMAT3(2, OP3_X(2,12), 0) | COND2(0,2)), "movlg", "o2d"},
+		{(FORMAT3(2, OP3_X(2,12), 1) | COND2(0,1)), "movne", "ojd"},
+		{(FORMAT3(2, OP3_X(2,12), 0) | COND2(0,1)), "movne", "o2d"},
+		{(FORMAT3(2, OP3_X(2,12), 1) | COND2(0,9)), "move", "ojd"},
+		{(FORMAT3(2, OP3_X(2,12), 0) | COND2(0,9)), "move", "o2d"},
+		{(FORMAT3(2, OP3_X(2,12), 1) | COND2(0,10)), "movue", "ojd"},
+		{(FORMAT3(2, OP3_X(2,12), 0) | COND2(0,10)), "movue", "o2d"},
+		{(FORMAT3(2, OP3_X(2,12), 1) | COND2(0,11)), "movge", "ojd"},
+		{(FORMAT3(2, OP3_X(2,12), 0) | COND2(0,11)), "movge", "o2d"},
+		{(FORMAT3(2, OP3_X(2,12), 1) | COND2(0,12)), "movuge", "ojd"},
+		{(FORMAT3(2, OP3_X(2,12), 0) | COND2(0,12)), "movuge", "o2d"},
+		{(FORMAT3(2, OP3_X(2,12), 1) | COND2(0,13)), "movle", "ojd"},
+		{(FORMAT3(2, OP3_X(2,12), 0) | COND2(0,13)), "movle", "o2d"},
+		{(FORMAT3(2, OP3_X(2,12), 1) | COND2(0,14)), "movule", "ojd"},
+		{(FORMAT3(2, OP3_X(2,12), 0) | COND2(0,14)), "movule", "o2d"},
+		{(FORMAT3(2, OP3_X(2,12), 1) | COND2(0,15)), "movo", "ojd"},
+		{(FORMAT3(2, OP3_X(2,12), 0) | COND2(0,15)), "movo", "o2d"},
+
+	{FORMAT3(2, OP3_X(3,12), 0), "save", "12d"},
+	{FORMAT3(2, OP3_X(3,12), 1), "save", "1id"},
+
+	{FORMAT3(2, OP3_X(0,13), 0), "udivx", "12d"},
+	{FORMAT3(2, OP3_X(0,13), 1), "udivx", "1id"},
+	{FORMAT3(2, OP3_X(1,13), 0), "UNDEFINED", ""},
+	{FORMAT3(2, OP3_X(2,13), 0), "sdivx", "12d"},
+	{FORMAT3(2, OP3_X(2,13), 1), "sdivx", "1id"},
+	{FORMAT3(2, OP3_X(3,13), 0), "restore", "12d"},
+	{FORMAT3(2, OP3_X(3,13), 1), "restore", "1id"},
+
+	{FORMAT3(2, OP3_X(0,14), 0), "udiv", "12d"},
+	{FORMAT3(2, OP3_X(0,14), 1), "udiv", "1id"},
+	{FORMAT3(2, OP3_X(1,14), 0), "udivcc", "12d"},
+	{FORMAT3(2, OP3_X(1,14), 1), "udivcc", "1id"},
+	{FORMAT3(2, OP3_X(2,14), 0), "popc", "2d"},
+	{FORMAT3(2, OP3_X(2,14), 1), "popc", "id"},
+
+	{FORMAT3(2, OP3_X(3,14), 0), "done", ""},
+	{FORMAT3(2, OP3_X(3,14) | EIF_F3_FCN(1), 1), "retry", ""},
+
+	{FORMAT3(2, OP3_X(0,15), 0), "sdiv", "12d"},
+	{FORMAT3(2, OP3_X(0,15), 1), "sdiv", "1id"},
+	{FORMAT3(2, OP3_X(1,15), 0), "sdivcc", "12d"},
+	{FORMAT3(2, OP3_X(1,15), 1), "sdivcc", "1id"},
+		/*
+		 * OP3 = (2,15): MOVr:
+		 * 	Move Integer Register on Register Condition
+		 */
+		{(FORMAT3(2, OP3_X(2,15), 1) | EIF_F3_RCOND(1)), "movrz", "1jd"},
+		{(FORMAT3(2, OP3_X(2,15), 0) | EIF_F3_RCOND(1)), "movrz", "12d"},
+		{(FORMAT3(2, OP3_X(2,15), 1) | EIF_F3_RCOND(2)), "movrlez", "1jd"},
+		{(FORMAT3(2, OP3_X(2,15), 0) | EIF_F3_RCOND(2)), "movrlez", "12d"},
+		{(FORMAT3(2, OP3_X(2,15), 1) | EIF_F3_RCOND(3)), "movrlz", "1jd"},
+		{(FORMAT3(2, OP3_X(2,15), 0) | EIF_F3_RCOND(3)), "movrlz", "12d"},
+		{(FORMAT3(2, OP3_X(2,15), 1) | EIF_F3_RCOND(5)), "movrnz", "1jd"},
+		{(FORMAT3(2, OP3_X(2,15), 0) | EIF_F3_RCOND(5)), "movrnz", "12d"},
+		{(FORMAT3(2, OP3_X(2,15), 1) | EIF_F3_RCOND(6)), "movrgz", "1jd"},
+		{(FORMAT3(2, OP3_X(2,15), 0) | EIF_F3_RCOND(6)), "movrgz", "12d"},
+		{(FORMAT3(2, OP3_X(2,15), 1) | EIF_F3_RCOND(7)), "movrgez", "1jd"},
+		{(FORMAT3(2, OP3_X(2,15), 0) | EIF_F3_RCOND(7)), "movrgez", "12d"},
+
+	{FORMAT3(2, OP3_X(3,15), 0), "UNDEFINED", ""},
+
+
+	/*
+	 * Format 3/3: Load and store (appendix E, table 33)
+	 */
+
+	/* Loads */
+	{(FORMAT3(3, OP3_X(0,0), 0)), "lduw", "pd"},
+	{(FORMAT3(3, OP3_X(0,0), 1)), "lduw", "qd"},
+	{(FORMAT3(3, OP3_X(1,0), 0)), "lduwa", "7d"},
+	{(FORMAT3(3, OP3_X(1,0), 1)), "lduwa", "8d"},
+	{(FORMAT3(3, OP3_X(2,0), 0)), "ldf", "pe"},
+	{(FORMAT3(3, OP3_X(2,0), 1)), "ldf", "qe"},
+	{(FORMAT3(3, OP3_X(3,0), 0)), "ldfa", "7e"},
+	{(FORMAT3(3, OP3_X(3,0), 1)), "ldfa", "8e"},
+
+	{(FORMAT3(3, OP3_X(0,1), 0)), "ldub", "pd"},
+	{(FORMAT3(3, OP3_X(0,1), 1)), "ldub", "qd"},
+	{(FORMAT3(3, OP3_X(1,1), 0)), "lduba", "7d"},
+	{(FORMAT3(3, OP3_X(1,1), 1)), "lduba", "8d"},
+	{(FORMAT3(3, OP3_X(2,1), 0) | EIF_F3_RD(0)), "lduw", "p5"},
+	{(FORMAT3(3, OP3_X(2,1), 1) | EIF_F3_RD(0)), "lduw", "q5"},
+	{(FORMAT3(3, OP3_X(2,1), 0) | EIF_F3_RD(1)), "ldx", "p6"},
+	{(FORMAT3(3, OP3_X(2,1), 1) | EIF_F3_RD(1)), "ldx", "q6"},
+
+	{(FORMAT3(3, OP3_X(0,2), 0)), "lduh", "pd"},
+	{(FORMAT3(3, OP3_X(0,2), 1)), "lduh", "qd"},
+	{(FORMAT3(3, OP3_X(1,2), 0)), "lduha", "7d"},
+	{(FORMAT3(3, OP3_X(1,2), 1)), "lduha", "8d"},
+	{(FORMAT3(3, OP3_X(2,2), 0)), "ldq", "pe"},
+	{(FORMAT3(3, OP3_X(2,2), 1)), "ldq", "qe"},
+	{(FORMAT3(3, OP3_X(3,2), 0)), "ldqa", "7e"},
+	{(FORMAT3(3, OP3_X(3,2), 1)), "ldqa", "8e"},
+
+	{(FORMAT3(3, OP3_X(0,3), 0)), "ldd", "pd"},
+	{(FORMAT3(3, OP3_X(0,3), 1)), "ldd", "qd"},
+	{(FORMAT3(3, OP3_X(1,3), 0)), "ldda", "7d"},
+	{(FORMAT3(3, OP3_X(1,3), 1)), "ldda", "8d"},
+	{(FORMAT3(3, OP3_X(2,3), 0)), "ldd", "pe"},
+	{(FORMAT3(3, OP3_X(2,3), 1)), "ldd", "qe"},
+	{(FORMAT3(3, OP3_X(3,3), 0)), "ldda", "7e"},
+	{(FORMAT3(3, OP3_X(3,3), 1)), "ldda", "8e"},
+
+	{(FORMAT3(3, OP3_X(0,4), 0)), "stw", "dp"},
+	{(FORMAT3(3, OP3_X(0,4), 1)), "stw", "dq"},
+	{(FORMAT3(3, OP3_X(1,4), 0)), "stwa", "d7"},
+	{(FORMAT3(3, OP3_X(1,4), 1)), "stwa", "d8"},
+	{(FORMAT3(3, OP3_X(2,4), 0)), "stf", "ep"},
+	{(FORMAT3(3, OP3_X(2,4), 1)), "stf", "eq"},
+	{(FORMAT3(3, OP3_X(3,4), 0)), "stfa", "e7"},
+	{(FORMAT3(3, OP3_X(3,4), 1)), "stfa", "e8"},
+
+	{(FORMAT3(3, OP3_X(0,5), 0)), "stb", "dp"},
+	{(FORMAT3(3, OP3_X(0,5), 1)), "stb", "dq"},
+	{(FORMAT3(3, OP3_X(1,5), 0)), "stba", "d7"},
+	{(FORMAT3(3, OP3_X(1,5), 1)), "stba", "d8"},
+	{(FORMAT3(3, OP3_X(2,5), 0)), "stw", "5p"},
+	{(FORMAT3(3, OP3_X(2,5), 1)), "stw", "5q"},
+	{(FORMAT3(3, OP3_X(2,5), 0) | EIF_F3_RD(1)), "stx", "6p"},
+	{(FORMAT3(3, OP3_X(2,5), 1) | EIF_F3_RD(1)), "stx", "6q"},
+
+	{(FORMAT3(3, OP3_X(0,6), 0)), "sth", "dp"},
+	{(FORMAT3(3, OP3_X(0,6), 1)), "sth", "dq"},
+	{(FORMAT3(3, OP3_X(1,6), 0)), "stha", "d7"},
+	{(FORMAT3(3, OP3_X(1,6), 1)), "stha", "d8"},
+	{(FORMAT3(3, OP3_X(2,6), 0)), "stq", "ep"},
+	{(FORMAT3(3, OP3_X(2,6), 1)), "stq", "eq"},
+	{(FORMAT3(3, OP3_X(3,6), 0)), "stqa", "e7"},
+	{(FORMAT3(3, OP3_X(3,6), 1)), "stqa", "e8"},
+
+	{(FORMAT3(3, OP3_X(0,7), 0)), "std", "dp"},
+	{(FORMAT3(3, OP3_X(0,7), 1)), "std", "dq"},
+	{(FORMAT3(3, OP3_X(1,7), 0)), "stda", "d7"},
+	{(FORMAT3(3, OP3_X(1,7), 1)), "stda", "d8"},
+	{(FORMAT3(3, OP3_X(2,7), 0)), "std", "ep"},
+	{(FORMAT3(3, OP3_X(2,7), 1)), "std", "eq"},
+	{(FORMAT3(3, OP3_X(3,7), 0)), "stda", "e7"},
+	{(FORMAT3(3, OP3_X(3,7), 1)), "stda", "e8"},
+
+	{(FORMAT3(3, OP3_X(0,8), 0)), "ldsw", "pd"},
+	{(FORMAT3(3, OP3_X(0,8), 1)), "ldsw", "qd"},
+	{(FORMAT3(3, OP3_X(1,8), 0)), "ldswa", "7d"},
+	{(FORMAT3(3, OP3_X(1,8), 1)), "ldswa", "8d"},
+
+	{(FORMAT3(3, OP3_X(0,9), 0)), "ldsb", "pd"},
+	{(FORMAT3(3, OP3_X(0,9), 1)), "ldsb", "qd"},
+	{(FORMAT3(3, OP3_X(1,9), 0)), "ldsba", "7d"},
+	{(FORMAT3(3, OP3_X(1,9), 1)), "ldsba", "8d"},
+
+	{(FORMAT3(3, OP3_X(0,10), 0)), "ldsh", "pd"},
+	{(FORMAT3(3, OP3_X(0,10), 1)), "ldsh", "qd"},
+	{(FORMAT3(3, OP3_X(1,10), 0)), "ldsha", "7d"},
+	{(FORMAT3(3, OP3_X(1,10), 1)), "ldsha", "8d"},
+
+	{(FORMAT3(3, OP3_X(0,11), 0)), "ldx", "pd"},
+	{(FORMAT3(3, OP3_X(0,11), 1)), "ldx", "qd"},
+	{(FORMAT3(3, OP3_X(1,11), 0)), "ldxa", "7d"},
+	{(FORMAT3(3, OP3_X(1,11), 1)), "ldxa", "8d"},
+
+	{(FORMAT3(3, OP3_X(3,12), 1)), "casa", "s2d"},
+	{(FORMAT3(3, OP3_X(3,12), 0)), "casa", "t2d"},
+
+	{(FORMAT3(3, OP3_X(0,13), 0)), "ldstub", "7d"},
+	{(FORMAT3(3, OP3_X(0,13), 1)), "ldstub", "8d"},
+	{(FORMAT3(3, OP3_X(1,13), 0)), "ldstuba", "pd"},
+	{(FORMAT3(3, OP3_X(1,13), 1)), "ldstuba", "qd"},
+	{(FORMAT3(3, OP3_X(2,13), 0)), "prefetch", "pr"},
+	{(FORMAT3(3, OP3_X(2,13), 1)), "prefetch", "qr"},
+	{(FORMAT3(3, OP3_X(3,13), 0)), "prefetcha", "7r"},
+	{(FORMAT3(3, OP3_X(3,13), 1)), "prefetcha", "8r"},
+
+	{(FORMAT3(3, OP3_X(0,14), 0)), "stx", "dp"},
+	{(FORMAT3(3, OP3_X(0,14), 1)), "stx", "dq"},
+	{(FORMAT3(3, OP3_X(1,14), 0)), "stxa", "d7"},
+	{(FORMAT3(3, OP3_X(1,14), 1)), "stxa", "d8"},
+	{(FORMAT3(3, OP3_X(3,14), 0)), "casxa", "t2d"},
+	{(FORMAT3(3, OP3_X(3,14), 1)), "casxa", "s2d"},
+
+	/* Swap Register */
+	{(FORMAT3(3, OP3_X(0,15), 0)), "swap", "pd"},
+	{(FORMAT3(3, OP3_X(0,15), 1)), "swap", "qd"},
+	{(FORMAT3(3, OP3_X(1,15), 0)), "swapa", "7d"},
+	{(FORMAT3(3, OP3_X(1,15), 1)), "swapa", "8d"},
+
+
+	/*
+	 * OP3 = (3,4): FPop1 (table 34)
+	 */
+	{(FORMAT3F(2, OP3_X(3,4), OPF_X(0,1))), "fmovs", ".4e"},
+	{(FORMAT3F(2, OP3_X(3,4), OPF_X(0,2))), "fmovd", ".4e"},
+	{(FORMAT3F(2, OP3_X(3,4), OPF_X(0,3))), "fmovq", ".4e"},
+	{(FORMAT3F(2, OP3_X(3,4), OPF_X(0,5))), "fnegs", "4e"},
+	{(FORMAT3F(2, OP3_X(3,4), OPF_X(0,6))), "fnegd", "4e"},
+	{(FORMAT3F(2, OP3_X(3,4), OPF_X(0,7))), "fnegq", "4e"},
+	{(FORMAT3F(2, OP3_X(3,4), OPF_X(0,9))), "fabss", "4e"},
+	{(FORMAT3F(2, OP3_X(3,4), OPF_X(0,10))), "fabsd", "4e"},
+	{(FORMAT3F(2, OP3_X(3,4), OPF_X(0,11))), "fabsq", "4e"},
+
+	{(FORMAT3F(2, OP3_X(3,4), OPF_X(2,9))), "fsqrts", "4e"},
+	{(FORMAT3F(2, OP3_X(3,4), OPF_X(2,10))), "fsqrtd", "4e"},
+	{(FORMAT3F(2, OP3_X(3,4), OPF_X(2,11))), "fsqrtq", "4e"},
+
+	{(FORMAT3F(2, OP3_X(3,4), OPF_X(4,1))), "fadds", "34e"},
+	{(FORMAT3F(2, OP3_X(3,4), OPF_X(4,2))), "faddd", "34e"},
+	{(FORMAT3F(2, OP3_X(3,4), OPF_X(4,3))), "faddq", "34e"},
+	{(FORMAT3F(2, OP3_X(3,4), OPF_X(4,5))), "fsubs", "34e"},
+	{(FORMAT3F(2, OP3_X(3,4), OPF_X(4,6))), "fsubd", "34e"},
+	{(FORMAT3F(2, OP3_X(3,4), OPF_X(4,7))), "fsubq", "34e"},
+	{(FORMAT3F(2, OP3_X(3,4), OPF_X(4,9))), "fmuls", "34e"},
+	{(FORMAT3F(2, OP3_X(3,4), OPF_X(4,10))), "fmuld", "34e"},
+	{(FORMAT3F(2, OP3_X(3,4), OPF_X(4,11))), "fmulq", "34e"},
+	{(FORMAT3F(2, OP3_X(3,4), OPF_X(4,13))), "fdivs", "34e"},
+	{(FORMAT3F(2, OP3_X(3,4), OPF_X(4,14))), "fdivd", "34e"},
+	{(FORMAT3F(2, OP3_X(3,4), OPF_X(4,15))), "fdivq", "34e"},
+
+	{(FORMAT3F(2, OP3_X(3,4), OPF_X(6,9))), "fsmuld", "34e"},
+	{(FORMAT3F(2, OP3_X(3,4), OPF_X(6,14))), "fdmulq", "34e"},
+
+	{(FORMAT3F(2, OP3_X(3,4), OPF_X(8,1))), "fstox", "4e"},
+	{(FORMAT3F(2, OP3_X(3,4), OPF_X(8,2))), "fdtox", "4e"},
+	{(FORMAT3F(2, OP3_X(3,4), OPF_X(8,3))), "fqtox", "4e"},
+	{(FORMAT3F(2, OP3_X(3,4), OPF_X(8,4))), "fxtos", "4e"},
+	{(FORMAT3F(2, OP3_X(3,4), OPF_X(8,8))), "fxtod", "4e"},
+	{(FORMAT3F(2, OP3_X(3,4), OPF_X(8,12))), "fxtoq", "4e"},
+
+	{(FORMAT3F(2, OP3_X(3,4), OPF_X(12,4))), "fitos", "4e"},
+	{(FORMAT3F(2, OP3_X(3,4), OPF_X(12,6))), "fdtos", "4e"},
+	{(FORMAT3F(2, OP3_X(3,4), OPF_X(12,7))), "fqtos", "4e"},
+	{(FORMAT3F(2, OP3_X(3,4), OPF_X(12,8))), "fitod", "4e"},
+	{(FORMAT3F(2, OP3_X(3,4), OPF_X(12,9))), "fstod", "4e"},
+	{(FORMAT3F(2, OP3_X(3,4), OPF_X(12,11))), "fqtod", "4e"},
+	{(FORMAT3F(2, OP3_X(3,4), OPF_X(12,12))), "fitoq", "4e"},
+	{(FORMAT3F(2, OP3_X(3,4), OPF_X(12,13))), "fstoq", "4e"},
+	{(FORMAT3F(2, OP3_X(3,4), OPF_X(12,14))), "fdtoq", "4e"},
+
+	{(FORMAT3F(2, OP3_X(3,4), OPF_X(13,1))), "fstoi", "4e"},
+	{(FORMAT3F(2, OP3_X(3,4), OPF_X(13,2))), "fdtoi", "4e"},
+	{(FORMAT3F(2, OP3_X(3,4), OPF_X(13,3))), "fqtoi", "4e"},
+
+
+#ifdef xxx
+	/*
+	 * OP3 =(3,5): FPop2 (table 35)
+	 */
+	{(FORMAT3F(2, OP3_X(3,5), 81)), "fcmps", "o34"},
+	{(FORMAT3F(2, OP3_X(3,5), 82)), "fcmpd", "o34"},
+	{(FORMAT3F(2, OP3_X(3,5), 83)), "fcmpq", "o34"},
+	{(FORMAT3F(2, OP3_X(3,5), 85)), "fcmpes", "o34"},
+	{(FORMAT3F(2, OP3_X(3,5), 86)), "fcmped", "o34"},
+	{(FORMAT3F(2, OP3_X(3,5), 87)), "fcmpeq", "o34"},
+
+	/* Move Floating-Point Register on Condition "FMOVcc" */
+	/* FIXME should check for single, double, and quad movements */
+	/* Integer Condition Codes */
+	{(FORMAT3(2, OP3_X(3,5), 0) | COND2(0,8)), "fmova", "04e"},
+	{(FORMAT3(2, OP3_X(3,5), 0) | COND2(0,0)), "fmovn", "04e"},
+	{(FORMAT3(2, OP3_X(3,5), 0) | COND2(0,9)), "fmovne", "04e"},
+	{(FORMAT3(2, OP3_X(3,5), 0) | COND2(0,1)), "fmove", "04e"},
+	{(FORMAT3(2, OP3_X(3,5), 0) | COND2(0,10)), "fmovg", "04e"},
+	{(FORMAT3(2, OP3_X(3,5), 0) | COND2(0,2)), "fmovle", "04e"},
+	{(FORMAT3(2, OP3_X(3,5), 0) | COND2(0,11)), "fmovge", "04e"},
+	{(FORMAT3(2, OP3_X(3,5), 0) | COND2(0,3)), "fmovl", "04e"},
+	{(FORMAT3(2, OP3_X(3,5), 0) | COND2(0,12)), "fmovgu", "04e"},
+	{(FORMAT3(2, OP3_X(3,5), 0) | COND2(0,4)), "fmovleu", "04e"},
+	{(FORMAT3(2, OP3_X(3,5), 0) | COND2(0,13)), "fmovcc", "04e"},
+	{(FORMAT3(2, OP3_X(3,5), 0) | COND2(0,5)), "fmovcs", "04e"},
+	{(FORMAT3(2, OP3_X(3,5), 0) | COND2(0,14)), "fmovpos", "04e"},
+	{(FORMAT3(2, OP3_X(3,5), 0) | COND2(0,6)), "fmovneg", "04e"},
+	{(FORMAT3(2, OP3_X(3,5), 0) | COND2(0,15)), "fmovvc", "04e"},
+	{(FORMAT3(2, OP3_X(3,5), 0) | COND2(0,7)), "fmovvs", "04e"},
+
+	/* Floating-Point Condition Codes */
+	{(FORMAT3(2, OP3_X(3,5), 0) | COND2(0,8)), "fmova", "o4e"},
+	{(FORMAT3(2, OP3_X(3,5), 0) | COND2(0,0)), "fmovn", "o4e"},
+	{(FORMAT3(2, OP3_X(3,5), 0) | COND2(0,7)), "fmovu", "o4e"},
+	{(FORMAT3(2, OP3_X(3,5), 0) | COND2(0,6)), "fmovg", "o4e"},
+	{(FORMAT3(2, OP3_X(3,5), 0) | COND2(0,5)), "fmovug", "o4e"},
+	{(FORMAT3(2, OP3_X(3,5), 0) | COND2(0,4)), "fmovk", "o4e"},
+	{(FORMAT3(2, OP3_X(3,5), 0) | COND2(0,3)), "fmovul", "o4e"},
+	{(FORMAT3(2, OP3_X(3,5), 0) | COND2(0,2)), "fmovlg", "o4e"},
+	{(FORMAT3(2, OP3_X(3,5), 0) | COND2(0,1)), "fmovne", "o4e"},
+	{(FORMAT3(2, OP3_X(3,5), 0) | COND2(0,9)), "fmove", "o4e"},
+	{(FORMAT3(2, OP3_X(3,5), 0) | COND2(0,10)), "fmovue", "o4e"},
+	{(FORMAT3(2, OP3_X(3,5), 0) | COND2(0,11)), "fmovge", "o4e"},
+	{(FORMAT3(2, OP3_X(3,5), 0) | COND2(0,12)), "fmovuge", "o4e"},
+	{(FORMAT3(2, OP3_X(3,5), 0) | COND2(0,13)), "fmovle", "o4e"},
+	{(FORMAT3(2, OP3_X(3,5), 0) | COND2(0,14)), "fmovule", "o4e"},
+	{(FORMAT3(2, OP3_X(3,5), 0) | COND2(0,15)), "fmovo", "o4e"},
+
+	/* Move F-P Register on Integer Register Condition "FMOVr" */
+	/* FIXME: check for short, double, and quad's */
+	{(FORMAT3(2, OP3_X(3,5), 0) | EIF_F3_RCOND(1)), "fmovre", "14e"},
+	{(FORMAT3(2, OP3_X(3,5), 0) | EIF_F3_RCOND(2)), "fmovrlez", "14e"},
+	{(FORMAT3(2, OP3_X(3,5), 0) | EIF_F3_RCOND(3)), "fmovrlz", "14e"},
+	{(FORMAT3(2, OP3_X(3,5), 0) | EIF_F3_RCOND(5)), "fmovrne", "14e"},
+	{(FORMAT3(2, OP3_X(3,5), 0) | EIF_F3_RCOND(6)), "fmovrgz", "14e"},
+	{(FORMAT3(2, OP3_X(3,5), 0) | EIF_F3_RCOND(7)), "fmovrgez", "14e"},
+#endif
+	/* FP logical insns -- UltraSPARC extens */
+	{(FORMAT3F(2, OP3_X(3,6), OPF_X(6,0))), "fzero", "e"},
+	{(FORMAT3F(2, OP3_X(3,6), OPF_X(6,1))), "fzeros", "e"},
+	{(FORMAT3F(2, OP3_X(3,6), OPF_X(7,14))), "fone", "e"},
+	{(FORMAT3F(2, OP3_X(3,6), OPF_X(7,15))), "fones", "e"},
+	{(FORMAT3F(2, OP3_X(3,6), OPF_X(7,4))), "fsrc1", "3e"},
+	{(FORMAT3F(2, OP3_X(3,6), OPF_X(7,5))), "fsrc1s", "3e"},
+	{(FORMAT3F(2, OP3_X(3,6), OPF_X(7,8))), "fsrc2", "4e"},
+	{(FORMAT3F(2, OP3_X(3,6), OPF_X(7,9))), "fsrc2s", "4e"},
+	{(FORMAT3F(2, OP3_X(3,6), OPF_X(6,10))), "fnot1", "3e"},
+	{(FORMAT3F(2, OP3_X(3,6), OPF_X(6,11))), "fnot1s", "3e"},
+	{(FORMAT3F(2, OP3_X(3,6), OPF_X(6,6))), "fnot2", "4e"},
+	{(FORMAT3F(2, OP3_X(3,6), OPF_X(6,7))), "fnot2s", "4e"},
+	{(FORMAT3F(2, OP3_X(3,6), OPF_X(7,12))), "for", "34e"},
+	{(FORMAT3F(2, OP3_X(3,6), OPF_X(7,13))), "fors", "34e"},
+	{(FORMAT3F(2, OP3_X(3,6), OPF_X(6,2))), "fnor", "34e"},
+	{(FORMAT3F(2, OP3_X(3,6), OPF_X(6,3))), "fnors", "34e"},
+	{(FORMAT3F(2, OP3_X(3,6), OPF_X(7,0))), "fand", "34e"},
+	{(FORMAT3F(2, OP3_X(3,6), OPF_X(7,1))), "fands", "34e"},
+	{(FORMAT3F(2, OP3_X(3,6), OPF_X(6,14))), "fnand", "34e"},
+	{(FORMAT3F(2, OP3_X(3,6), OPF_X(6,15))), "fnands", "34e"},
+	{(FORMAT3F(2, OP3_X(3,6), OPF_X(6,12))), "fxor", "34e"},
+	{(FORMAT3F(2, OP3_X(3,6), OPF_X(6,13))), "fxors", "34e"},
+	{(FORMAT3F(2, OP3_X(3,6), OPF_X(7,2))), "fxnor", "34e"},
+	{(FORMAT3F(2, OP3_X(3,6), OPF_X(7,3))), "fxnors", "34e"},
+	{(FORMAT3F(2, OP3_X(3,6), OPF_X(7,10))), "fornot1", "34e"},
+	{(FORMAT3F(2, OP3_X(3,6), OPF_X(7,11))), "fornot1s", "34e"},
+	{(FORMAT3F(2, OP3_X(3,6), OPF_X(7,6))), "fornot2", "34e"},
+	{(FORMAT3F(2, OP3_X(3,6), OPF_X(7,7))), "fornot2s", "34e"},
+	{(FORMAT3F(2, OP3_X(3,6), OPF_X(6,8))), "fandnot1", "34e"},
+	{(FORMAT3F(2, OP3_X(3,6), OPF_X(6,9))), "fandnot1s", "34e"},
+	{(FORMAT3F(2, OP3_X(3,6), OPF_X(6,4))), "fandnot2", "34e"},
+	{(FORMAT3F(2, OP3_X(3,6), OPF_X(6,5))), "fandnot2s", "34e"},
+
+	/* grrrr.... */
+	{0, 0, 0}
+
+};
+
+db_addr_t
+db_disasm(db_addr_t loc, boolean_t altfmt)
+{
+	struct sparc_insn*	i_ptr = (struct sparc_insn *)&sparc_i;
+
+	unsigned int insn, you_lose, bitmask;
+	int matchp;
+	char* f_ptr, *cp;
+
+	you_lose = 0;
+	matchp = 0;
+	insn = db_get_value(loc, 4, 0);
+
+	if (insn == 0x01000000) {
+		db_printf("nop\n");
+		return loc + 4;
+	}
+
+	while (i_ptr->name) {
+		/* calculate YOU_LOSE value */
+		bitmask= (i_ptr->match);
+		you_lose = (~bitmask);
+
+		if (((bitmask>>30) & 0x3) == 0x1) {
+			/* Call */
+			you_lose = ((~0x1)<<30);
+		} else if (((bitmask>>30) & 0x3) == 0x0) {
+			if (((bitmask>>22) & 0x7) == 0x4) {
+				/* Sethi */
+				you_lose &= (FORMAT2(0x3,0x7));
+			} else {
+				/* Branches */
+				you_lose &= (FORMAT2(0x3,0x7) |
+				    EIF_F2_COND(0xf));
+			}
+		} else if (((bitmask>>30) & 0x3) == 0x2 &&
+			   ((bitmask>>19) & 0x3f) == 0x34) /* XXX */ {
+			/* FPop1 */
+			you_lose &= (FORMAT3(0x3,0x3f,0x1) |
+			    EIF_F3_OPF(0x1ff));
+		} else if (((bitmask>>30) & 0x3) == 0x2 &&
+			   ((bitmask>>19) & 0x3f) == 0x3a) /* XXX */ {
+			/* Tcc */
+			you_lose &= (FORMAT3(0x3,0x3f,0x1) | EIF_F4_TCOND(0xf));
+		} else if (((bitmask>>30) & 0x3) == 0x2 &&
+			   ((bitmask>>21) & 0xf) == 0x9 &&
+			   ((bitmask>>19) & 0x3) != 0) /* XXX */ {
+			/* shifts */
+			you_lose &= (FORMAT3(0x3,0x3f,0x1)) | EIF_F3_X(1);
+		} else if (((bitmask>>30) & 0x3) == 0x2 &&
+			   ((bitmask>>19) & 0x3f) == 0x2c) /* XXX */ {
+			/* cmov */
+			you_lose &= (FORMAT3(0x3,0x3f,0x1) | COND2(1,0xf));
+		} else if (((bitmask>>30) & 0x3) == 0x2 &&
+			   ((bitmask>>19) & 0x3f) == 0x35) /* XXX */ {
+			/* fmov */
+			you_lose &= (FORMAT3(0x3,0x3f,0x1) | COND2(1,0xf));
+		} else {
+			you_lose &= (FORMAT3(0x3,0x3f,0x1));
+		}
+
+		if (((bitmask & insn) == bitmask) && ((you_lose & insn) == 0)) {
+			matchp = 1;
+			break;
+		}
+		i_ptr++;
+	};
+
+	if (!matchp) {
+		db_printf("undefined\n");
+		return loc + 4;
+	}
+
+	db_printf("%s", i_ptr->name);
+
+	f_ptr = i_ptr->format;
+
+	for (cp = f_ptr; *cp; cp++) {
+		if (*cp == ',') {
+			for (;f_ptr < cp; f_ptr++)
+				switch (*f_ptr) {
+				case 'a':
+					if (insn & EIF_F2_A(1))
+						db_printf(",a");
+					break;
+				case 'p':
+					if (insn & EIF_F2_P(1))
+						db_printf(",pt");
+					else
+						db_printf(",pn");
+					break;
+				}
+			f_ptr++;
+			break;
+		}
+	}
+	db_printf("      \t");
+
+	while (*f_ptr) {
+		switch (*f_ptr) {
+			int64_t val;
+		case '1':
+			db_printf("%%%s", regs[((insn >> 14) & 0x1f)]);
+			break;
+		case '2':
+			db_printf("%%%s", regs[(insn & 0x1f)]);
+			break;
+		case 'd':
+			db_printf("%%%s", regs[((insn >> 25) & 0x1f)]);
+			break;
+		case '3':
+			db_printf("%%f%d", ((insn >> 14) & 0x1f));
+			break;
+		case '4':
+			db_printf("%%f%d", (insn & 0x1f));
+			break;
+		case 'e':
+			db_printf("%%f%d", ((insn >> 25) & 0x1f));
+			break;
+		case 'i':
+			/* simm13 -- signed */
+			val = IF_SIMM(insn, 13);
+			db_printf("%s0x%x", SIGN(val), (int)abs(val));
+			break;
+		case 'j':
+			/* simm11 -- signed */
+			val = IF_SIMM(insn, 11);
+			db_printf("%s0x%x", SIGN(val), (int)abs(val));
+			break;
+		case 'l':
+			val = (((insn>>20)&0x3)<<13)|(insn & 0x1fff);
+			val = IF_SIMM(val, 16);
+			db_printsym((db_addr_t)(loc + (4 * val)), DB_STGY_ANY);
+			break;
+		case 'm':
+			db_printsym((db_addr_t)(loc + (4 * IF_SIMM(insn, 22))),
+				DB_STGY_ANY);
+			break;
+		case 'u':
+			db_printsym((db_addr_t)(loc + (4 * IF_SIMM(insn, 19))),
+			    DB_STGY_ANY);
+			break;
+		case 'n':
+			db_printsym((db_addr_t)(loc + (4 * IF_SIMM(insn, 30))),
+			    DB_STGY_PROC);
+			break;
+		case 's':
+			db_printf("%%asi");
+			break;
+		case 't':
+			db_printf("0x%-2.2x", ((insn >> 5) & 0xff));
+			break;
+		case 'o':
+			db_printf("%%fcc%d", ((insn >> 25) & 0x3));
+			break;
+		case 'p':
+		case '7':
+			db_printf("[%%%s + %%%s]",
+				  regs[((insn >> 14) & 0x1f)],
+				  regs[(insn & 0x1f)]);
+			if (*f_ptr == '7')
+				db_printf(" %d", ((insn >> 5) & 0xff));
+			break;
+		case 'q':
+		case '8':
+			val = IF_SIMM(insn, 13);
+			db_printf("[%%%s %c 0x%x]",
+				regs[((insn >> 14) & 0x1f)],
+				(int)((val<0)?'-':'+'),
+				(int)abs(val));
+			if (*f_ptr == '8')
+				db_printf(" %%asi");
+			break;
+		case '5':
+			db_printf("%%fsr");
+			break;
+		case '6':
+			db_printf("%%fsr");
+			break;
+		case '9':
+			db_printf("0x%xl",
+				  ((insn & 0xf) | ((insn >> 4) & 0x7)));
+			break;
+		case '0':
+			db_printf("%%%s", ccodes[((insn >> 11) & 0x3) + 4]);
+			break;
+		case '.':
+			db_printf("%%%s", ccodes[((insn >> 11) & 0x7)]);
+			break;
+		case 'r':
+			db_printf("#%s", prefetch[((insn >> 25) & 0x1f)]);
+			break;
+		case 'A':
+			db_printf("%%%s", priv_regs[((insn >> 14) & 0x1f)]);
+			break;
+		case 'B':
+			db_printf("%%%s", state_regs[((insn >> 14) & 0x1f)]);
+			break;
+		case 'C':
+			db_printf("%%hi(0x%x)", ((insn & 0x3fffff) << 10));
+			break;
+		case 'D':
+			db_printf("0x%x", (insn & 0x1f));
+			break;
+		case 'E':
+			db_printf("%d", (insn & 0x3f));
+			break;
+		case 'F':
+			db_printf("%d", (insn & 0x3f));
+			break;
+		case 'G':
+			db_printf("%%%s", priv_regs[((insn >> 25) & 0x1f)]);
+			break;
+		case 'H':
+			db_printf("%%%s", state_regs[((insn >> 25) & 0x1f)]);
+			break;
+		default:
+			db_printf("(UNKNOWN)");
+			break;
+		}
+		if (*(++f_ptr))
+			db_printf(", ");
+	};
+
+	db_printf("\n");
+
+	return (loc + 4);
+}
+
diff --git a/sys/sun4v/sun4v/db_hwwatch.c b/sys/sun4v/sun4v/db_hwwatch.c
new file mode 100644
index 000000000000..0a878ca0b9dd
--- /dev/null
+++ b/sys/sun4v/sun4v/db_hwwatch.c
@@ -0,0 +1,229 @@
+/*-
+ * Copyright (c) 2001 Jake Burkholder.
+ * All rights reserved.
+ * Copyright 2001 by Thomas Moestl <tmm@FreeBSD.org>.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <sys/param.h>
+
+#include <vm/vm.h>
+
+#include <machine/asi.h>
+#include <machine/cpufunc.h>
+#include <machine/lsu.h>
+#include <machine/watch.h>
+
+#include <ddb/ddb.h>
+#include <ddb/db_access.h>
+#include <ddb/db_sym.h>
+#include <ddb/db_variables.h>
+#include <ddb/db_watch.h>
+#if 0
+static void db_watch_print(vm_offset_t wp, int bm);
+
+int
+watch_phys_set_mask(vm_paddr_t pa, u_long mask)
+{
+	u_long lsucr;
+
+	stxa_sync(AA_DMMU_PWPR, ASI_DMMU, pa & (((2UL << 38) - 1) << 3));
+	lsucr = ldxa(0, ASI_LSU_CTL_REG);
+	lsucr = ((lsucr | LSU_PW) & ~LSU_PM_MASK) |
+	    (mask << LSU_PM_SHIFT);
+	stxa_sync(0, ASI_LSU_CTL_REG, lsucr);
+	return (0);
+}
+
+int
+watch_phys_set(vm_paddr_t pa, int sz)
+{
+	u_long off;
+
+	off = (u_long)pa & 7;
+	/* Test for misaligned watch points. */
+	if (off + sz > 8)
+		return (-1);
+	return (watch_phys_set_mask(pa, ((1 << sz) - 1) << off));
+}
+
+vm_paddr_t
+watch_phys_get(int *bm)
+{
+	vm_paddr_t pa;
+	u_long lsucr;
+	
+	if (!watch_phys_active())
+		return (0);
+
+	pa = ldxa(AA_DMMU_PWPR, ASI_DMMU);
+	lsucr = ldxa(0, ASI_LSU_CTL_REG);
+	*bm = (lsucr & LSU_PM_MASK) >> LSU_PM_SHIFT;
+	
+	return (pa);
+}
+
+void
+watch_phys_clear()
+{
+	stxa_sync(0, ASI_LSU_CTL_REG,
+	    ldxa(0, ASI_LSU_CTL_REG) & ~LSU_PW);
+}
+
+int
+watch_phys_active()
+{
+
+	return (ldxa(0, ASI_LSU_CTL_REG) & LSU_PW);
+}
+
+int
+watch_virt_set_mask(vm_offset_t va, u_long mask)
+{
+	u_long lsucr;
+
+	stxa_sync(AA_DMMU_VWPR, ASI_DMMU, va & (((2UL << 41) - 1) << 3));
+	lsucr = ldxa(0, ASI_LSU_CTL_REG);
+	lsucr = ((lsucr | LSU_VW) & ~LSU_VM_MASK) |
+	    (mask << LSU_VM_SHIFT);
+	stxa_sync(0, ASI_LSU_CTL_REG, lsucr);
+	return (0);
+}
+
+int
+watch_virt_set(vm_offset_t va, int sz)
+{
+	u_long off;
+
+	off = (u_long)va & 7;
+	/* Test for misaligned watch points. */
+	if (off + sz > 8)
+		return (-1);
+	return (watch_virt_set_mask(va, ((1 << sz) - 1) << off));
+}
+
+vm_offset_t
+watch_virt_get(int *bm)
+{
+	u_long va;
+	u_long lsucr;
+	
+	if (!watch_virt_active())
+		return (0);
+
+	va = ldxa(AA_DMMU_VWPR, ASI_DMMU);
+	lsucr = ldxa(0, ASI_LSU_CTL_REG);
+	*bm = (lsucr & LSU_VM_MASK) >> LSU_VM_SHIFT;
+	
+	return ((vm_offset_t)va);
+}
+
+void
+watch_virt_clear()
+{
+	stxa_sync(0, ASI_LSU_CTL_REG,
+	    ldxa(0, ASI_LSU_CTL_REG) & ~LSU_VW);
+}
+
+int
+watch_virt_active()
+{
+
+	return (ldxa(0, ASI_LSU_CTL_REG) & LSU_VW);
+}
+
+int
+db_md_set_watchpoint(db_expr_t addr, db_expr_t size)
+{
+	int dummy;
+
+	if (watch_virt_active()) {
+		db_printf("Overwriting previously active watch point at "
+		    "0x%lx\n", watch_virt_get(&dummy));
+	}
+	return (watch_virt_set(addr, size));
+}
+
+int
+db_md_clr_watchpoint(db_expr_t addr, db_expr_t size)
+{
+
+	watch_virt_clear();
+	return (0);
+}
+
+void
+db_watch_print(vm_offset_t wp, int bm)
+{
+	int i;
+
+	db_printf("\tat 0x%lx, active bytes: ", (u_long)wp);
+	for (i = 0; i < 8; i++) {
+		if ((bm & (1 << i)) != 0)
+			db_printf("%d ", i);
+	}
+	if (bm == 0)
+		db_printf("none");
+	db_printf("\n");
+}
+
+void
+db_md_list_watchpoints(void)
+{
+	vm_offset_t va;
+	vm_paddr_t pa;
+	int bm;
+
+	db_printf("Physical address watchpoint:\n");
+	if (watch_phys_active()) {
+		pa = watch_phys_get(&bm);
+		db_watch_print(pa, bm);
+	} else
+		db_printf("\tnot active.\n");
+	db_printf("Virtual address watchpoint:\n");
+	if (watch_virt_active()) {
+		va = watch_virt_get(&bm);
+		db_watch_print(va, bm);
+	} else
+		db_printf("\tnot active.\n");
+}
+#endif
+void
+db_md_list_watchpoints(void)
+{
+}
+
+int
+db_md_clr_watchpoint(db_expr_t addr, db_expr_t size)
+{
+
+	return (0);
+}
+
+int
+db_md_set_watchpoint(db_expr_t addr, db_expr_t size)
+{
+	return (0);
+}
diff --git a/sys/sun4v/sun4v/db_interface.c b/sys/sun4v/sun4v/db_interface.c
new file mode 100644
index 000000000000..77e1493652eb
--- /dev/null
+++ b/sys/sun4v/sun4v/db_interface.c
@@ -0,0 +1,106 @@
+/*-
+ * Copyright (c) 2001 Jake Burkholder.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <sys/param.h> 
+#include <sys/systm.h>
+#include <sys/reboot.h>
+#include <sys/cons.h>
+#include <sys/kdb.h>
+#include <sys/ktr.h>
+#include <sys/linker_set.h>
+#include <sys/lock.h>
+#include <sys/pcpu.h>
+#include <sys/proc.h>
+#include <sys/smp.h>
+
+#include <machine/cpu.h>
+#include <machine/md_var.h>
+
+#include <vm/vm.h>
+#include <vm/pmap.h>
+
+#include <ddb/ddb.h>
+#include <ddb/db_access.h>
+#include <ddb/db_sym.h>
+#include <ddb/db_variables.h>
+
+#include <machine/atomic.h>
+#include <machine/setjmp.h>
+
+int
+db_read_bytes(vm_offset_t addr, size_t size, char *data)
+{
+	jmp_buf jb;
+	void *prev_jb;
+	char *src;
+	int ret;
+
+	prev_jb = kdb_jmpbuf(jb);
+	ret = setjmp(jb);
+	if (ret == 0) {
+		src = (char *)addr;
+		while (size-- > 0)
+			*data++ = *src++;
+	}
+	(void)kdb_jmpbuf(prev_jb);
+	return (ret);
+}
+
+int
+db_write_bytes(vm_offset_t addr, size_t size, char *data)
+{
+	jmp_buf jb;
+	void *prev_jb;
+	char *dst;
+	int ret;
+
+	prev_jb = kdb_jmpbuf(jb);
+	ret = setjmp(jb);
+	if (ret == 0) {
+		dst = (char *)addr;
+		while (size-- > 0)
+			*dst++ = *data++;
+	}
+	(void)kdb_jmpbuf(prev_jb);
+	return (ret);
+}
+
+void
+db_show_mdpcpu(struct pcpu *pc)
+{
+}
+
+DB_COMMAND(reboot, db_reboot)
+{
+	cpu_reset();
+}
+
+DB_COMMAND(halt, db_halt)
+{
+	cpu_halt();
+}
diff --git a/sys/sun4v/sun4v/db_trace.c b/sys/sun4v/sun4v/db_trace.c
new file mode 100644
index 000000000000..5c07ecc04bca
--- /dev/null
+++ b/sys/sun4v/sun4v/db_trace.c
@@ -0,0 +1,326 @@
+/*-
+ * Copyright (c) 2001 Jake Burkholder.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/kdb.h>
+#include <sys/linker_set.h>
+#include <sys/proc.h>
+#include <sys/stack.h>
+#include <sys/sysent.h>
+
+#include <vm/vm.h>
+#include <vm/vm_page.h>
+#include <vm/vm_map.h>
+
+#include <machine/cpu.h>
+#include <machine/pcb.h>
+#include <machine/trap.h>
+#include <machine/vmparam.h>
+
+#include <ddb/ddb.h>
+#include <ddb/db_access.h>
+#include <ddb/db_sym.h>
+#include <ddb/db_variables.h>
+#include <ddb/db_watch.h>
+
+#define	INKERNEL(va) \
+	((va) >= VM_MIN_KERNEL_ADDRESS && (va) <= VM_MAX_KERNEL_ADDRESS)
+
+static db_varfcn_t db_frame;
+
+#define	DB_OFFSET(x)	(db_expr_t *)offsetof(struct trapframe, x)
+struct	db_variable db_regs[] = {
+	{ "g0",		DB_OFFSET(tf_global[0]),	db_frame },
+	{ "g1",		DB_OFFSET(tf_global[1]),	db_frame },
+	{ "g2",		DB_OFFSET(tf_global[2]),	db_frame },
+	{ "g3",		DB_OFFSET(tf_global[3]),	db_frame },
+	{ "g4",		DB_OFFSET(tf_global[4]),	db_frame },
+	{ "g5",		DB_OFFSET(tf_global[5]),	db_frame },
+	{ "g6",		DB_OFFSET(tf_global[6]),	db_frame },
+	{ "g7",		DB_OFFSET(tf_global[7]),	db_frame },
+	{ "i0",		DB_OFFSET(tf_out[0]),		db_frame },
+	{ "i1",		DB_OFFSET(tf_out[1]),		db_frame },
+	{ "i2",		DB_OFFSET(tf_out[2]),		db_frame },
+	{ "i3",		DB_OFFSET(tf_out[3]),		db_frame },
+	{ "i4",		DB_OFFSET(tf_out[4]),		db_frame },
+	{ "i5",		DB_OFFSET(tf_out[5]),		db_frame },
+	{ "i6",		DB_OFFSET(tf_out[6]),		db_frame },
+	{ "i7",		DB_OFFSET(tf_out[7]),		db_frame },
+	{ "tnpc",	DB_OFFSET(tf_tnpc),		db_frame },
+	{ "tpc",	DB_OFFSET(tf_tpc),		db_frame },
+	{ "tstate",	DB_OFFSET(tf_tstate),		db_frame },
+};
+struct db_variable *db_eregs = db_regs + sizeof(db_regs)/sizeof(db_regs[0]);
+
+static int
+db_frame(struct db_variable *vp, db_expr_t *valuep, int op)
+{
+	uint64_t *reg;
+
+	if (kdb_frame == NULL)
+		return (0);
+	reg = (uint64_t*)((uintptr_t)kdb_frame + (uintptr_t)vp->valuep);
+	if (op == DB_VAR_GET)
+		*valuep = *reg;
+	else
+		*reg = *valuep;
+	return (1);
+}
+
+/*
+ * User stack trace (debugging aid).
+ */
+static void
+db_utrace(struct thread *td, struct trapframe *tf, int count)
+{
+	struct pcb *pcb;
+	db_addr_t sp, rsp, o7, pc;
+	int i, found;
+
+	pcb = td->td_pcb;
+	sp = db_get_value((db_addr_t)&tf->tf_sp, sizeof(tf->tf_sp), FALSE);
+	o7 = db_get_value((db_addr_t)&tf->tf_out[7], sizeof(tf->tf_out[7]),
+	    FALSE);
+	pc = db_get_value((db_addr_t)&tf->tf_tpc, sizeof(tf->tf_tpc), FALSE);
+	db_printf("user trace: trap %%o7=%#lx\n", o7);
+	while (count-- && sp != 0 && !db_pager_quit) {
+		db_printf("pc %#lx, sp %#lx\n", pc, sp);
+		/* First, check whether the frame is in the pcb. */
+		found = 0;
+		for (i = 0; i < pcb->pcb_nsaved; i++) {
+			if (pcb->pcb_rwsp[i] == sp) {
+				found = 1;
+				sp = pcb->pcb_rw[i].rw_in[6];
+				pc = pcb->pcb_rw[i].rw_in[7];
+				break;
+			}
+		}
+		if (!found) {
+			rsp = sp + SPOFF;
+			sp = 0;
+			if (copyin((void *)(rsp + offsetof(struct frame, fr_fp)),
+			    &sp, sizeof(sp)) != 0 ||
+			    copyin((void *)(rsp + offsetof(struct frame, fr_pc)),
+			    &pc, sizeof(pc)) != 0)
+				break;
+		}
+	}
+	db_printf("done\n");
+}
+
+static int
+db_print_trap(struct thread *td, struct trapframe *tf, int count)
+{
+	struct proc *p;
+	const char *symname;
+	c_db_sym_t sym;
+	db_expr_t diff;
+	db_addr_t func;
+	db_addr_t tpc;
+	u_long type;
+#if 0
+	u_long sfar;
+	u_long sfsr;
+	u_long tar;
+	u_long level;
+	u_long pil;
+#endif
+	u_long code;
+	u_long o7;
+	int user;
+
+	type = 5;
+	p = td->td_proc;
+#if 0
+	type = db_get_value((db_addr_t)&tf->tf_type,
+	    sizeof(tf->tf_type), FALSE);
+	db_printf("-- %s", trap_msg[type & ~T_KERNEL]);
+#endif
+	switch (type & ~T_KERNEL) {
+	case T_DATA_PROTECTION:
+#if 0
+		tar = (u_long)db_get_value((db_addr_t)&tf->tf_tar,
+		    sizeof(tf->tf_tar), FALSE);
+		db_printf(" tar=%#lx", tar);
+#endif
+		/* fall through */
+	case T_DATA_EXCEPTION:
+	case T_INSTRUCTION_EXCEPTION:
+	case T_MEM_ADDRESS_NOT_ALIGNED:
+#if 0
+		sfar = (u_long)db_get_value((db_addr_t)&tf->tf_sfar,
+		    sizeof(tf->tf_sfar), FALSE);
+		sfsr = (u_long)db_get_value((db_addr_t)&tf->tf_sfsr,
+		    sizeof(tf->tf_sfsr), FALSE);
+		db_printf(" sfar=%#lx sfsr=%#lx", sfar, sfsr);
+#endif
+		break;
+	case T_DATA_MISS:
+	case T_INSTRUCTION_MISS:
+#if 0
+		tar = (u_long)db_get_value((db_addr_t)&tf->tf_tar,
+		    sizeof(tf->tf_tar), FALSE);
+		db_printf(" tar=%#lx", tar);
+#endif
+		break;
+	case T_SYSCALL:
+		code = db_get_value((db_addr_t)&tf->tf_global[1],
+		    sizeof(tf->tf_global[1]), FALSE);
+		db_printf(" (%ld", code);
+		if (code >= 0 && code < p->p_sysent->sv_size) {
+			func = (db_addr_t)p->p_sysent->sv_table[code].sy_call;
+			sym = db_search_symbol(func, DB_STGY_ANY, &diff);
+			if (sym != DB_SYM_NULL && diff == 0) {
+				db_symbol_values(sym, &symname, NULL);
+				db_printf(", %s, %s", p->p_sysent->sv_name,
+				    symname);
+			}
+			db_printf(")");
+		}
+		break;
+	case T_INTERRUPT:
+#if 0
+		level = (u_long)db_get_value((db_addr_t)&tf->tf_level,
+		    sizeof(tf->tf_level), FALSE);
+		pil = (u_long)db_get_value((db_addr_t)&tf->tf_pil,
+		    sizeof(tf->tf_pil), FALSE);
+		db_printf(" level=%#lx pil=%#lx", level, pil);
+#endif
+		break;
+	default:
+		break;
+	}
+	o7 = (u_long)db_get_value((db_addr_t)&tf->tf_out[7],
+	    sizeof(tf->tf_out[7]), FALSE);
+	db_printf(" %%o7=%#lx --\n", o7);
+	user = (type & T_KERNEL) == 0;
+	if (user) {
+		tpc = db_get_value((db_addr_t)&tf->tf_tpc,
+		    sizeof(tf->tf_tpc), FALSE);
+		db_printf("userland() at ");
+		db_printsym(tpc, DB_STGY_PROC);
+		db_printf("\n");
+		db_utrace(td, tf, count);
+	}
+	return (user);
+}
+
+static int
+db_backtrace(struct thread *td, struct frame *fp, int count)
+{
+	struct trapframe *tf;
+	const char *name;
+	c_db_sym_t sym;
+	db_expr_t offset;
+	db_expr_t value;
+	db_addr_t npc;
+	db_addr_t pc;
+	int trap;
+	int user;
+	int quit;
+
+	if (count == -1)
+		count = 1024;
+
+	trap = 0;
+	user = 0;
+	npc = 0;
+	quit = 0;
+	while (count-- && !user && !db_pager_quit) {
+		pc = (db_addr_t)db_get_value((db_addr_t)&fp->fr_pc,
+		    sizeof(fp->fr_pc), FALSE);
+		if (trap) {
+			pc = npc;
+			trap = 0;
+		}
+		if (!INKERNEL((vm_offset_t)pc))
+			break;
+		sym = db_search_symbol(pc, DB_STGY_ANY, &offset);
+		if (sym == C_DB_SYM_NULL) {
+			value = 0;
+			name = NULL;
+		} else
+			db_symbol_values(sym, &name, &value);
+		if (name == NULL)
+			name = "(null)";
+		fp = (struct frame *)(db_get_value((db_addr_t)&fp->fr_fp,
+		   sizeof(fp->fr_fp), FALSE) + SPOFF);
+		if (bcmp(name, "tl0_", 4) == 0 ||
+		    bcmp(name, "tl1_", 4) == 0) {
+			tf = (struct trapframe *)(fp + 1);
+			npc = db_get_value((db_addr_t)&tf->tf_tpc,
+			    sizeof(tf->tf_tpc), FALSE);
+			user = db_print_trap(td, tf, count);
+			trap = 1;
+		} else {
+			db_printf("%s() at ", name);
+			db_printsym(pc, DB_STGY_PROC);
+			db_printf("\n");
+		}
+	}
+	return (0);
+}
+
+void
+db_trace_self(void)
+{
+	db_expr_t addr;
+
+	addr = (db_expr_t)__builtin_frame_address(1);
+	db_backtrace(curthread, (struct frame *)(addr + SPOFF), -1);
+}
+
+int
+db_trace_thread(struct thread *td, int count)
+{
+	struct pcb *ctx;
+
+	ctx = kdb_thr_ctx(td);
+	return (db_backtrace(td, (struct frame*)(ctx->pcb_sp + SPOFF), count));
+}
+
+void
+stack_save(struct stack *st)
+{
+	struct frame *fp;
+	db_expr_t addr;
+	vm_offset_t callpc;
+
+	stack_zero(st);
+	addr = (db_expr_t)__builtin_frame_address(1);
+	fp = (struct frame *)(addr + SPOFF);
+	while (1) {
+		callpc = fp->fr_pc;
+		if (!INKERNEL(callpc))
+			break;
+		if (stack_put(st, callpc) == -1)
+			break;
+		fp = (struct frame *)(fp->fr_fp + SPOFF);
+	}
+}
diff --git a/sys/sun4v/sun4v/dump_machdep.c b/sys/sun4v/sun4v/dump_machdep.c
new file mode 100644
index 000000000000..9082cbbf5ad0
--- /dev/null
+++ b/sys/sun4v/sun4v/dump_machdep.c
@@ -0,0 +1,249 @@
+/*-
+ * Copyright (c) 2002 Marcel Moolenaar
+ * Copyright (c) 2002 Thomas Moestl
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/conf.h>
+#include <sys/cons.h>
+#include <sys/kernel.h>
+#include <sys/kerneldump.h>
+
+#include <vm/vm.h>
+#include <vm/vm_param.h>
+#include <vm/pmap.h>
+
+#include <machine/metadata.h>
+#include <machine/kerneldump.h>
+#include <machine/ofw_mem.h>
+#include <machine/tte.h>
+#include <machine/tsb.h>
+
+
+CTASSERT(sizeof(struct kerneldumpheader) == DEV_BSIZE);
+
+static struct kerneldumpheader kdh;
+static off_t dumplo, dumppos;
+
+/* Handle buffered writes. */
+static char buffer[DEV_BSIZE];
+static vm_size_t fragsz;
+
+#define	MAXDUMPSZ	(MAXDUMPPGS << PAGE_SHIFT)
+
+/* XXX should be MI */
+static void
+mkdumpheader(struct kerneldumpheader *kdh, uint32_t archver, uint64_t dumplen,
+    uint32_t blksz)
+{
+
+	bzero(kdh, sizeof(*kdh));
+	strncpy(kdh->magic, KERNELDUMPMAGIC, sizeof(kdh->magic));
+	strncpy(kdh->architecture, MACHINE_ARCH, sizeof(kdh->architecture));
+	kdh->version = htod32(KERNELDUMPVERSION);
+	kdh->architectureversion = htod32(archver);
+	kdh->dumplength = htod64(dumplen);
+	kdh->dumptime = htod64(time_second);
+	kdh->blocksize = htod32(blksz);
+	strncpy(kdh->hostname, hostname, sizeof(kdh->hostname));
+	strncpy(kdh->versionstring, version, sizeof(kdh->versionstring));
+	if (panicstr != NULL)
+		strncpy(kdh->panicstring, panicstr, sizeof(kdh->panicstring));
+	kdh->parity = kerneldump_parity(kdh);
+}
+
+static int
+buf_write(struct dumperinfo *di, char *ptr, size_t sz)
+{
+	size_t len;
+	int error;
+
+	while (sz) {
+		len = DEV_BSIZE - fragsz;
+		if (len > sz)
+			len = sz;
+		bcopy(ptr, buffer + fragsz, len);
+		fragsz += len;
+		ptr += len;
+		sz -= len;
+		if (fragsz == DEV_BSIZE) {
+			error = di->dumper(di->priv, buffer, 0, dumplo,
+			    DEV_BSIZE);
+			if (error)
+				return error;
+			dumplo += DEV_BSIZE;
+			fragsz = 0;
+		}
+	}
+
+	return (0);
+}
+
+static int
+buf_flush(struct dumperinfo *di)
+{
+	int error;
+
+	if (fragsz == 0)
+		return (0);
+
+	error = di->dumper(di->priv, buffer, 0, dumplo, DEV_BSIZE);
+	dumplo += DEV_BSIZE;
+	return (error);
+}
+
+static int
+reg_write(struct dumperinfo *di, vm_paddr_t pa, vm_size_t size)
+{
+	struct sparc64_dump_reg r;
+
+	r.dr_pa = pa;
+	r.dr_size = size;
+	r.dr_offs = dumppos;
+	dumppos += size;
+	return (buf_write(di, (char *)&r, sizeof(r)));
+}
+
+static int
+blk_dump(struct dumperinfo *di, vm_paddr_t pa, vm_size_t size)
+{
+	vm_size_t pos, rsz;
+	vm_offset_t va;
+	int c, counter, error, twiddle;
+
+	printf("  chunk at %#lx: %ld bytes ", (u_long)pa, (long)size);
+
+	va = 0L;
+	error = counter = twiddle = 0;
+	for (pos = 0; pos < size; pos += MAXDUMPSZ, counter++) {
+		if (counter % 128 == 0)
+			printf("%c\b", "|/-\\"[twiddle++ & 3]);
+		rsz = size - pos;
+		rsz = (rsz > MAXDUMPSZ) ? MAXDUMPSZ : rsz;
+#ifdef notyet
+		va = TLB_PHYS_TO_DIRECT(pa + pos);
+#endif
+		error = di->dumper(di->priv, (void *)va, 0, dumplo, rsz);
+		if (error)
+			break;
+		dumplo += rsz;
+
+		/* Check for user abort. */
+		c = cncheckc();
+		if (c == 0x03)
+			return (ECANCELED);
+		if (c != -1)
+			printf("(CTRL-C to abort)  ");
+	}
+	printf("... %s\n", (error) ? "fail" : "ok");
+	return (error);
+}
+
+void
+dumpsys(struct dumperinfo *di)
+{
+	struct sparc64_dump_hdr hdr;
+	vm_size_t size, totsize, hdrsize;
+	int error, i, nreg;
+
+	/* Calculate dump size. */
+	size = 0;
+	nreg = sparc64_nmemreg;
+	for (i = 0; i < sparc64_nmemreg; i++)
+		size += sparc64_memreg[i].mr_size;
+	/* Account for the header size. */
+	hdrsize = roundup2(sizeof(hdr) + sizeof(struct sparc64_dump_reg) * nreg,
+	    DEV_BSIZE);
+	size += hdrsize;
+
+	totsize = size + 2 * sizeof(kdh);
+	if (totsize > di->mediasize) {
+		printf("Insufficient space on device (need %ld, have %ld), "
+		    "refusing to dump.\n", (long)totsize,
+		    (long)di->mediasize);
+		error = ENOSPC;
+		goto fail;
+	}
+
+	/* Determine dump offset on device. */
+	dumplo = di->mediaoffset + di->mediasize - totsize;
+
+	mkdumpheader(&kdh, KERNELDUMP_SPARC64_VERSION, size, di->blocksize);
+
+	printf("Dumping %lu MB (%d chunks)\n", (u_long)(size >> 20), nreg);
+
+	/* Dump leader */
+	error = di->dumper(di->priv, &kdh, 0, dumplo, sizeof(kdh));
+	if (error)
+		goto fail;
+	dumplo += sizeof(kdh);
+
+	/* Dump the private header. */
+	hdr.dh_hdr_size = hdrsize;
+#ifdef notyet
+	/* XXX SUN4V_FIXME */
+	hdr.dh_tsb_pa = tsb_kernel_phys;
+	hdr.dh_tsb_size = tsb_kernel_size;
+#endif
+	hdr.dh_nregions = nreg;
+
+	if (buf_write(di, (char *)&hdr, sizeof(hdr)) != 0)
+		goto fail;
+
+	dumppos = hdrsize;
+	/* Now, write out the region descriptors. */
+	for (i = 0; i < sparc64_nmemreg; i++) {
+		error = reg_write(di, sparc64_memreg[i].mr_start,
+		    sparc64_memreg[i].mr_size);
+		if (error != 0)
+			goto fail;
+	}
+	buf_flush(di);
+
+	/* Dump memory chunks. */
+	for (i = 0; i < sparc64_nmemreg; i++) {
+		error = blk_dump(di, sparc64_memreg[i].mr_start,
+		    sparc64_memreg[i].mr_size);
+		if (error != 0)
+			goto fail;
+	}
+
+	/* Dump trailer */
+	error = di->dumper(di->priv, &kdh, 0, dumplo, sizeof(kdh));
+	if (error)
+		goto fail;
+
+	/* Signal completion, signoff and exit stage left. */
+	di->dumper(di->priv, NULL, 0, 0, 0);
+	printf("\nDump complete\n");
+	return;
+
+ fail:
+	/* XXX It should look more like VMS :-) */
+	printf("** DUMP FAILED (ERROR %d) **\n", error);
+}
diff --git a/sys/sun4v/sun4v/eeprom.c b/sys/sun4v/sun4v/eeprom.c
new file mode 100644
index 000000000000..68390070e78e
--- /dev/null
+++ b/sys/sun4v/sun4v/eeprom.c
@@ -0,0 +1,192 @@
+/*-
+ * Copyright (c) 1992, 1993
+ *	The Regents of the University of California.  All rights reserved.
+ * Copyright (c) 1994 Gordon W. Ross
+ * Copyright (c) 1993 Adam Glass
+ * Copyright (c) 1996 Paul Kranenburg
+ * Copyright (c) 1996
+ * 	The President and Fellows of Harvard College. All rights reserved.
+ *
+ * This software was developed by the Computer Systems Engineering group
+ * at Lawrence Berkeley Laboratory under DARPA contract BG 91-66 and
+ * contributed to Berkeley.
+ *
+ * All advertising materials mentioning features or use of this software
+ * must display the following acknowledgement:
+ *	This product includes software developed by Harvard University.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ *    must display the following acknowledgement:
+ *	This product includes software developed by Paul Kranenburg.
+ *	This product includes software developed by Harvard University.
+ * 4. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	from: @(#)clock.c	8.1 (Berkeley) 6/11/93
+ *	from: NetBSD: clock.c,v 1.41 2001/07/24 19:29:25 eeh Exp
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+/*
+ * clock (eeprom) attaches at EBus, FireHose or SBus
+ */
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/bus.h>
+#include <sys/eventhandler.h>
+#include <sys/kernel.h>
+#include <sys/lock.h>
+#include <sys/module.h>
+#include <sys/mutex.h>
+#include <sys/resource.h>
+
+#include <dev/ofw/ofw_bus.h>
+
+#include <machine/bus.h>
+#include <machine/idprom.h>
+#include <machine/resource.h>
+
+#include <sys/rman.h>
+
+#include <dev/mk48txx/mk48txxvar.h>
+
+#include "clock_if.h"
+
+#define	IDPROM_OFFSET	40
+
+static devclass_t eeprom_devclass;
+
+static device_probe_t eeprom_probe;
+static device_attach_t eeprom_attach;
+
+static device_method_t eeprom_methods[] = {
+	/* Device interface */
+	DEVMETHOD(device_probe,		eeprom_probe),
+	DEVMETHOD(device_attach,	eeprom_attach),
+
+	/* clock interface */
+	DEVMETHOD(clock_gettime,	mk48txx_gettime),
+	DEVMETHOD(clock_settime,	mk48txx_settime),
+
+	{ 0, 0 }
+};
+
+static driver_t eeprom_driver = {
+	"eeprom",
+	eeprom_methods,
+	sizeof(struct mk48txx_softc),
+};
+
+DRIVER_MODULE(eeprom, ebus, eeprom_driver, eeprom_devclass, 0, 0);
+DRIVER_MODULE(eeprom, fhc, eeprom_driver, eeprom_devclass, 0, 0);
+DRIVER_MODULE(eeprom, sbus, eeprom_driver, eeprom_devclass, 0, 0);
+
+static int
+eeprom_probe(device_t dev)
+{
+ 
+	if (strcmp("eeprom", ofw_bus_get_name(dev)) == 0) {
+		device_set_desc(dev, "EEPROM/clock");
+		return (0);
+	}
+	return (ENXIO);
+}
+
+static int
+eeprom_attach(device_t dev)
+{
+	struct mk48txx_softc *sc;
+	struct resource *res;
+	struct timespec ts;
+	uint32_t h;
+	int error, i, rid;
+
+	sc = device_get_softc(dev);
+
+	mtx_init(&sc->sc_mtx, "eeprom_mtx", NULL, MTX_DEF);
+
+	rid = 0;
+	res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE);
+	if (res == NULL) {
+		device_printf(dev, "cannot allocate resources\n");
+		error = ENXIO;
+		goto fail_mtx;
+	}
+	sc->sc_bst = rman_get_bustag(res);
+	sc->sc_bsh = rman_get_bushandle(res);
+
+	if ((sc->sc_model = ofw_bus_get_model(dev)) == NULL) {
+		device_printf(dev, "cannot determine model\n");
+		error = ENXIO;
+		goto fail_res;
+	}
+
+	/* Our TOD clock year 0 is 1968 */
+	sc->sc_year0 = 1968;
+	/* Use default register read/write functions. */
+	sc->sc_flag = 0;
+	if ((error = mk48txx_attach(dev)) != 0) {
+		device_printf(dev, "cannot attach time of day clock\n");
+		goto fail_res;
+	}
+
+	/*
+	 * Get the hostid from the NVRAM. This serves no real purpose other
+	 * than being able to display it below as not all sparc64 models
+	 * have an `eeprom' device and even some that do store the hostid
+	 * elsewhere. The hostid in the NVRAM of the MK48Txx reads all zero
+	 * on the latter models. A generic way to retrieve the hostid is to
+	 * use the `idprom' node.
+	 */
+	mtx_lock(&sc->sc_mtx);
+	h = bus_space_read_1(sc->sc_bst, sc->sc_bsh, sc->sc_nvramsz -
+	    IDPROM_OFFSET + offsetof(struct idprom, id_machine)) << 24;
+	for (i = 0; i < 3; i++) {
+		h |= bus_space_read_1(sc->sc_bst, sc->sc_bsh, sc->sc_nvramsz -
+		    IDPROM_OFFSET + offsetof(struct idprom, id_hostid[i])) <<
+		    ((2 - i) * 8);
+	}
+	mtx_unlock(&sc->sc_mtx);
+	if (h != 0)
+		device_printf(dev, "hostid %x\n", (u_int)h);
+
+	if (bootverbose) {
+		mk48txx_gettime(dev, &ts);
+		device_printf(dev, "current time: %ld.%09ld\n", (long)ts.tv_sec,
+		    ts.tv_nsec);
+	}
+
+	return (0);
+
+ fail_res:
+	bus_release_resource(dev, SYS_RES_MEMORY, rid, res);
+ fail_mtx:
+	mtx_destroy(&sc->sc_mtx);
+
+	return (error);
+}
diff --git a/sys/sun4v/sun4v/elf_machdep.c b/sys/sun4v/sun4v/elf_machdep.c
new file mode 100644
index 000000000000..1b9bbc23ece9
--- /dev/null
+++ b/sys/sun4v/sun4v/elf_machdep.c
@@ -0,0 +1,361 @@
+/*-
+ * Copyright (c) 2001 Jake Burkholder.
+ * Copyright (c) 2000 Eduardo Horvath.
+ * Copyright (c) 1999 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The NetBSD Foundation
+ * by Paul Kranenburg.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ *    must display the following acknowledgement:
+ *        This product includes software developed by the NetBSD
+ *        Foundation, Inc. and its contributors.
+ * 4. Neither the name of The NetBSD Foundation nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ *	from: NetBSD: mdreloc.c,v 1.5 2001/04/25 12:24:51 kleink Exp
+ * $FreeBSD$
+ */
+
+#include <sys/param.h>
+#include <sys/kernel.h>
+#include <sys/systm.h>
+#include <sys/exec.h>
+#include <sys/imgact.h>
+#include <sys/linker.h>
+#include <sys/sysent.h>
+#include <sys/imgact_elf.h>
+#include <sys/syscall.h>
+#include <sys/signalvar.h>
+#include <sys/vnode.h>
+
+#include <vm/vm.h>
+#include <vm/vm_param.h>
+
+#include <machine/elf.h>
+
+#include "linker_if.h"
+
+struct sysentvec elf64_freebsd_sysvec = {
+	SYS_MAXSYSCALL,
+	sysent,
+	0,
+	0,
+	NULL,
+	0,
+	NULL,
+	NULL,
+	__elfN(freebsd_fixup),
+	sendsig,
+	NULL,
+	NULL,
+	NULL,
+	"FreeBSD ELF64",
+	__elfN(coredump),
+	NULL,
+	MINSIGSTKSZ,
+	PAGE_SIZE,
+	VM_MIN_ADDRESS,
+	VM_MAXUSER_ADDRESS,
+	USRSTACK,
+	PS_STRINGS,
+	VM_PROT_READ | VM_PROT_WRITE,
+	exec_copyout_strings,
+	exec_setregs,
+	NULL
+};
+
+static Elf64_Brandinfo freebsd_brand_info = {
+						ELFOSABI_FREEBSD,
+						EM_SPARCV9,
+						"FreeBSD",
+						NULL,
+						"/libexec/ld-elf.so.1",
+						&elf64_freebsd_sysvec,
+						NULL,
+						0,
+					  };
+
+SYSINIT(elf64, SI_SUB_EXEC, SI_ORDER_ANY,
+	(sysinit_cfunc_t) elf64_insert_brand_entry,
+	&freebsd_brand_info);
+
+static Elf64_Brandinfo freebsd_brand_oinfo = {
+						ELFOSABI_FREEBSD,
+						EM_SPARCV9,
+						"FreeBSD",
+						NULL,
+						"/usr/libexec/ld-elf.so.1",
+						&elf64_freebsd_sysvec,
+						NULL,
+						0,
+					  };
+
+SYSINIT(oelf64, SI_SUB_EXEC, SI_ORDER_ANY,
+	(sysinit_cfunc_t) elf64_insert_brand_entry,
+	&freebsd_brand_oinfo);
+
+
+void
+elf64_dump_thread(struct thread *td __unused, void *dst __unused,
+    size_t *off __unused)
+{
+}
+
+
+/*
+ * The following table holds for each relocation type:
+ *	- the width in bits of the memory location the relocation
+ *	  applies to (not currently used)
+ *	- the number of bits the relocation value must be shifted to the
+ *	  right (i.e. discard least significant bits) to fit into
+ *	  the appropriate field in the instruction word.
+ *	- flags indicating whether
+ *		* the relocation involves a symbol
+ *		* the relocation is relative to the current position
+ *		* the relocation is for a GOT entry
+ *		* the relocation is relative to the load address
+ *
+ */
+#define _RF_S		0x80000000		/* Resolve symbol */
+#define _RF_A		0x40000000		/* Use addend */
+#define _RF_P		0x20000000		/* Location relative */
+#define _RF_G		0x10000000		/* GOT offset */
+#define _RF_B		0x08000000		/* Load address relative */
+#define _RF_U		0x04000000		/* Unaligned */
+#define _RF_SZ(s)	(((s) & 0xff) << 8)	/* memory target size */
+#define _RF_RS(s)	( (s) & 0xff)		/* right shift */
+static int reloc_target_flags[] = {
+	0,							/* NONE */
+	_RF_S|_RF_A|		_RF_SZ(8)  | _RF_RS(0),		/* RELOC_8 */
+	_RF_S|_RF_A|		_RF_SZ(16) | _RF_RS(0),		/* RELOC_16 */
+	_RF_S|_RF_A|		_RF_SZ(32) | _RF_RS(0),		/* RELOC_32 */
+	_RF_S|_RF_A|_RF_P|	_RF_SZ(8)  | _RF_RS(0),		/* DISP_8 */
+	_RF_S|_RF_A|_RF_P|	_RF_SZ(16) | _RF_RS(0),		/* DISP_16 */
+	_RF_S|_RF_A|_RF_P|	_RF_SZ(32) | _RF_RS(0),		/* DISP_32 */
+	_RF_S|_RF_A|_RF_P|	_RF_SZ(32) | _RF_RS(2),		/* WDISP_30 */
+	_RF_S|_RF_A|_RF_P|	_RF_SZ(32) | _RF_RS(2),		/* WDISP_22 */
+	_RF_S|_RF_A|		_RF_SZ(32) | _RF_RS(10),	/* HI22 */
+	_RF_S|_RF_A|		_RF_SZ(32) | _RF_RS(0),		/* 22 */
+	_RF_S|_RF_A|		_RF_SZ(32) | _RF_RS(0),		/* 13 */
+	_RF_S|_RF_A|		_RF_SZ(32) | _RF_RS(0),		/* LO10 */
+	_RF_G|			_RF_SZ(32) | _RF_RS(0),		/* GOT10 */
+	_RF_G|			_RF_SZ(32) | _RF_RS(0),		/* GOT13 */
+	_RF_G|			_RF_SZ(32) | _RF_RS(10),	/* GOT22 */
+	_RF_S|_RF_A|_RF_P|	_RF_SZ(32) | _RF_RS(0),		/* PC10 */
+	_RF_S|_RF_A|_RF_P|	_RF_SZ(32) | _RF_RS(10),	/* PC22 */
+	      _RF_A|_RF_P|	_RF_SZ(32) | _RF_RS(2),		/* WPLT30 */
+				_RF_SZ(32) | _RF_RS(0),		/* COPY */
+	_RF_S|_RF_A|		_RF_SZ(64) | _RF_RS(0),		/* GLOB_DAT */
+				_RF_SZ(32) | _RF_RS(0),		/* JMP_SLOT */
+	      _RF_A|	_RF_B|	_RF_SZ(64) | _RF_RS(0),		/* RELATIVE */
+	_RF_S|_RF_A|	_RF_U|	_RF_SZ(32) | _RF_RS(0),		/* UA_32 */
+
+	      _RF_A|		_RF_SZ(32) | _RF_RS(0),		/* PLT32 */
+	      _RF_A|		_RF_SZ(32) | _RF_RS(10),	/* HIPLT22 */
+	      _RF_A|		_RF_SZ(32) | _RF_RS(0),		/* LOPLT10 */
+	      _RF_A|_RF_P|	_RF_SZ(32) | _RF_RS(0),		/* PCPLT32 */
+	      _RF_A|_RF_P|	_RF_SZ(32) | _RF_RS(10),	/* PCPLT22 */
+	      _RF_A|_RF_P|	_RF_SZ(32) | _RF_RS(0),		/* PCPLT10 */
+	_RF_S|_RF_A|		_RF_SZ(32) | _RF_RS(0),		/* 10 */
+	_RF_S|_RF_A|		_RF_SZ(32) | _RF_RS(0),		/* 11 */
+	_RF_S|_RF_A|		_RF_SZ(64) | _RF_RS(0),		/* 64 */
+	_RF_S|_RF_A|/*extra*/	_RF_SZ(32) | _RF_RS(0),		/* OLO10 */
+	_RF_S|_RF_A|		_RF_SZ(32) | _RF_RS(42),	/* HH22 */
+	_RF_S|_RF_A|		_RF_SZ(32) | _RF_RS(32),	/* HM10 */
+	_RF_S|_RF_A|		_RF_SZ(32) | _RF_RS(10),	/* LM22 */
+	_RF_S|_RF_A|_RF_P|	_RF_SZ(32) | _RF_RS(42),	/* PC_HH22 */
+	_RF_S|_RF_A|_RF_P|	_RF_SZ(32) | _RF_RS(32),	/* PC_HM10 */
+	_RF_S|_RF_A|_RF_P|	_RF_SZ(32) | _RF_RS(10),	/* PC_LM22 */
+	_RF_S|_RF_A|_RF_P|	_RF_SZ(32) | _RF_RS(2),		/* WDISP16 */
+	_RF_S|_RF_A|_RF_P|	_RF_SZ(32) | _RF_RS(2),		/* WDISP19 */
+	_RF_S|_RF_A|		_RF_SZ(32) | _RF_RS(0),		/* GLOB_JMP */
+	_RF_S|_RF_A|		_RF_SZ(32) | _RF_RS(0),		/* 7 */
+	_RF_S|_RF_A|		_RF_SZ(32) | _RF_RS(0),		/* 5 */
+	_RF_S|_RF_A|		_RF_SZ(32) | _RF_RS(0),		/* 6 */
+	_RF_S|_RF_A|_RF_P|	_RF_SZ(64) | _RF_RS(0),		/* DISP64 */
+	      _RF_A|		_RF_SZ(64) | _RF_RS(0),		/* PLT64 */
+	_RF_S|_RF_A|		_RF_SZ(32) | _RF_RS(10),	/* HIX22 */
+	_RF_S|_RF_A|		_RF_SZ(32) | _RF_RS(0),		/* LOX10 */
+	_RF_S|_RF_A|		_RF_SZ(32) | _RF_RS(22),	/* H44 */
+	_RF_S|_RF_A|		_RF_SZ(32) | _RF_RS(12),	/* M44 */
+	_RF_S|_RF_A|		_RF_SZ(32) | _RF_RS(0),		/* L44 */
+	_RF_S|_RF_A|		_RF_SZ(64) | _RF_RS(0),		/* REGISTER */
+	_RF_S|_RF_A|	_RF_U|	_RF_SZ(64) | _RF_RS(0),		/* UA64 */
+	_RF_S|_RF_A|	_RF_U|	_RF_SZ(16) | _RF_RS(0),		/* UA16 */
+};
+
+#if 0
+static const char *reloc_names[] = {
+	"NONE", "RELOC_8", "RELOC_16", "RELOC_32", "DISP_8",
+	"DISP_16", "DISP_32", "WDISP_30", "WDISP_22", "HI22",
+	"22", "13", "LO10", "GOT10", "GOT13",
+	"GOT22", "PC10", "PC22", "WPLT30", "COPY",
+	"GLOB_DAT", "JMP_SLOT", "RELATIVE", "UA_32", "PLT32",
+	"HIPLT22", "LOPLT10", "LOPLT10", "PCPLT22", "PCPLT32",
+	"10", "11", "64", "OLO10", "HH22",
+	"HM10", "LM22", "PC_HH22", "PC_HM10", "PC_LM22", 
+	"WDISP16", "WDISP19", "GLOB_JMP", "7", "5", "6",
+	"DISP64", "PLT64", "HIX22", "LOX10", "H44", "M44", 
+	"L44", "REGISTER", "UA64", "UA16"
+};
+#endif
+
+#define RELOC_RESOLVE_SYMBOL(t)		((reloc_target_flags[t] & _RF_S) != 0)
+#define RELOC_PC_RELATIVE(t)		((reloc_target_flags[t] & _RF_P) != 0)
+#define RELOC_BASE_RELATIVE(t)		((reloc_target_flags[t] & _RF_B) != 0)
+#define RELOC_UNALIGNED(t)		((reloc_target_flags[t] & _RF_U) != 0)
+#define RELOC_USE_ADDEND(t)		((reloc_target_flags[t] & _RF_A) != 0)
+#define RELOC_TARGET_SIZE(t)		((reloc_target_flags[t] >> 8) & 0xff)
+#define RELOC_VALUE_RIGHTSHIFT(t)	(reloc_target_flags[t] & 0xff)
+
+static long reloc_target_bitmask[] = {
+#define _BM(x)	(~(-(1ULL << (x))))
+	0,				/* NONE */
+	_BM(8), _BM(16), _BM(32),	/* RELOC_8, _16, _32 */
+	_BM(8), _BM(16), _BM(32),	/* DISP8, DISP16, DISP32 */
+	_BM(30), _BM(22),		/* WDISP30, WDISP22 */
+	_BM(22), _BM(22),		/* HI22, _22 */
+	_BM(13), _BM(10),		/* RELOC_13, _LO10 */
+	_BM(10), _BM(13), _BM(22),	/* GOT10, GOT13, GOT22 */
+	_BM(10), _BM(22),		/* _PC10, _PC22 */  
+	_BM(30), 0,			/* _WPLT30, _COPY */
+	_BM(32), _BM(32), _BM(32),	/* _GLOB_DAT, JMP_SLOT, _RELATIVE */
+	_BM(32), _BM(32),		/* _UA32, PLT32 */
+	_BM(22), _BM(10),		/* _HIPLT22, LOPLT10 */
+	_BM(32), _BM(22), _BM(10),	/* _PCPLT32, _PCPLT22, _PCPLT10 */
+	_BM(10), _BM(11), -1,		/* _10, _11, _64 */
+	_BM(10), _BM(22),		/* _OLO10, _HH22 */
+	_BM(10), _BM(22),		/* _HM10, _LM22 */
+	_BM(22), _BM(10), _BM(22),	/* _PC_HH22, _PC_HM10, _PC_LM22 */
+	_BM(16), _BM(19),		/* _WDISP16, _WDISP19 */
+	-1,				/* GLOB_JMP */
+	_BM(7), _BM(5), _BM(6)		/* _7, _5, _6 */
+	-1, -1,				/* DISP64, PLT64 */
+	_BM(22), _BM(13),		/* HIX22, LOX10 */
+	_BM(22), _BM(10), _BM(13),	/* H44, M44, L44 */
+	-1, -1, _BM(16),		/* REGISTER, UA64, UA16 */
+#undef _BM
+};
+#define RELOC_VALUE_BITMASK(t)	(reloc_target_bitmask[t])
+
+int
+elf_reloc_local(linker_file_t lf, Elf_Addr relocbase, const void *data,
+    int type, elf_lookup_fn lookup)
+{
+	const Elf_Rela *rela;
+	Elf_Addr value;
+	Elf_Addr *where;
+
+	if (type != ELF_RELOC_RELA)
+		return (-1);
+
+	rela = (const Elf_Rela *)data;
+	if (ELF_R_TYPE(rela->r_info) != R_SPARC_RELATIVE)
+		return (-1);
+
+	value = rela->r_addend + (Elf_Addr)lf->address;
+	where = (Elf_Addr *)((Elf_Addr)lf->address + rela->r_offset);
+
+	*where = value;
+
+	return (0);
+}
+
+/* Process one elf relocation with addend. */
+int
+elf_reloc(linker_file_t lf, Elf_Addr relocbase, const void *data, int type,
+    elf_lookup_fn lookup)
+{
+	const Elf_Rela *rela;
+	Elf_Word *where32;
+	Elf_Addr *where;
+	Elf_Size rtype, symidx;
+	Elf_Addr value;
+	Elf_Addr mask;
+	Elf_Addr addr;
+
+	if (type != ELF_RELOC_RELA)
+		return (-1);
+
+	rela = (const Elf_Rela *)data;
+	where = (Elf_Addr *)(relocbase + rela->r_offset);
+	where32 = (Elf_Word *)where;
+	rtype = ELF_R_TYPE(rela->r_info);
+	symidx = ELF_R_SYM(rela->r_info);
+
+	if (rtype == R_SPARC_NONE || rtype == R_SPARC_RELATIVE)
+		return (0);
+
+	if (rtype == R_SPARC_JMP_SLOT || rtype == R_SPARC_COPY ||
+	    rtype > R_SPARC_UA16)
+		return (-1);
+
+	if (RELOC_UNALIGNED(rtype))
+		return (-1);
+
+	value = rela->r_addend;
+
+	if (RELOC_RESOLVE_SYMBOL(rtype)) {
+		addr = lookup(lf, symidx, 1);
+		if (addr == 0)
+			return (-1);
+		value += addr;
+	}
+
+	if (RELOC_PC_RELATIVE(rtype))
+		value -= (Elf_Addr)where;
+
+	if (RELOC_BASE_RELATIVE(rtype))
+		value += relocbase;
+
+	mask = RELOC_VALUE_BITMASK(rtype);
+	value >>= RELOC_VALUE_RIGHTSHIFT(rtype);
+	value &= mask;
+
+	if (RELOC_TARGET_SIZE(rtype) > 32) {
+		*where &= ~mask;
+		*where |= value;
+	} else {
+		*where32 &= ~mask;
+		*where32 |= value;
+	}
+
+	return (0);
+}
+
+int
+elf_cpu_load_file(linker_file_t lf __unused)
+{
+
+	return (0);
+}
+
+int
+elf_cpu_unload_file(linker_file_t lf __unused)
+{
+
+	return (0);
+}
diff --git a/sys/sun4v/sun4v/exception.S b/sys/sun4v/sun4v/exception.S
new file mode 100644
index 000000000000..33251ca29c7e
--- /dev/null
+++ b/sys/sun4v/sun4v/exception.S
@@ -0,0 +1,1921 @@
+/*
+ * Copyright (c) 2006 Kip Macy
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+	
+#include <machine/asm.h>
+__FBSDID("$FreeBSD$")			
+
+#include "opt_compat.h"
+#include "opt_ddb.h"
+#include "opt_simulator.h"
+
+#include <machine/asi.h>
+#include <machine/asmacros.h>
+#include <machine/ktr.h>
+#include <machine/pstate.h>
+#include <machine/trap.h>
+#include <machine/tstate.h>
+#include <machine/wstate.h>
+#include <machine/hypervisorvar.h>
+#include <machine/errata.h>
+
+#include "assym.s"
+#include <machine/mmu.h>
+
+#define PMAP_DEBUG
+
+#define SPILL_FILL_MAGIC_TRAP_ON	nop
+#define SPILL_FILL_MAGIC_TRAP_OFF	nop
+
+#define REGSIZE         8
+
+	.register %g2,#ignore
+	.register %g3,#ignore
+	.register %g6,#ignore
+	.register %g7,#ignore
+
+
+#define PCB_REG %g6
+
+#define PUTCHAR(x) \
+	mov	x, %o0		; \
+	mov	CONS_WRITE, %o5	; \
+	ta	FAST_TRAP
+/*
+ * Atomically set the reference bit in a tte.
+ */
+#define	TTE_SET_BIT(r1, r2, r3, bit) \
+	add	r1, TTE_DATA, r1 ; \
+	ldx	[r1], r2 ; \
+9:	or	r2, bit, r3 ; \
+	casxa	[r1] ASI_N, r2, r3 ; \
+	cmp	r2, r3 ; \
+	bne,pn	%xcc, 9b ; \
+	 mov	r3, r2
+
+#define	TTE_SET_REF(r1, r2, r3)		TTE_SET_BIT(r1, r2, r3, TD_REF)
+#define	TTE_SET_W(r1, r2, r3)		TTE_SET_BIT(r1, r2, r3, TD_W)
+
+		
+/*
+ * Macros for spilling and filling live windows.
+ * Here we use the more complicated [regaddr] format which requires
+ * us to interleave setting the globals in order to be able to use
+ * imm_asi - we don't ever implicitly assume kernel context as in
+ * Solaris' spill/fill handlers so that we have the option of using
+ * block initializing stores - twin doubleword loads could also be
+ * advantageous but will require an additional macro
+ *
+ */
+
+
+#define	SPILL(storer, base, size) \
+	storer	%l0, [base + (0 * size)] ; \
+	storer	%l1, [base + (1 * size)] ; \
+	storer	%l2, [base + (2 * size)] ; \
+	storer	%l3, [base + (3 * size)] ; \
+	storer	%l4, [base + (4 * size)] ; \
+	storer	%l5, [base + (5 * size)] ; \
+	storer	%l6, [base + (6 * size)] ; \
+	storer	%l7, [base + (7 * size)] ; \
+	storer	%i0, [base + (8 * size)] ; \
+	storer	%i1, [base + (9 * size)] ; \
+	storer	%i2, [base + (10 * size)] ; \
+	storer	%i3, [base + (11 * size)] ; \
+	storer	%i4, [base + (12 * size)] ; \
+	storer	%i5, [base + (13 * size)] ; \
+	storer	%i6, [base + (14 * size)] ; \
+	storer	%i7, [base + (15 * size)]
+
+#define	SPILL_ASI(storer, bias, size, asi) \
+	mov	0 + bias, %g1	                ;\
+	storer	%l0, [%sp + %g1]asi             ;\
+	mov	size + bias, %g2	        ;\
+	storer	%l1, [%sp + %g2]asi             ;\
+	mov	(2 * size) + bias, %g3	        ;\
+	storer	%l2, [%sp + %g3]asi             ;\
+	mov	(3 * size) + bias, %g4	        ;\
+	storer	%l3, [%sp + %g4]asi             ;\
+	add	%sp,  (4 * size), %g5           ;\
+	storer	%l4, [%g5 + %g1]asi             ;\
+	storer	%l5, [%g5 + %g2]asi             ;\
+	storer	%l6, [%g5 + %g3]asi             ;\
+	storer	%l7, [%g5 + %g4]asi             ;\
+	add	%g5,  (4 * size), %g5           ;\
+	storer	%i0, [%g5 + %g1]asi             ;\
+	storer	%i1, [%g5 + %g2]asi             ;\
+	storer	%i2, [%g5 + %g3]asi             ;\
+	storer	%i3, [%g5 + %g4]asi             ;\
+	add	%g5,  (4 * size), %g5           ;\
+	storer	%i4, [%g5 + %g1]asi             ;\
+	storer	%i5, [%g5 + %g2]asi             ;\
+	storer	%i6, [%g5 + %g3]asi             ;\
+	storer	%i7, [%g5 + %g4]asi  
+
+/* 16 instructions */
+#define	SPILL_ASI_64 \
+	stxa	%l0, [%g1 + 0x0]%asi             ;\
+	stxa	%i0, [%g1 + 0x40]%asi             ;\
+	stxa	%l1, [%g1 + 0x8]%asi              ;\
+	stxa	%l2, [%g1 + 0x10]%asi             ;\
+	stxa	%l3, [%g1 + 0x18]%asi             ;\
+	stxa	%l4, [%g1 + 0x20]%asi             ;\
+	stxa	%l5, [%g1 + 0x28]%asi             ;\
+	stxa	%l6, [%g1 + 0x30]%asi             ;\
+	stxa	%l7, [%g1 + 0x38]%asi             ;\
+	stxa	%i1, [%g1 + 0x48]%asi             ;\
+	stxa	%i2, [%g1 + 0x50]%asi             ;\
+	stxa	%i3, [%g1 + 0x58]%asi             ;\
+	stxa	%i4, [%g1 + 0x60]%asi             ;\
+	stxa	%i5, [%g1 + 0x68]%asi             ;\
+	stxa	%i6, [%g1 + 0x70]%asi             ;\
+	stxa	%i7, [%g1 + 0x78]%asi  
+
+/* 23 instructions */
+#define	FILL(loader, bias, size, asi) \
+	mov	0 + bias, %g1			;\
+	loader	[%sp + %g1]asi, %l0		;\
+	mov	size + bias, %g2		;\
+	loader	[%sp + %g2]asi, %l1		;\
+	mov	(2 * size) + bias, %g3		;\
+	loader	[%sp + %g3]asi, %l2		;\
+	mov	(3 * size) + bias, %g4		;\
+	loader	[%sp + %g4]asi, %l3		;\
+	add	%sp, (4 * size), %g5		;\
+	loader	[%g5 + %g1]asi, %l4		;\
+	loader	[%g5 + %g2]asi, %l5		;\
+	loader	[%g5 + %g3]asi, %l6		;\
+	loader	[%g5 + %g4]asi, %l7		;\
+	add	%g5, (4 * size), %g5		;\
+	loader	[%g5 + %g1]asi, %i0		;\
+	loader	[%g5 + %g2]asi, %i1		;\
+	loader	[%g5 + %g3]asi, %i2		;\
+	loader	[%g5 + %g4]asi, %i3		;\
+	add	%g5, (4 * size), %g5		;\
+	loader	[%g5 + %g1]asi, %i4		;\
+	loader	[%g5 + %g2]asi, %i5		;\
+	loader	[%g5 + %g3]asi, %i6		;\
+	loader	[%g5 + %g4]asi, %i7
+
+#define	SPILL_ASI_SET(storer, size) \
+	storer	%l0, [%g1 + (0 * size)]%asi     ;\
+	storer	%l1, [%g1 + (1 * size)]%asi     ;\
+	storer	%l2, [%g1 + (2 * size)]%asi     ;\
+	storer	%l3, [%g1 + (3 * size)]%asi     ;\
+	storer	%l4, [%g1 + (4 * size)]%asi     ;\
+	storer	%l5, [%g1 + (5 * size)]%asi     ;\
+	storer	%l6, [%g1 + (6 * size)]%asi     ;\
+	storer	%l7, [%g1 + (7 * size)]%asi     ;\
+	storer	%i0, [%g1 + (8 * size)]%asi     ;\
+	storer	%i1, [%g1 + (9 * size)]%asi     ;\
+	storer	%i2, [%g1 + (10 * size)]%asi     ;\
+	storer	%i3, [%g1 + (11 * size)]%asi     ;\
+	storer	%i4, [%g1 + (12 * size)]%asi     ;\
+	storer	%i5, [%g1 + (13 * size)]%asi     ;\
+	storer	%i6, [%g1 + (14 * size)]%asi     ;\
+	storer	%i7, [%g1 + (15 * size)]%asi 
+
+/* 16 instructions */
+#define	FILL_ASI_SET(loader, size) \
+	loader	[%g1 + 0x0]%asi, %l0		;\
+	loader	[%g1 + (size * 1)]%asi, %l1	;\
+	loader	[%g1 + (size * 2)]%asi, %l2	;\
+	loader	[%g1 + (size * 3)]%asi, %l3	;\
+	loader	[%g1 + (size * 4)]%asi, %l4	;\
+	loader	[%g1 + (size * 5)]%asi, %l5	;\
+	loader	[%g1 + (size * 6)]%asi, %l6	;\
+	loader	[%g1 + (size * 7)]%asi, %l7	;\
+	loader	[%g1 + (size * 8)]%asi, %i0	;\
+	loader	[%g1 + (size * 9)]%asi, %i1	;\
+	loader	[%g1 + (size * 10)]%asi, %i2	;\
+	loader	[%g1 + (size * 11)]%asi, %i3	;\
+	loader	[%g1 + (size * 12)]%asi, %i4	;\
+	loader	[%g1 + (size * 13)]%asi, %i5	;\
+	loader	[%g1 + (size * 14)]%asi, %i6	;\
+	loader	[%g1 + (size * 15)]%asi, %i7	
+	
+/* 9 instructions */
+#define	FILL_DW \
+	prefetch [%g1 + 0x40], #one_read        ;\
+	ldda	[%g1 + 0]%asi, %l0		;\
+	ldda	[%g1 + 0x10]%asi, %l2		;\
+	ldda	[%g1 + 0x20]%asi, %l4		;\
+	ldda	[%g1 + 0x30]%asi, %l6		;\
+	ldda	[%g1 + 0x40]%asi, %i0		;\
+	ldda	[%g1 + 0x50]%asi, %i2		;\
+	ldda	[%g1 + 0x60]%asi, %i4		;\
+	ldda	[%g1 + 0x70]%asi, %i6		
+
+#include <sun4v/sun4v/wbuf.S>	
+	/*
+	 * Clean window traps occur when %cleanwin is zero to ensure that data
+	 * is not leaked between address spaces in registers.
+	 */
+	.macro	clean_window
+	clr	%o0
+	clr	%o1
+	clr	%o2
+	clr	%o3
+	clr	%o4
+	clr	%o5
+	clr	%o6
+	clr	%o7
+	clr	%l0
+	clr	%l1
+	clr	%l2
+	clr	%l3
+	clr	%l4
+	clr	%l5
+	clr	%l6
+	rdpr	%cleanwin, %l7
+	inc	%l7
+	wrpr	%l7, 0, %cleanwin
+	clr	%l7
+	retry
+	.align	128
+	.endm
+
+	.macro	tl0_split
+	.endm
+
+	.macro	tl0_setup	type
+	rdpr	%tt, %g3
+	sub	%g0, 1, %g4
+	set	trap, %g1
+	ba	%xcc, tl0_trap
+	  mov	\type, %g2
+	.endm
+
+	/*
+	 * Generic trap type.  Call trap() with the specified type.
+	 */
+	.macro	tl0_gen		type
+	tl0_setup \type
+	.align	32
+	.endm
+
+	/*
+	 * This is used to suck up the massive swaths of reserved trap types.
+	 * Generates count "reserved" trap vectors.
+	 */
+
+	.macro	tl0_reserved	count
+	.rept	\count
+	tl0_gen	T_RESERVED
+	.align	32
+	.endr
+	.endm
+
+	.macro	tl1_setup	type
+	rdpr	%tt, %g3
+	sub	%g0, 1, %g4
+	set	trap, %g1
+	ba	%xcc, tl1_trap
+	  mov	\type, %g2
+	.endm
+
+	.macro	tl1_gen		type
+	tl1_setup \type
+	.align	32
+	.endm
+
+	.macro	tl1_reserved	count
+	.rept	\count
+	tl1_gen	T_RESERVED
+	.endr
+	.endm
+
+	.macro	insn_excptn
+	GET_MMFSA_SCRATCH(%g1)			
+	mov	MMFSA_D_ADDR, %g2
+	ldxa	[%g1 + %g2]ASI_REAL, %g3
+	sub	%g0, 1, %g4
+	set	trap, %g1
+	ba	%xcc, tl0_trap 
+	  mov	T_INSTRUCTION_EXCEPTION, %g2
+
+	.align	32
+	.endm
+
+	.macro	insn_miss
+	GET_MMFSA_SCRATCH(%g1)	
+	mov	MMFSA_I_TYPE, %g2 
+	mov	MMFSA_I_ADDR, %g3
+	mov	MMFSA_I_CTX, %g7 
+	ldxa	[%g1 + %g2]ASI_REAL, %g4
+	ldxa	[%g1 + %g3]ASI_REAL, %g5
+	ba,pt	%xcc, tsb_miss_handler
+          mov   T_INSTRUCTION_MISS, %g3
+	.align	32
+	.endm
+	
+	.macro	data_excptn
+	GET_MMFSA_SCRATCH(%g1)			
+	mov	MMFSA_D_ADDR, %g2
+	ldxa	[%g1 + %g2]ASI_REAL, %g3
+	ba,a,pt %xcc, data_excptn_fault
+	.align	32
+	.endm
+
+ENTRY(data_excptn_fault)
+	mov	MMFSA_D_CTX, %g7 
+	ldxa	[%g1 + %g7]ASI_REAL, %g4
+	sllx	%g4, TRAP_CTX_SHIFT, %g4
+	or	%g4, T_DATA_EXCEPTION, %g2
+	set	trap, %g1
+	sub	%g0, 1, %g4
+	ba,a,pt	%xcc, tl0_trap 
+END(data_excptn_fault)
+
+	.macro	data_miss
+	GET_MMFSA_SCRATCH(%g1)			
+	mov	MMFSA_D_TYPE, %g2 
+	mov	MMFSA_D_ADDR, %g3
+	mov	MMFSA_D_CTX, %g7 
+	ldxa	[%g1 + %g2]ASI_REAL, %g4
+	ldxa	[%g1 + %g3]ASI_REAL, %g5
+	ba,pt	%xcc, tsb_miss_handler
+          mov   T_DATA_MISS, %g3
+	.align	32
+	.endm
+
+	.macro	data_prot
+	GET_MMFSA_SCRATCH(%g1)		
+	mov	MMFSA_D_ADDR, %g3
+	mov	MMFSA_D_CTX,  %g7
+	ldxa	[%g1 + %g3]ASI_REAL, %g5
+	ba,pt	%xcc, tsb_miss_handler
+	  mov   T_DATA_PROTECTION, %g3
+	.align	32
+	.endm
+
+	.macro	tl0_align
+	GET_MMFSA_SCRATCH(%g1)	
+	mov	MMFSA_D_ADDR, %g3
+	mov	MMFSA_D_CTX,  %g7
+	ldxa	[%g1 + %g3]ASI_REAL, %g3
+	ba,a,pt	%xcc, align_fault
+	.align	32
+	.endm
+
+ENTRY(align_fault)
+	ldxa	[%g1 + %g7]ASI_REAL, %g4
+	sllx	%g4, TRAP_CTX_SHIFT, %g4
+	or	%g4, T_MEM_ADDRESS_NOT_ALIGNED, %g2
+	sub	%g0, 1, %g4
+	set	trap, %g1
+	ba,a,pt	%xcc, tl0_trap
+END(align_fault)
+		
+	.macro	cpu_mondo
+	ba,a,pt %xcc, cpu_mondo
+	.align	32
+	.endm
+
+	.macro	dev_mondo
+	ba,a,pt %xcc, dev_mondo
+	.align	32
+	.endm
+
+	.macro	resumable_error
+	!MAGIC_TRAP_ON
+	!MAGIC_EXIT
+	clr	%g3
+	sub	%g0, 1, %g4
+	set	trap, %g1
+	ba	%xcc, tl0_trap
+	  mov	T_RESUMABLE_ERROR, %g2
+	.align	32
+	.endm
+
+	.macro	nonresumable_error
+	clr	%g3
+	sub	%g0, 1, %g4
+	set	trap, %g1
+	ba	%xcc, tl0_trap
+	  mov	T_NONRESUMABLE_ERROR, %g2
+	.align	32
+	.endm
+
+
+#define ALIGN_128	.align  128
+#define SYNC		#Sync
+#define LOOKASIDE	#Lookaside
+
+#ifdef USE_FAST_SPILLFILL
+#define spill_64bit_asi(asival, asival_unaligned, target)	\
+	wr	%g0, asival, %asi  ;            \
+	add	%sp, SPOFF, %g1	   ;            \
+	SPILL_ASI_64               ;	        \
+	membar	LOOKASIDE          ;            \
+	saved			   ;		\
+	retry			   ;		\
+	.skip (31-21)*4		   ;		\
+	ba,a,pt %xcc, fault_64bit_##target ; \
+	ALIGN_128	
+
+#define	spill_64clean(asival, asival_unaligned, target)		\
+	wr	%g0, asival, %asi  ;            \
+	add	%sp, SPOFF, %g1	   ;            \
+	SPILL_ASI_64               ; 	        \
+	membar	LOOKASIDE          ;            \
+	b	spill_clean	   ;		\
+	  mov	WSTATE_USER64, %g7 ; 		\
+	.skip (31-21)*4		   ;		\
+	ba,a,pt %xcc, fault_64bit_##target ; 	\
+	ALIGN_128	
+
+#define fill_64bit_asi(asival, asival_unaligned, target)	\
+	add	%sp, SPOFF, %g1	   ;            \
+	wr	%g0, asival, %asi  ;            \
+	FILL_DW                    ; 		\
+	restored		   ;		\
+	retry			   ;		\
+	.skip (31-13)*4		   ;		\
+	ba,a,pt %xcc, fault_64bit_##target ; \
+	ALIGN_128	
+#else
+#define spill_64bit_asi(asival, asival_unaligned, target)	\
+	wr	%g0, asival_unaligned, %asi ;   \
+	add	%sp, SPOFF, %g1	  ;             \
+	SPILL_ASI_SET(stxa, 8)     ;	        \
+	saved			   ;		\
+	retry			   ;		\
+	.skip (31-20)*4		   ;		\
+	ba,a,pt %xcc, fault_64bit_##target ; \
+	ALIGN_128	
+
+#define	spill_64clean(asival, asival_unaligned, target)		\
+	wr	%g0, asival_unaligned, %asi  ;            \
+	add	%sp, SPOFF, %g1	   ;            \
+	SPILL_ASI_SET(stxa, 8)     ; 	        \
+	b	spill_clean	   ;		\
+	  mov	WSTATE_USER64, %g7 ; 		\
+	.skip (31-20)*4		   ;		\
+	ba,a,pt %xcc, fault_64bit_##target ; 	\
+	ALIGN_128	
+
+#define fill_64bit_asi(asival, asival_unaligned, target)	\
+	wr	%g0, asival_unaligned, %asi  ;  \
+	add	%sp, SPOFF, %g1	  ;             \
+	FILL_ASI_SET(ldxa, 8)      ; 		\
+	restored		   ;		\
+	retry			   ;		\
+	.skip (31-20)*4		   ;		\
+	ba,a,pt %xcc, fault_64bit_##target ; \
+	ALIGN_128	
+#endif
+
+#define spill_32bit_asi(asi, target)		\
+	srl	%sp, 0, %sp	;		\
+	SPILL_FILL_MAGIC_TRAP_ON;               \
+	SPILL_ASI(sta, 0, 4, asi)	;	\
+	saved			; 		\
+	SPILL_FILL_MAGIC_TRAP_OFF;		\
+	retry			; 		\
+	.skip (31-28)*4		; 		\
+	ba,a,pt %xcc, fault_32bit_##target ; \
+	ALIGN_128
+
+#define	spill_32clean(asi, target)		\
+	srl	%sp, 0, %sp	; 		\
+	SPILL_FILL_MAGIC_TRAP_ON;               \
+	SPILL_ASI(sta, 0, 4, asi)	; 	\
+	b	spill_clean	; 		\
+	  mov	WSTATE_USER32, %g7 ; 		\
+	.skip (31-27)*4		; 		\
+	ba,a,pt    %xcc, fault_32bit_##target ; \
+	ALIGN_128	
+	
+#define fill_32bit_asi(asi, target)		\
+	srl	%sp, 0, %sp	;		\
+	SPILL_FILL_MAGIC_TRAP_ON;               \
+	FILL(lda, 0, 4, asi)	; 		\
+	restored		; 		\
+	retry			; 		\
+	.skip (31-27)*4		; 		\
+	ba,a,pt %xcc, fault_32bit_##target ; \
+	ALIGN_128	
+
+.align 128
+ENTRY(fill_64bit_slow_fn0)                      
+fill_slow_start:		
+	FILL_ASI_SET(ldxa, 8);                  
+	restored		;               
+	retry 			;               
+	.skip (31-18)*4		   ;		
+	ba,a,pt %xcc, fault_64bit_fn0 ;
+	.align 128	
+END(fill_64bit_slow_fn0)
+ENTRY(fill_64bit_slow_not)                      
+	FILL_ASI_SET(ldxa, 8);                  
+	restored		;               
+	retry 			;               
+	.skip (31-18)*4		   ;		
+	ba,a,pt %xcc, fault_64bit_not ; 
+	.align 128	
+END(fill_64bit_slow_not)
+fill_slow_end:	
+		
+	.macro	spill_32bit_primary_sn0
+	spill_32bit_asi(ASI_AIUP, sn0)
+	.endm
+
+	.macro	spill_64bit_primary_sn0
+	spill_64bit_asi(ASI_LDSTBI_AIUP, ASI_AIUP, sn0)
+	.endm
+
+	.macro spill_32clean_primary_sn0
+	spill_32clean(ASI_AIUP, sn0)
+	.endm
+			
+	.macro spill_64clean_primary_sn0
+	spill_64clean(ASI_LDSTBI_AIUP, ASI_AIUP, sn0)
+	.endm
+
+	.macro spill_32bit_nucleus_not
+	spill_32bit_asi(ASI_N, not)
+	.endm
+
+	.macro spill_64bit_nucleus_not
+	spill_64bit_asi(ASI_LDSTBI_N, ASI_N, not)
+	.endm
+
+	.macro	spill_32bit_secondary_so0
+	spill_32bit_asi(ASI_AIUS, so0)
+	.endm
+
+	.macro	spill_64bit_secondary_so0
+	spill_64bit_asi(ASI_LDSTBI_AIUS, ASI_AIUS, so0)
+	.endm
+	
+	.macro	fill_32bit_primary_fn0
+	fill_32bit_asi(ASI_AIUP, fn0)
+	.endm
+
+	.macro	fill_64bit_primary_fn0
+	fill_64bit_asi(ASI_LDSTBI_AIUP, ASI_AIUP, fn0)
+	.endm
+
+	.macro fill_32bit_nucleus_not
+	fill_32bit_asi(ASI_N, not)
+	.endm
+
+	.macro fill_64bit_nucleus_not
+	fill_64bit_asi(ASI_LDSTBI_N, ASI_N, not)
+	.endm
+
+	.macro	spill_32bit_tt1_primary_sn1
+	ba,a,pt %xcc, fault_32bit_sn1
+	  nop
+	.align 128
+	.endm
+
+	.macro	spill_64bit_tt1_primary_sn1 
+	ba,a,pt %xcc, fault_64bit_sn1
+	  nop
+	.align 128
+	.endm
+
+	.macro	spill_64bit_ktt1_sk
+	ba,a,pt %xcc, fault_64bit_sk
+	  nop
+	.align 128
+	.endm
+
+	.macro	spill_mixed_ktt1_sk
+	btst	1, %sp
+	bz,a,pt	%xcc, fault_32bit_sk
+	  srl	%sp, 0, %sp
+	ba,a,pt	%xcc, fault_64bit_sk
+	  nop
+	.align 128
+	.endm
+
+	.macro	spill_32bit_tt1_secondary_so1
+	ba,a,pt %xcc, fault_32bit_so1
+	  nop
+	.align 128
+	.endm
+
+	.macro	spill_64bit_tt1_secondary_so1
+	ba,a,pt %xcc, fault_64bit_so1
+	  nop
+	.align 128
+	.endm
+
+	.macro	spill_mixed
+!	MAGIC_EXIT
+	nop
+	.align	128
+	.endm
+
+	.macro	fill_mixed
+!	MAGIC_EXIT
+	nop
+	.align	128
+	.endm
+
+	.macro	tl1_align
+	ba,a,pt	%xcc, tl1_trap
+	.align	32
+	.endm		
+		
+	.macro	tl0_pil_entry level, mask
+	wrpr	%g0, 1, %gl
+	set	\mask, %g1
+	clr	%g2
+	clr	%g3
+	wr	%g1, 0, %clear_softint
+	ba	%xcc, tl0_intr
+	  mov	\level, %g4
+	.align	32
+	.endm
+
+#define	INTR(level, traplvl)						\
+	tl ## traplvl ## _pil_entry	level, 1 << level
+
+#define	TICK(traplvl) \
+	tl ## traplvl ## _pil_entry	PIL_TICK, 1
+
+#define	INTR_LEVEL(tl)							\
+	INTR(1, tl) ;							\
+	INTR(2, tl) ;							\
+	INTR(3, tl) ;							\
+	INTR(4, tl) ;							\
+	INTR(5, tl) ;							\
+	INTR(6, tl) ;							\
+	INTR(7, tl) ;							\
+	INTR(8, tl) ;							\
+	INTR(9, tl) ;							\
+	INTR(10, tl) ;							\
+	INTR(11, tl) ;							\
+	INTR(12, tl) ;							\
+	INTR(13, tl) ;							\
+tick_ ## tl ## _entry:							\
+	TICK(tl) ;							\
+	INTR(15, tl) ;
+
+	.macro	tl0_pil
+	INTR_LEVEL(0)
+	.endm
+
+
+	.macro	tl0_syscall
+	clr	%g3
+	sub	%g0, 1, %g4
+        set	syscall, %g1
+	ba	%xcc, tl0_trap
+	  mov	T_SYSCALL, %g2
+	.align	32
+	.endm
+
+	.macro	dtrace_fasttrap
+	sethi	%hi(dtrace_fasttrap_probe_ptr), %g4
+	ldx	[%g4 + %lo(dtrace_fasttrap_probe_ptr)], %g4
+	set	dtrace_fasttrap_probe, %g1
+	brnz,pn	%g4, tl0_utrap
+	  sub	%g0, 1, %g4
+	.align	32
+	.endm
+
+	.macro	dtrace_pid
+	set	dtrace_pid_probe, %g1
+	ba,pt	%xcc, tl0_utrap
+	  sub	%g0, 1, %g4
+	.align	32
+	.endm
+
+	.macro	dtrace_return
+	set	dtrace_return_probe, %g1
+	ba,pt	%xcc, tl0_utrap
+	  sub	%g0, 1, %g4
+	.align 32
+	.endm
+	
+	.macro	tl0_fp_restore
+	GET_PCB(PCB_REG)			! 3 instructions
+	ldx	[%g6 + PCB_FLAGS], %g1
+	ba,pt	%xcc, tl0_fp_restore
+	  wr	%g0, FPRS_FEF, %fprs
+	.align	32
+	.endm
+
+	.macro tl0_fp_enable
+	GET_PCB(PCB_REG)			! 3 instructions
+	ldx	[PCB_REG + PCB_FLAGS], %g1
+	andcc	%g1, PCB_FEF, %g0
+	bnz,pt	%xcc, tl0_fp_restore		
+	  wr	%g0, FPRS_FEF, %fprs
+	retry
+	.align	32
+	.endm
+	
+ENTRY(tl0_fp_restore)
+	andn	%g1, PCB_FEF, %g1
+	stx	%g1, [%g6 + PCB_FLAGS]
+	
+	ldd	[PCB_REG + PCB_UFP + (0 * 64)], %f0
+	ldd	[PCB_REG + PCB_UFP + (1 * 64)], %f16
+	ldd	[PCB_REG + PCB_UFP + (2 * 64)], %f32
+	ldd	[PCB_REG + PCB_UFP + (3 * 64)], %f48
+	retry
+END(tl0_fp_restore)
+
+	.macro	tl1_insn_excptn
+	 nop
+	.align	32
+	.endm
+
+
+	.macro	tl1_soft	count
+	.rept	\count
+	tl1_gen	T_SOFT | T_KERNEL
+	.endr
+	.endm
+
+	.sect	.trap
+	.align	0x8000
+	.globl	tl0_base
+tl0_base:
+ 	tl0_reserved	8				! 0x0-0x7
+tl0_insn_excptn:
+ 	insn_excptn					! 0x8
+tl0_insn_miss:
+	insn_miss					! 0x9
+	tl0_reserved	6				! 0xa-0xf
+tl0_insn_illegal:
+ 	tl0_gen		T_ILLEGAL_INSTRUCTION		! 0x10
+tl0_priv_opcode:
+ 	tl0_gen		T_PRIVILEGED_OPCODE		! 0x11
+ 	tl0_reserved	14				! 0x12-0x1f
+tl0_fp_disabled:
+	tl0_fp_enable					! 0x20
+tl0_fp_ieee:
+ 	tl0_gen		T_FP_EXCEPTION_IEEE_754		! 0x21
+tl0_fp_other:
+ 	tl0_gen		T_FP_EXCEPTION_OTHER		! 0x22
+tl0_tag_ovflw:
+ 	tl0_gen		T_TAG_OVERFLOW			! 0x23
+tl0_clean_window:
+ 	clean_window					! 0x24
+tl0_divide:
+ 	tl0_gen		T_DIVISION_BY_ZERO		! 0x28
+ 	tl0_reserved	7				! 0x29-0x2f
+tl0_data_excptn:
+ 	data_excptn					! 0x30
+tl0_data_miss:
+ 	data_miss					! 0x31
+ 	tl0_reserved	2				! 0x32-0x33
+tl0_align:
+ 	tl0_align					! 0x34
+tl0_align_lddf:
+ 	tl0_gen		T_RESERVED			! 0x35
+tl0_align_stdf:
+	tl0_gen		T_RESERVED			! 0x36
+tl0_priv_action:
+	tl0_gen		T_PRIVILEGED_ACTION		! 0x37
+	tl0_reserved	9				! 0x38-0x40
+tl0_intr_level_41:
+	tl0_pil						! 0x41-0x4f
+	tl0_reserved	18				! 0x50-0x61
+tl0_watch_virt_62:
+	tl0_gen		T_VA_WATCHPOINT			! 0x62
+	tl0_reserved	9				! 0x63-0x6b
+tl0_data_prot_6c:	
+	data_prot					! 0x6c
+	tl0_reserved	9				! 0x6d-0x75
+tl0_breakpoint_76:
+	tl0_gen		T_BREAKPOINT			! 0x76
+	tl0_reserved	5				! 0x77-0x7b
+tl0_cpu_mondo_7c:
+	cpu_mondo					! 0x7c
+tl0_dev_mondo_7d:	
+	dev_mondo					! 0x7d	
+tl0_resumable_error_7e:
+	resumable_error					! 0x7e
+tl0_nonresumable_error_7f:	
+	nonresumable_error				! 0x7f	
+tl0_spill_n_normal_80:
+tl0_spill_0_normal:	
+	tl0_reserved		4			! 0x80
+tl0_spill_1_normal:	
+	spill_32bit_primary_sn0				! 0x84
+tl0_spill_2_normal:	
+	spill_64bit_primary_sn0				! 0x88
+tl0_spill_3_normal:	
+	spill_32clean_primary_sn0			! 0x8c
+tl0_spill_4_normal:	
+	spill_64clean_primary_sn0			! 0x90
+tl0_spill_5_normal:	
+	spill_32bit_nucleus_not				! 0x94
+tl0_spill_6_normal:	
+	spill_64bit_nucleus_not				! 0x98
+tl0_spill_7_normal:	
+	spill_mixed					! 0x9c
+tl0_spill_0_other:
+	tl0_reserved		4			! 0xa0
+tl0_spill_1_other:
+	spill_32bit_secondary_so0			! 0xa4
+tl0_spill_2_other:
+	spill_64bit_secondary_so0			! 0xa8
+tl0_spill_3_other:
+	spill_32bit_secondary_so0			! 0xac
+tl0_spill_4_other:
+	spill_64bit_secondary_so0			! 0xb0
+tl0_spill_5_other:
+	tl0_reserved		4			! 0xb4
+tl0_spill_6_other:
+	tl0_reserved		4			! 0xb8
+tl0_spill_7_other:
+	tl0_reserved		4			! 0xbc
+tl0_fill_n_normal:
+	tl0_reserved		4			! 0xc0
+tl0_fill_1_normal:	
+	fill_32bit_primary_fn0				! 0xc4 
+tl0_fill_2_normal:	
+	fill_64bit_primary_fn0				! 0xc8
+tl0_fill_3_normal:	
+	fill_32bit_primary_fn0				! 0xcc
+tl0_fill_4_normal:	
+	fill_64bit_primary_fn0				! 0xd0
+tl0_fill_5_normal:	
+	fill_32bit_nucleus_not				! 0xd4
+tl0_fill_6_normal:	
+	fill_64bit_nucleus_not				! 0xd8
+tl0_fill_7_normal:	
+	fill_mixed					! 0xdc
+tl0_fill_n_other_e0:
+	tl0_reserved		32			! 0xe0-0xff
+tl0_soft_100:
+	tl0_gen		T_SYSCALL			! 0x100
+	tl0_gen		T_BREAKPOINT			! 0x101
+	tl0_gen		T_DIVISION_BY_ZERO		! 0x102
+	tl0_reserved	1				! 0x103
+	tl0_gen		T_CLEAN_WINDOW			! 0x104
+	tl0_gen		T_RANGE_CHECK			! 0x105
+	tl0_gen		T_FIX_ALIGNMENT			! 0x106
+	tl0_gen		T_INTEGER_OVERFLOW		! 0x107
+	tl0_gen		T_SYSCALL			! 0x108
+	tl0_gen		T_SYSCALL			! 0x109
+	tl0_fp_restore					! 0x10a
+	tl0_reserved	5				! 0x10b-0x10f
+	tl0_gen		T_TRAP_INSTRUCTION_16		! 0x110
+	tl0_gen		T_TRAP_INSTRUCTION_17		! 0x111
+	tl0_gen		T_TRAP_INSTRUCTION_18		! 0x112
+	tl0_gen		T_TRAP_INSTRUCTION_19		! 0x113
+	tl0_gen		T_TRAP_INSTRUCTION_20		! 0x114
+	tl0_gen		T_TRAP_INSTRUCTION_21		! 0x115
+	tl0_gen		T_TRAP_INSTRUCTION_22		! 0x116
+	tl0_gen		T_TRAP_INSTRUCTION_23		! 0x117
+	tl0_gen		T_TRAP_INSTRUCTION_24		! 0x118
+	tl0_gen		T_TRAP_INSTRUCTION_25		! 0x119
+	tl0_gen		T_TRAP_INSTRUCTION_26		! 0x11a
+	tl0_gen		T_TRAP_INSTRUCTION_27		! 0x11b
+	tl0_gen		T_TRAP_INSTRUCTION_28		! 0x11c
+	tl0_gen		T_TRAP_INSTRUCTION_29		! 0x11d
+	tl0_gen		T_TRAP_INSTRUCTION_30		! 0x11e
+	tl0_gen		T_TRAP_INSTRUCTION_31		! 0x11f
+	tl0_reserved	24				! 0x120-0x137
+tl0_dtrace_pid:						
+	dtrace_pid					! 0x138
+tl0_dtrace_fasttrap:					
+	dtrace_fasttrap					! 0x139
+tl0_dtrace_return:					
+	dtrace_return					! 0x13a
+	tl0_reserved	5				! 0x13b - 0x13f
+	tl0_gen		T_SYSCALL			! 0x140       LP64 system call
+	tl0_syscall					! 0x141
+	tl0_gen		T_SYSCALL			! 0x142
+	tl0_gen		T_SYSCALL			! 0x143
+	tl0_reserved	188				! 0x144-0x1ff
+tl1_base:
+ 	tl1_reserved	9				! 0x200-0x208
+tl1_insn_miss_209:
+	insn_miss					! 0x209
+	tl1_reserved	26				! 0x20a-0x223
+tl1_clean_window_224:
+ 	clean_window					! 0x224
+tl1_divide_228:
+ 	tl1_reserved	8				! 0x228-0x22f
+tl1_data_excptn_230:
+ 	data_excptn					! 0x230
+tl1_data_miss_231:
+	data_miss					! 0x231
+ 	tl1_reserved	2				! 0x232-0x233
+tl1_align:
+ 	tl1_align					! 0x234
+	tl1_reserved	55				! 0x235-0x26b
+tl1_data_prot:
+	data_prot					! 0x26c
+	tl1_reserved	18				! 0x26c-0x27e
+tl1_nonresumable_error:	
+	nonresumable_error				! 0x27f	
+tl1_spill_n_normal:
+tl1_spill_0_normal:	
+	tl1_reserved		4			! 0x280
+tl1_spill_1_normal:
+	spill_32bit_tt1_primary_sn1			! 0x284
+tl1_spill_2_normal:
+	spill_64bit_tt1_primary_sn1			! 0x288
+tl1_spill_3_normal:
+	spill_32bit_tt1_primary_sn1			! 0x28c
+tl1_spill_4_normal:
+	spill_64bit_tt1_primary_sn1			! 0x290
+tl1_spill_5_normal:
+	tl1_reserved		4			! 0x294
+tl1_spill_6_normal:
+	spill_64bit_ktt1_sk				! 0x298
+tl1_spill_7_normal:
+	spill_mixed_ktt1_sk				! 0x29c
+tl1_spill_n_other:
+tl1_spill_0_other:	
+	tl1_reserved		4			! 0x2a0
+tl1_spill_1_other:	
+	spill_32bit_tt1_secondary_so1			! 0x2a4
+tl1_spill_2_other:	
+	spill_64bit_tt1_secondary_so1			! 0x2a8
+tl1_spill_3_other:	
+	spill_32bit_tt1_secondary_so1			! 0x2ac
+tl1_spill_4_other:	
+	spill_64bit_tt1_secondary_so1			! 0x2b0
+tl1_spill_5_other:	
+	tl1_reserved		4			! 0x2b4
+tl1_spill_6_other:	
+	tl1_reserved		4			! 0x2b8
+tl1_spill_7_other:	
+	tl1_reserved		4			! 0x2bc
+tl1_fill_n_normal:
+	tl1_reserved		32			! 0x2c0-0x2df
+tl1_fill_n_other:
+	tl1_reserved		32			! 0x2e0-0x2ff
+tl1_soft_traps:
+	tl1_reserved		256
+.globl tl1_end
+tl1_end:					
+
+spill_clean:
+	sethi	%hi(nwin_minus_one), %g5
+	ld	[%g5 + %lo(nwin_minus_one)], %g5
+	rdpr	%cwp, %g6
+	deccc	%g6
+	movneg	%xcc, %g5, %g6				! if (--%cwp < 0) %g6 = nwin-1
+	wrpr	%g6, %cwp
+	clr	%l0
+	clr	%l1
+	clr	%l2
+	clr	%l3
+	clr	%l4
+	clr	%l5
+	clr	%l6
+	clr	%l7
+	wrpr	%g0, %g7, %wstate
+	saved
+	retry
+
+	
+	
+#define	KWBUF64_TO_STACK(SBP,SPP,TMP) \
+	ldx	[SBP + (0*8)], TMP; \
+	stx	TMP, [SPP + SPOFF + 0]; \
+	ldx	[SBP + (1*8)], TMP; \
+	stx	TMP, [SPP + SPOFF + 8]; \
+	ldx	[SBP + (2*8)], TMP; \
+	stx	TMP, [SPP + SPOFF + 16]; \
+	ldx	[SBP + (3*8)], TMP; \
+	stx	TMP, [SPP + SPOFF + 24]; \
+	ldx	[SBP + (4*8)], TMP; \
+	stx	TMP, [SPP + SPOFF + 32]; \
+	ldx	[SBP + (5*8)], TMP; \
+	stx	TMP, [SPP + SPOFF + 40]; \
+	ldx	[SBP + (6*8)], TMP; \
+	stx	TMP, [SPP + SPOFF + 48]; \
+	ldx	[SBP + (7*8)], TMP; \
+	stx	TMP, [SPP + SPOFF + 56]; \
+	ldx	[SBP + (8*8)], TMP; \
+	stx	TMP, [SPP + SPOFF + 64]; \
+	ldx	[SBP + (9*8)], TMP; \
+	stx	TMP, [SPP + SPOFF + 72]; \
+	ldx	[SBP + (10*8)], TMP; \
+	stx	TMP, [SPP + SPOFF + 80]; \
+	ldx	[SBP + (11*8)], TMP; \
+	stx	TMP, [SPP + SPOFF + 88]; \
+	ldx	[SBP + (12*8)], TMP; \
+	stx	TMP, [SPP + SPOFF + 96]; \
+	ldx	[SBP + (13*8)], TMP; \
+	stx	TMP, [SPP + SPOFF + 104]; \
+	ldx	[SBP + (14*8)], TMP; \
+	stx	TMP, [SPP + SPOFF + 112]; \
+	ldx	[SBP + (15*8)], TMP; \
+	stx	TMP, [SPP + SPOFF + 120];
+
+
+#define	fill_64bit_rtt(asi_num)				\
+	wr	%g0, asi_num, %asi				;\
+	rdpr	%cwp, %g1					;\
+	sub	%g1, 1, %g1					;\
+	wrpr	%g1, %cwp					;\
+	ldxa	[%sp + SPOFF + 0]%asi, %l0			;\
+	ldxa	[%sp + SPOFF + 8]%asi, %l1			;\
+	ldxa	[%sp + SPOFF + 16]%asi, %l2			;\
+	ldxa	[%sp + SPOFF + 24]%asi, %l3			;\
+	ldxa	[%sp + SPOFF + 32]%asi, %l4			;\
+	ldxa	[%sp + SPOFF + 40]%asi, %l5			;\
+	ldxa	[%sp + SPOFF + 48]%asi, %l6			;\
+	ldxa	[%sp + SPOFF + 56]%asi, %l7			;\
+	ldxa	[%sp + SPOFF + 64]%asi, %i0			;\
+	ldxa	[%sp + SPOFF + 72]%asi, %i1			;\
+	ldxa	[%sp + SPOFF + 80]%asi, %i2			;\
+	ldxa	[%sp + SPOFF + 88]%asi, %i3			;\
+	ldxa	[%sp + SPOFF + 96]%asi, %i4			;\
+	ldxa	[%sp + SPOFF + 104]%asi, %i5			;\
+	ldxa	[%sp + SPOFF + 112]%asi, %i6			;\
+	ldxa	[%sp + SPOFF + 120]%asi, %i7			;\
+	restored						;\
+	add	%g1, 1, %g1					;\
+	wrpr	%g1, %cwp
+
+
+	
+	
+ENTRY(utl0)
+	SAVE_GLOBALS(%l7)
+	rd	%asi, %g1
+	SAVE_OUTS(%l7)
+	stx	%g1, [%l7 + TF_ASI]
+	GET_PCPU_SCRATCH_SLOW(%g6)
+	wrpr	%g0, PSTATE_KERNEL, %pstate	! enable ints
+
+	brnz	%o1, common_utrap
+	  nop		
+	call	spinlock_enter
+	  nop
+common_uintr:	
+	jmpl	%l3, %o7			! call interrupt handler
+	  mov	%l7, %o0
+	call	spinlock_exit
+	  nop
+	ba,pt	%xcc, user_rtt
+	  nop
+common_utrap:
+	jmpl	%l3, %o7			! call trap handler / syscall
+	  mov	%l7, %o0
+
+	ldx	[PCPU_REG + PC_CURPCB], %g6
+	ldx	[%g6 + PCB_KSTACK], %g6
+	sub	%g6, TF_SIZEOF, %sp
+	add	%sp, REGOFF + SPOFF, %l7
+ENTRY(user_rtt)
+	nop
+	! pil handling needs to be re-visited
+	wrpr	%g0, PIL_TICK, %pil
+	ldx	[PCPU(CURTHREAD)], %l4
+	lduw	[%l4 + TD_FLAGS], %l1
+	ldx	[%l4 + TD_MD + MD_SAVED_PIL], %l0
+	set	TDF_ASTPENDING | TDF_NEEDRESCHED, %l2
+	and	%l1, %l2, %l1
+	brz,a,pt %l1, 1f
+	 nop
+
+	! handle AST and retry return
+	wrpr	%g0, %l0, %pil
+	call	ast
+	  mov   %l7, %o0
+	ba,pt	%xcc, user_rtt
+	 nop
+
+1:	
+	ldx	[PCPU_REG + PC_CURPCB], %g6
+	ldx	[%g6 + PCB_NSAVED], %l1
+	brz,a,pt %l1, 2f
+	  nop
+	wrpr	%g0, %l0, %pil
+	mov	T_SPILL, %o1
+	call	trap
+	  mov	%l7, %o0
+	ba,pt	%xcc, user_rtt
+	 nop
+2:
+
+	ld	[%l7 + TF_WSTATE], %l3	
+	!
+	! restore user globals and outs
+	!
+	rdpr	%pstate, %l1
+	ldx	[%l7 + TF_ASI], %g1
+	wrpr	%l1, PSTATE_IE, %pstate
+	wr	%g1, 0, %asi
+	RESTORE_GLOBALS_USER(%l7)
+	wrpr	%g0, 1, %gl
+	RESTORE_OUTS(%l7)
+
+	wrpr	%g0, 0, %pil	! drop pil to 0
+	wrpr	%g0, 1, %tl	! raise tl -> 1 before setting pcontext
+	
+	mov	MMU_CID_S, %g1
+	GET_MMU_CONTEXT(%g1, %g2)
+	mov	MMU_CID_P, %g1
+	sethi	%hi(FLUSH_ADDR), %g3
+	SET_MMU_CONTEXT(%g1, %g2)
+	flush	%g3				! flush required by immu
+						! hangover from US I
+	!
+	! setup trap regs
+	!
+
+	ldx	[%l7 + TF_TPC], %g1
+	ldx	[%l7 + TF_TNPC], %g2
+	ldx	[%l7 + TF_TSTATE], %l0
+	ldx	[%l7 + TF_FPRS], %l1
+
+	wrpr	%g1, %tpc
+	wrpr	%g2, %tnpc
+	andn	%l0, TSTATE_CWP_MASK, %g6
+
+	wr	%g0, FPRS_FEF, %fprs
+	ldx	[%l7 + TF_FSR], %fsr
+	wr	%l1, 0, %fprs	
+
+
+		!
+	! switch "other" windows back to "normal" windows and
+	! restore to window we originally trapped in
+	!
+	rdpr	%otherwin, %g1
+	wrpr	%g0, 0, %otherwin
+	add	%l3, WSTATE_CLEAN_OFFSET, %l3	! convert to "clean" wstate
+	wrpr	%g0, %l3, %wstate
+	wrpr	%g0, %g1, %canrestore
+	
+	rdpr	%canrestore, %g1
+	brnz	%g1, 3f
+	  nop				! no trap, use restore directly
+	rdpr	%cwp, %g1
+	wrpr	%g1, %g6, %tstate	! needed by wbuf recovery code
+	! hand craft the restore to avoid getting to TL > 2
+	rdpr	%wstate, %g1
+	btst	1, %g1
+	beq	4f
+	  nop
+	.global	rtt_fill_start
+rtt_fill_start:
+#if 0
+	fill_32bit_rtt(ASI_AIUP)
+	ba,a	3f
+#endif
+4:
+	membar	#Lookaside
+	fill_64bit_rtt(ASI_AIUP)
+	.global	rtt_fill_end
+rtt_fill_end:
+3:
+	restore				! should not trap
+2:
+	
+	!
+	! set %cleanwin to %canrestore
+	! set %tstate to the correct %cwp
+	! retry resumes user execution
+	!
+	rdpr	%canrestore, %g1
+	wrpr	%g0, %g1, %cleanwin
+	rdpr	%cwp, %g1
+	wrpr	%g1, %g6, %tstate
+	retry
+END(user_rtt)
+END(utl0)		
+
+ENTRY(ktl0)
+	nop
+	SAVE_GLOBALS(%l7)
+	rd	%asi, %g1
+	SAVE_OUTS(%l7)
+	stx	%g1, [%l7 + TF_ASI]
+	GET_PCPU_SCRATCH_SLOW(%g6)		! we really shouldn't need this ...	
+	wrpr	%g0, PSTATE_KERNEL, %pstate	! enable interrupts
+
+	brnz	%o1, common_ktrap
+	  nop	
+	call	spinlock_enter
+	  nop
+common_kintr:	
+	jmpl	%l3, %o7			! call trap handler
+	  mov	%l7, %o0
+	call	spinlock_exit
+	  nop
+	b	common_rtt
+	  nop
+common_ktrap:
+	jmpl	%l3, %o7			! call trap handler
+	  mov	%l7, %o0
+	
+ENTRY(krtt)
+common_rtt:	
+	!
+	! restore globals and outs
+	!
+	rdpr	%pstate, %l1
+	ldx	[%l7 + TF_ASI], %g1
+	wrpr	%l1, PSTATE_IE, %pstate
+	wr	%g1, 0, %asi	
+
+	RESTORE_GLOBALS_KERNEL(%l7)
+	
+	! switch to global set 1
+	wrpr	%g0, 1, %gl
+	RESTORE_OUTS(%l7)
+#ifdef notyet
+	!
+	! set %pil from max(old pil, cur_thread_spl)
+	!
+	ldn	[%l0 + T_CPU], %l0
+	ld	[%l0 + CPU_BASE_SPL], %l0
+	cmp	%l6, %l0
+	movg	%xcc, %l6, %l0
+	wrpr	%g0, %l0, %pil
+#endif
+	GET_PCPU_SCRATCH
+	ldx	[PCPU(CURTHREAD)], %l0
+	ldx	[%l0 + TD_MD + MD_SAVED_PIL], %l0
+	wrpr	%g0, %l0, %pil
+	!
+	! raise tl
+	! setup trap regs
+	! restore to window we originally trapped in
+	!
+	wrpr	%g0, 1, %tl
+	
+	ldx	[%l7 + TF_TSTATE], %l0
+	ldx	[%l7 + TF_TPC], %g1
+	ldx	[%l7 + TF_TNPC], %g2
+	ldx	[%l7 + TF_FPRS], %l1
+			
+	andn	%l0, TSTATE_CWP_MASK, %g6
+	wrpr	%g1, %tpc
+	wrpr	%g2, %tnpc
+
+
+	wr	%g0, FPRS_FEF, %fprs
+	ldx	[%l7 + TF_FSR], %fsr
+	wr	%l1, 0, %fprs	
+	
+	rdpr	%canrestore, %g1
+	brnz	%g1, 3f
+	  nop				! can use restore directly
+	rdpr	%cwp, %g1
+	wrpr	%g1, %g6, %tstate	! needed by wbuf recovery code
+
+	! avoid going above TL2
+	fill_64bit_rtt(ASI_N)
+
+3:
+	restore
+	!
+	! set %tstate to the correct %cwp
+	!
+	rdpr	%cwp, %g1
+	wrpr	%g1, %g6, %tstate
+	retry
+END(krtt)	
+END(ktl0)
+
+
+
+ENTRY(tl0_ktrap)
+	GET_PCPU_SCRATCH
+        set     ktl0, %g6
+	
+	save	%sp, -(CCFSZ + TF_SIZEOF), %sp
+		
+	brz	%g2, 2f
+	  nop
+	or	%g2, T_KERNEL, %g2	
+2:			
+	! if the kwbuf is full we need to save to the stack now
+	ld	[PCPU_REG + PC_KWBUF_FULL], %o0   
+	brz,pt	%o0, 1f
+	nop
+	st	%g0, [PCPU_REG + PC_KWBUF_FULL]
+	ldx	[PCPU_REG + PC_KWBUF_SP], %o1
+	add	PCPU_REG, PC_KWBUF, %o0
+	KWBUF64_TO_STACK(%o0, %o1, %o2)
+1:			
+	ba,a,pt %xcc, win_saved
+END(tl0_ktrap)	
+	
+
+	! register convention:	
+	! %g2=level %g1=mask
+	
+ENTRY(tl0_intr)
+	SET(intr_handlers, %g7, %g6)
+	sllx	%g4, IH_SHIFT, %g7
+	ldx	[%g6 + %g7], %g1	! pointer to interrupt handler
+	rdpr	%pil, %g5
+	mov	%g5, %g4
+				
+	! %g1		pc of trap handler
+	! %g2, %g3	args of trap handler
+	! %g4		desired pil
+	! %g5, %g6	temps
+	! %g7		saved
+
+	! %l0, %l1	temps
+	! %l3		saved %g1
+	! %l4		flags
+	! %l5		memory fault info
+	! %l6		%pil for priv traps
+	! %l7		trapframe
+
+ENTRY(tl0_trap)
+	/* if we're at tl2 we have some extra work to do */
+	rdpr	%tl, %g5
+	cmp	%g5, 2
+	be,pn	%xcc, tl1_trap
+	  nop
+	
+	rdpr	%tstate, %g5
+	btst	TSTATE_PRIV, %g5
+	and	%g5, TSTATE_CWP_MASK, %g6
+	wrpr	%g0, %g6, %cwp
+	bnz,pn	%xcc, tl0_ktrap
+	  nop
+ENTRY(tl0_utrap)
+#ifdef	notyet
+	/* we need to determine from the hardware the number of register windows */
+	sethi	%hi(nwin_minus_one), %g5
+	ld	[%g5 + %lo(nwin_minus_one)], %g5
+#else
+	mov	nwin_minus_one, %g5
+#endif
+	GET_PCB(%g6)
+	wrpr	%g0, %g5, %cleanwin
+	ldx	[%g6 + PCB_KSTACK], %g6
+	sub	%g6, TF_SIZEOF, %g6
+#ifdef DEBUG_KSTACK
+	mov	%o0, %g5
+	mov	%o3, %l0
+	mov	%o4, %l1
+	mov	%o5, %l2
+	mov	%o6, %l3
+	mov	%o7, %l4
+	mov	0x10, %o0
+	mov	%g6, %o1
+	ta	TTRACE_ADDENTRY
+	mov	%g5, %o0
+	mov	%l0, %o3
+	mov	%l1, %o4
+	mov	%l2, %o5
+	mov	%l3, %o6
+	mov	%l4, %o7
+#endif
+	save	%g6, 0, %sp
+
+	rdpr	%canrestore, %l0
+	rdpr	%wstate, %l1
+	wrpr	%g0, 0, %canrestore
+	sllx	%l1, WSTATE_SHIFT, %l1
+	wrpr	%l1, WSTATE_K64, %wstate
+	wrpr	%g0, %l0, %otherwin
+	!
+	! set pcontext to run kernel
+	!
+	mov	KCONTEXT, %l0
+	mov	MMU_CID_P, %l1
+	sethi	%hi(FLUSH_ADDR), %l2
+	SET_MMU_CONTEXT(%l1, %l0)
+	flush	%l2			! flush/membar required by immu for 
+					! consistency guarantee
+	set	utl0, %g6
+win_saved:
+	mov	%g1, %l3		! set trap/interrupt for tl0
+	mov	%g2, %o1		! trap type
+	mov	%g3, %o2		! fault info if set
+	mov	%g5, %l6		! %pil if priv trap
+	!
+	! save state in trapframe
+	!
+	add	%sp, REGOFF + SPOFF, %l7
+	rdpr	%tpc, %l0
+	rdpr	%tnpc, %l1
+	rdpr	%tstate, %l2
+	stx	%l0, [%l7 + TF_TPC]
+	rd	%fprs, %l0
+	stx	%l1, [%l7 + TF_TNPC]
+	stx	%l2, [%l7 + TF_TSTATE]
+	stx	%l0, [%l7 + TF_FPRS]
+
+	/*
+	 * According to the sparc64 port fp must me enabled
+	 * before reading %fsr
+	 */
+	wr	%g0, FPRS_FEF, %fprs
+	stx	%fsr, [%l7 + TF_FSR]
+	wr	%g0, 0, %fprs
+	!
+	!  setup pil
+	!
+	brlz,pt	%g4, 1f
+	  nop
+#ifdef PMAP_DEBUG
+	rdpr	%pil, %l0
+	cmp	%g4, %l0
+	bge,pt	%xcc, 0f
+	  nop
+	call	panic
+0:	
+#endif
+	
+	wrpr	%g0, %g4, %pil
+1:		
+	wrpr	%g0, %g6, %tnpc
+	wrpr	%g0, 0, %gl
+	stx	%g7, [%l7 + TF_G7]		! save g7 before it can be overwritten by PCPU when returning from an interrupt 
+	wrpr	%g0, 1, %gl
+	rdpr	%cwp, %l0
+	set	TSTATE_KERNEL, %l1
+	wrpr	%l1, %l0, %tstate
+	done	
+END(tl0_utrap)
+END(tl0_trap)
+END(tl0_intr)
+
+
+/* 
+ * workaround for CPP brokenness
+ */
+#define LOADLOAD #LoadLoad
+#define LOADSTORE #LoadStore 
+#define STORESTORE #StoreStore
+
+
+#define WORKING
+#ifdef WORKING
+#define ENTER LOADLOAD
+#define EXIT LOADSTORE|STORESTORE
+#else
+#define ENTER #Sync
+#define EXIT  #Sync
+#endif
+	
+#define THE_LOCK_ENTER(addr, lock_bit, oldval, newval, label1) \
+	mov	1, lock_bit	; \
+	add	addr, 8, addr	; \
+	sllx	lock_bit, 56, lock_bit ; \
+	stxa	%o7, [PCPU(CALLER)]%asi	; /* XXX DEBUG */\
+label1:				; \
+	ldxa	[addr]%asi, oldval;	\
+	or	oldval, lock_bit, newval; \
+	andn	oldval, lock_bit, oldval; \
+	casxa	[addr]%asi, oldval, newval; \
+	cmp	newval, oldval	; \
+	bne,pn	%xcc, label1 ## b	; \
+ 	  membar ENTER	; \
+	sub	addr, 8, addr	;
+
+#define THE_LOCK_EXIT(addr, lock_bit, tmp)\
+	membar	EXIT ; \
+	ldxa	[addr + 8]%asi, tmp ; \
+	andn	tmp, lock_bit, tmp ; \
+	stxa	tmp, [addr + 8]%asi ; 
+
+#define HASH_LOOKUP(addr, tag, searchtag, faillabel, matchlabel) \
+	ldda	[addr]%asi, tag	; \
+	cmp	tag, %g0	; \
+	be,pn	%xcc, faillabel	; \
+	  nop			; \
+	cmp	tag, searchtag	; \
+	be,pn	%xcc, matchlabel ;\
+	  nop
+
+#define RESTORE_TRAPWIN(pcpu, cansave, label1, label2) \
+	brz	cansave, label1 ## f; \
+	  nop			; \
+	restore			; \
+	ba,a,pt	%xcc, label2 ## f ; \
+label1:				; \
+	rdpr	%tl, cansave	; \
+	dec	cansave		; \
+	sll	cansave, RW_SHIFT, cansave ; \
+	add	cansave, PC_TSBWBUF, cansave ; \
+	add	pcpu, cansave, cansave ; \
+	RESTORE_LOCALS_ASI(cansave) ; \
+label2:	
+
+ENTRY(hash_bucket_lock)
+	wr	%g0, ASI_N, %asi 
+	rdpr	%pstate, %o1
+	and	%o1, PSTATE_INTR_DISABLE, %o2
+	wrpr	%o2, %pstate
+	THE_LOCK_ENTER(%o0, %o3, %o4, %o5, 1)
+	mov	%o1, %o0
+	retl
+	  nop
+END(hash_bucket_lock)
+
+
+ENTRY(hash_bucket_unlock)
+	mov	1, %g2
+	wr	%g0, ASI_N, %asi
+	sllx	%g2, 56, %g2
+	THE_LOCK_EXIT(%o0, %g2, %g3)
+	wrpr	%o1, %pstate
+	retl	  
+	  nop
+END(hash_bucket_unlock)
+
+			
+! %g3==trap type
+! %g4==fault type (if data miss)
+! %g5==fault addr
+! internal usage:
+! %g1==absolute index
+! %g2==hash base, pointer to hash entry
+! %g3==flag bits, TSB (RA)
+! %g4==fault type,entry tag
+! %g5==tag
+! %g6==context
+! %g7 temp
+ENTRY(tsb_miss_handler)
+	ldxa	[%g1 + %g7]ASI_REAL, %g6	! load in the context
+
+
+	GET_HASH_SCRATCH_USER(%g2)
+	GET_TSB_SCRATCH_USER(%g4)
+
+	brnz,pn	%g6, 2f
+	  nop	
+	GET_HASH_SCRATCH_KERNEL(%g2)
+	GET_TSB_SCRATCH_KERNEL(%g4)
+2:
+
+	rdpr	%tl, %g1			! need to use real addresses?
+	mov	ASI_LDTD_N, %g3
+	wr	%g0, ASI_N, %asi
+	dec	%g1
+	GET_PCPU_SCRATCH
+
+	brz,pt	%g1, 3f			! for tl == 1
+	  nop
+	sethi	%uhi(VM_MIN_DIRECT_ADDRESS), %g1
+	wr	%g0, ASI_REAL, %asi
+	sllx	%g1, 32, %g1
+	mov	ASI_LDTD_REAL, %g3
+	andn	%g2, %g1, %g2
+	andn	%g4, %g1, %g4
+	andn	%g7, %g1, %g7
+3:	
+#ifdef notyet
+	rdpr	%cansave, %g1
+	/* XXX use save operation if %g1 > 0 and tl == 1 */
+#endif
+	rdpr	%tl, %g1
+	dec	%g1	
+	sll	%g1, RW_SHIFT, %g1
+	add	%g1, PC_TSBWBUF, %g1
+	add	PCPU_REG, %g1, %g1
+	SAVE_LOCALS_ASI(%g1)
+	mov	0, %g1			! cansave is 0
+	! %g1 == %cansave
+	! %g2 == hash scratch value
+	! %g3 == TWDW ASI
+	! %g4 == TSB RA
+	! %g5 == fault addr
+	! %g6 == context
+
+	srlx	%g5, TTARGET_VA_SHIFT, %l0
+	sllx	%g6, TTARGET_CTX_SHIFT, %l1
+	or	%l0, %l1, %l2			! %l2 == search tag
+
+tsb_miss_compute_hash_addr:
+	sethi	%hi(PAGE_SIZE), %l0
+	sub	%l0, 1, %l1			! %l1==PAGE_MASK
+
+	and	%g2, %l1, %l3			! size stored in lower 13 bits
+	andn	%g2, %l1, %g2			! actual VA/RA of hash
+
+	! XXX only handle 8k page miss
+	! calculate hash index
+	srlx	%g5, PAGE_SHIFT, %l4		! absolute hash index
+	sllx	%l3, (PAGE_SHIFT - THE_SHIFT), %l0 ! size of hash in THEs
+	sub	%l0, 1, %l5			! THE_MASK
+	and	%l4, %l5, %l5			! masked hash index
+	sllx	%l5, THE_SHIFT, %l5		! masked hash offset
+	! fetch hash entries - exit when we find what were looking for 
+
+	! %g2==entry base
+	add	%g2, %l5, %g2			! base + offset == entry base
+	
+
+	THE_LOCK_ENTER(%g2, %l0, %l7, %l6, 6)
+	
+	! %g1 == cansave
+	! %g2 == THE
+	! %g3 == TWDW ASI
+	! %g4 == TSB RA
+	! %g5 == fault addr
+	! %g6 == context
+	! %g7 == PCPU_REG
+	! %l0 == VTD_LOCK
+	! %l1 == PAGE_MASK
+	! %l2 == search tag
+	! %l4 == absolute index
+
+	! %l3 == ASI
+	! %l5 == saved head of bucket
+	! %l6 == tag
+	! %l7 == data
+
+	rd	%asi, %l3
+	wr	%g0, %g3, %asi
+	mov	%g2, %l5			! save head of bucket
+	rdpr	%tt, %g3			! reload trap type
+
+tsb_miss_lookup_0:
+	HASH_LOOKUP(%g2, %l6, %l2, tsb_miss_not_found, tsb_miss_found)
+tsb_miss_lookup_1:
+	add	%g2, 16, %g2
+	HASH_LOOKUP(%g2, %l6, %l2, tsb_miss_not_found, tsb_miss_found)
+tsb_miss_lookup_2:
+	add	%g2, 16, %g2
+	HASH_LOOKUP(%g2, %l6, %l2, tsb_miss_not_found, tsb_miss_found)
+#if HASH_ENTRY_SHIFT > 2 
+tsb_miss_lookup_3:
+	add	%g2, 16, %g2
+	HASH_LOOKUP(%g2, %l6, %l2, tsb_miss_not_found, tsb_miss_found)
+tsb_miss_lookup_4:
+	add	%g2, 16, %g2
+	HASH_LOOKUP(%g2, %l6, %l2, tsb_miss_not_found, tsb_miss_found)
+tsb_miss_lookup_5:
+	add	%g2, 16, %g2
+	HASH_LOOKUP(%g2, %l6, %l2, tsb_miss_not_found, tsb_miss_found)
+tsb_miss_lookup_6:
+	add	%g2, 16, %g2
+	HASH_LOOKUP(%g2, %l6, %l2, tsb_miss_not_found, tsb_miss_found)
+#endif			
+tsb_miss_collision:	
+	add	%g2, 16, %g2
+	ldda	[%g2]%asi, %l6
+	
+	sethi	%uhi(VM_MIN_DIRECT_ADDRESS), %g3
+	cmp	%l3, ASI_N
+	sllx	%g3, 32, %g3
+	beq,pt	%xcc, 7f
+	  nop
+	andn	%l7, %g3, %l7			! generate real address
+7:
+	srl	%l6, 0, %l6
+	sethi	%hi(0xcafebabe), %g3
+	mov	%l7, %g2
+	or	%g3, %lo(0xcafebabe), %g3
+	cmp	%g3, %l6
+	rdpr	%tt, %g3	
+	beq,pt %xcc, tsb_miss_lookup_0
+	  nop
+
+tsb_miss_not_found:	
+	! we need to jump to tl0_trap to drop us back down to tl0
+	! and take us to trap(...) to service the fault
+	wr	%g0, %l3, %asi
+	THE_LOCK_EXIT(%l5, %l0, %g2)
+
+	andn	%g5, %l1, %g5			! fault page PA
+
+	RESTORE_TRAPWIN(PCPU_REG, %g1, 14, 15)
+
+	mov	%g3, %g2			! trap type	
+        sethi	%hi(trap), %g1
+	or	%g6, %g5, %g3			! trap data
+	sub	%g0, 1, %g4			! pil info
+	ba	%xcc, tl0_trap
+	  or	%g1, %lo(trap), %g1
+
+tsb_miss_found:
+
+	wr	%g0, %l3, %asi
+	cmp	%g3, T_DATA_MISS		! TSB data miss
+	be,pt	%xcc, 9f
+	  or	%l7, VTD_REF, %l7		! set referenced unconditionally
+	cmp	%g3, T_INSTRUCTION_MISS		! TSB instruction miss
+	be,pt	%xcc, 9f
+	  nop
+	cmp	%g3, T_DATA_PROTECTION		! protection fault
+	bne,pn	%xcc, unsupported_fault_trap	! we don't handle any other fault types currently
+	  nop
+	andcc	%l7, VTD_SW_W, %g0		! write enabled?
+	bz,a,pn	%xcc, prot_fault_trap		! write to read only page
+	  nop
+	or	%l7, VTD_W, %l7			! add modifed bit 
+9:	
+
+	andn	%l7, %l0, %l7			! remove lock bit
+	
+        and	%g4, %l1, %g3			! size of TSB in pages
+	andn	%g4, %l1, %l3			! TSB real address
+	
+	sllx	%g3, (PAGE_SHIFT - TTE_SHIFT), %g3	! nttes
+	subx	%g3, 1, %g3			! TSB_MASK
+	and	%g3, %l4, %g3			! masked index
+	sllx	%g3, TTE_SHIFT, %g3		! masked byte offset
+	add	%g3, %l3, %g3			! TTE RA
+
+#if 0
+#ifdef PMAP_DEBUG
+	ldxa	[%g3]%asi, %l2
+	ldxa	[%g3 + 8]%asi, %l3
+	cmp	%l3, %l7
+	bne,pt	%xcc, 12f
+	cmp	%l2, %l6
+	bne,pt	%xcc, 12f
+	  nop
+#ifndef SMP	
+!	MAGIC_TRAP_ON;MAGIC_TRAP_ON;MAGIC_EXIT	! die if all we're doing 
+						! is storing same data
+#else
+	mov	%o0, %l2
+	mov	%o5, %g4
+	PUTCHAR(0x5a)
+	lda	[PCPU_REG + PC_CPUID]%asi, %o0
+	add	%o0, 0x30, %o0
+	PUTCHAR(%o0)
+	PUTCHAR(0x5a)	
+	mov	%l2, %o0
+	mov	%g4, %o5
+!	MAGIC_TRAP_ON; MAGIC_TRAP_OFF
+#endif
+12:
+#endif
+#endif
+	stxa	%g0, [%g3 + 8]%asi		! invalidate data
+#ifndef WORKING
+	membar #Sync
+#endif
+	stxa	%l6, [%g3]%asi			! store tag
+	stxa	%l7, [%g3 + 8]%asi		! store data
+	stxa	%l7, [%g2 + 8]%asi		! update TTE with ref bit  
+	membar	#StoreLoad
+	
+	THE_LOCK_EXIT(%l5, %l0, %l7)
+	RESTORE_TRAPWIN(PCPU_REG, %g1, 13, 16)	
+upgrade_demap:
+	rdpr	%tt, %g3
+	cmp	%g3, T_DATA_PROTECTION
+	beq,pn	%xcc, demap_begin
+	  nop
+	retry
+demap_begin:
+	sethi	%hi(PAGE_SIZE), %g1
+	sub	%g1, 1, %g1		
+	mov	%o0, %g1
+	mov	%o1, %g2
+	mov	%o2, %g3
+	mov	MAP_DTLB, %o2
+	mov	%g5, %o0
+	mov	%g6, %o1							
+	ta	MMU_UNMAP_ADDR
+	mov	%g1, %o0
+	mov	%g2, %o1
+	mov	%g3, %o2
+	retry
+END(tsb_miss_handler)
+
+
+/*
+ * Write to read-only page
+ */
+! %g1 == cansave
+! %g4 == tag
+! %g5 == fault addr
+! %g6 == context
+! %l0 == VTD_LOCK
+! %l5 == head of bucket
+	
+ENTRY(prot_fault_trap)
+	THE_LOCK_EXIT(%l5, %l0, %g2)
+	RESTORE_TRAPWIN(PCPU_REG, %g1, 14, 15)
+        sethi	%hi(trap), %g1
+	mov	T_DATA_PROTECTION, %g2			
+	or 	%g5, %g6, %g3
+	sub	%g0, 1, %g4
+	ba	%xcc, tl0_trap
+	  or	%g1, %lo(trap), %g1
+END(prot_fault_trap)
+/*
+ * Programming error
+ */
+ENTRY(unsupported_fault_trap)
+	PUTCHAR(0x5b)
+	PUTCHAR(0x5b)
+	PUTCHAR(0x5b)
+!	MAGIC_TRAP_ON;MAGIC_TRAP_ON;MAGIC_EXIT
+END(unsupported_fault_trap)
+
+
+/*
+ * Freshly forked processes come here when switched to for the first time.
+ * The arguments to fork_exit() have been setup in the locals, we must move
+ * them to the outs.
+ */
+ENTRY(fork_trampoline)
+	mov	%l0, %o0
+	mov	%l1, %o1
+	call	fork_exit
+	  mov	%l2, %o2
+	add     %sp, CCFSZ + SPOFF, %l7
+	ba,a,pt	%xcc, user_rtt
+END(fork_trampoline)
+
+
+
+	.comm	intrnames, IV_NAMLEN
+	.comm	eintrnames, 0
+
+	.comm	intrcnt, IV_MAX * 8
+	.comm	eintrcnt, 0
+
+#define	TRAP_ENTRY_SHIFT	5
+#define	TRAP_ENTRY_MASK		0x1ff
+ENTRY(tl1_trap)
+	! assume no tl1 handler
+	rdpr	%tpc, %g7
+
+	set	rtt_fill_start, %g6
+	cmp	%g7, %g6
+	blu,pn	%xcc, 1f
+	 .empty
+	set	rtt_fill_end, %g6
+	cmp	%g7, %g6
+	bgeu,pn %xcc, 1f
+	 nop
+	set	fault_rtt_fn1, %g7
+	ba,a	4f
+1:
+	set	fill_slow_start, %g6
+	cmp	%g7, %g6
+	bleu,a,pn %xcc, 2f
+	  nop
+	set	fill_slow_end, %g6
+	cmp	%g7, %g6
+	blu,a,pn %xcc, 3f
+	  nop
+2:		
+	set	tl1_end, %g6
+	cmp	%g7, %g6
+	bgeu,a,pn %xcc, ptl1_panic
+	  mov	PTL1_BAD_TRAP, %g1
+	! tpc is in the trap table
+	! convert to trap index
+	srl	%g7, TRAP_ENTRY_SHIFT, %g6
+	and	%g6, TRAP_ENTRY_MASK, %g6
+	! check for window trap type
+	and	%g6, WTRAP_TTMASK, %g6
+	cmp	%g6, WTRAP_TYPE
+	bne,a,pn %xcc, ptl1_panic
+	  mov	PTL1_BAD_TRAP, %g1
+3:		
+	andn	%g7, WTRAP_ALIGN, %g7
+	add	%g7, WTRAP_FAULTOFF, %g7
+4:	
+	wrpr	%g0, %g7, %tnpc
+	wrpr	%g0, 1, %gl
+	rdpr	%tt, %g5
+	GET_MMFSA_SCRATCH(%g7)
+	wr	%g0, ASI_REAL, %asi
+	ldxa	[%g7 + MMFSA_D_ADDR]%asi, %g6
+	ldxa	[%g7 + MMFSA_D_CTX]%asi, %g7
+	cmp	%g5, T_ALIGNMENT
+	be,pn	%xcc, 5f
+	  nop
+	srlx	%g6, PAGE_SHIFT, %g6 
+	sllx	%g6, PAGE_SHIFT, %g6	! mask off bottom
+	or	%g6, %g7, %g6
+	done
+5:
+	sllx	%g7, TRAP_CTX_SHIFT, %g7
+	or	%g7, %g5, %g5
+	done
+END(tl1_trap)
diff --git a/sys/sun4v/sun4v/fpemu.c b/sys/sun4v/sun4v/fpemu.c
new file mode 100644
index 000000000000..3cb858bbb776
--- /dev/null
+++ b/sys/sun4v/sun4v/fpemu.c
@@ -0,0 +1,191 @@
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+/* FPop1 */
+#define FMOVQ    0x0003
+#define FNEGQ    0x0007
+#define FADDQ    0x0403
+#define FADDQ    0x0407
+#define FQTOI    0x0D03
+#define FABSQ    0x000B
+#define FSQRTS   0x0209
+#define FSQRTD   0x020A
+#define FSQRTQ   0x020B
+#define FMULQ    0x040B
+#define FDIVQ    0x040F
+#define FDMULQ   0x060E
+#define FXTOQ    0x080C
+#define FQTOD    0x0C0B
+#define FITOQ    0x0C0D
+#define FSTOQ    0x0C0E
+
+/* FPop2 */
+#define FMOVQZ    0x027
+#define FMOVQLE   0x047
+#define FCMPQ     0x053
+#define FCMPEQ    0x057
+#define FMOVQLZ   0x067
+#define FMOVQNZ   0x0A7
+#define FMOVQ0    0x003
+#define FMOVQ1    0x043
+#define FMOVQ2    0x083
+#define FMOVQ3    0x0C3
+#define FMOVQI    0x103
+#define FMOVQX    0x183
+
+#define REGINFO(ftt, rd, rdu, rs2, rs2u, rs1, rs1u) \
+     reginfo = (rs1u << 2) | (rs1 << 0) | (rs2u << 5) | (rs2 << 3) \
+            | (rdu << 8) | (rd << 6) | (ftt << 9)
+
+void fpemu(struct trapframe *tf, uint64_t type, uint64_t fsr)
+{
+
+	uint32_t insn;
+
+	if (tf->tf_tstate & TSTATE_PRIV)
+		panic("unimplemented FPop in kernel");
+	if (copyin(tf->tf_tpc, &insn) != EFAULT) {
+		if ((insn & FPOP_MASK) == FPOP1) {
+			switch ((insn >> 5) & 0x1ff) {
+			case FMOVQ:
+			case FNEGQ:
+			case FABSQ: REGINFO(3,3,0,3,0,0,0); break;
+			case FSQRTQ: REGINFO(3,3,1,3,1,0,0); break;
+			case FADDQ:
+			case FSUBQ:
+			case FMULQ:
+			case FDIVQ: REGINFO(3,3,1,3,1,3,1); break;
+			case FDMULQ: REGINFO(3,3,1,2,1,2,1); break;
+			case FQTOX: REGINFO(3,2,0,3,1,0,0); break;
+			case FXTOQ: REGINFO(3,3,1,2,0,0,0); break;
+			case FQTOS: REGINFO(3,1,1,3,1,0,0); break;
+			case FQTOD: REGINFO(3,2,1,3,1,0,0); break;
+			case FITOQ: REGINFO(3,3,1,1,0,0,0); break;
+			case FSTOQ: REGINFO(3,3,1,1,1,0,0); break;
+			case FDTOQ: REGINFO(3,3,1,2,1,0,0); break;
+			case FQTOI: REGINFO(3,1,0,3,1,0,0); break;
+				/* SUBNORMAL - ftt == 2 */
+			case FSQRTS: REGINFO(2,1,1,1,1,0,0); break;
+			case FSQRTD: REGINFO(2,2,1,2,1,0,0); break;
+			case FADDD:
+			case FSUBD:
+			case FMULD:
+			case FDIVD: REGINFO(2,2,1,2,1,2,1); break;
+			case FADDS:
+			case FSUBS:
+			case FMULS:
+			case FDIVS: REGINFO(2,1,1,1,1,1,1); break;
+			case FSMULD: REGINFO(2,2,1,1,1,1,1); break;
+			case FSTOX: REGINFO(2,2,0,1,1,0,0); break;
+			case FDTOX: REGINFO(2,2,0,2,1,0,0); break;
+			case FDTOS: REGINFO(2,1,1,2,1,0,0); break;
+			case FSTOD: REGINFO(2,2,1,1,1,0,0); break;
+			case FSTOI: REGINFO(2,1,0,1,1,0,0); break;
+			case FDTOI: REGINFO(2,1,0,2,1,0,0); break;
+			}
+
+		} else if ((insn & FPOP_MASK) == FPOP2) {
+			IR = 2;
+			switch ((insn >> 5) & 0x1ff) {
+			case FCMPQ: TYPE(3,0,0,3,1,3,1); break;
+			case FCMPEQ: TYPE(3,0,0,3,1,3,1); break;
+				/* Now the conditional fmovq support */
+			case FMOVQ0:
+			case FMOVQ1:
+			case FMOVQ2:
+			case FMOVQ3:
+				/* fmovq %fccX, %fY, %fZ */
+				if (!((insn >> 11) & 3))
+					XR = tf->tf_fsr >> 10;
+				else
+					XR = tf->tf_fsr >> (30 + ((insn >> 10) & 0x6));
+				XR &= 3;
+				IR = 0;
+				switch ((insn >> 14) & 0x7) {
+					/* case 0: IR = 0; break; *//* Never */
+				case 1: if (XR) IR = 1; break;/* Not Equal */
+				case 2: if (XR == 1 || XR == 2) IR = 1; break;/* Less or Greater */
+				case 3: if (XR & 1) IR = 1; break;/* Unordered or Less */
+				case 4: if (XR == 1) IR = 1; break;/* Less */
+				case 5: if (XR & 2) IR = 1; break;/* Unordered or Greater */
+				case 6: if (XR == 2) IR = 1; break;/* Greater */
+				case 7: if (XR == 3) IR = 1; break;/* Unordered */
+				}
+				if ((insn >> 14) & 8)
+					IR ^= 1;
+				break;
+			case FMOVQI:
+			case FMOVQX:
+				/* fmovq %[ix]cc, %fY, %fZ */
+				XR = tf->tf_tstate >> 32;
+				if ((insn >> 5) & 0x80)
+					XR >>= 4;
+				XR &= 0xf;
+				IR = 0;
+				freg = ((XR >> 2) ^ XR) & 2;
+				switch ((insn >> 14) & 0x7) {
+					/* case 0: IR = 0; break; *//* Never */
+				case 1: if (XR & 4) IR = 1; break;/* Equal */
+				case 2: if ((XR & 4) || freg) IR = 1; break;/* Less or Equal */
+				case 3: if (freg) IR = 1; break;/* Less */
+				case 4: if (XR & 5) IR = 1; break;/* Less or Equal Unsigned */
+				case 5: if (XR & 1) IR = 1; break;/* Carry Set */
+				case 6: if (XR & 8) IR = 1; break;/* Negative */
+				case 7: if (XR & 2) IR = 1; break;/* Overflow Set */
+				}
+				if ((insn >> 14) & 8)
+					IR ^= 1;
+				break;
+			case FMOVQZ:
+			case FMOVQLE:
+			case FMOVQLZ:
+			case FMOVQNZ:
+			case FMOVQGZ:
+			case FMOVQGE:
+				freg = (insn >> 14) & 0x1f;
+				KASSERT(freg < 16, ("freg too large"));
+
+				if (!freg)
+					XR = 0;
+				else if (freg < 16)
+					XR = regs->u_regs[freg];
+
+#if 0 
+				else if (test_thread_flag(TIF_32BIT)) {
+					struct reg_window32 __user *win32;
+					flushw_user ();
+					win32 = (struct reg_window32 __user *)((unsigned long)((u32)regs->u_regs[UREG_FP]));
+					get_user(XR, &win32->locals[freg - 16]);
+				} else {
+					struct reg_window __user *win;
+					flushw_user ();
+					win = (struct reg_window __user *)(regs->u_regs[UREG_FP] + STACK_BIAS);
+					get_user(XR, &win->locals[freg - 16]);
+				}
+#endif 
+				IR = 0;
+				switch ((insn >> 10) & 3) {
+				case 1: if (!XR) IR = 1; break;/* Register Zero */
+				case 2: if (XR <= 0) IR = 1; break;/* Register Less Than or Equal to Zero */
+				case 3: if (XR < 0) IR = 1; break;/* Register Less Than Zero */
+				}
+				if ((insn >> 10) & 4)
+					IR ^= 1;
+				break;
+			}
+			if (IR == 0) {
+				/* The fmov test was false. Do a nop instead */
+				tf->tf_fsr &= ~(FSR_CEXC_MASK);
+				tf->tf_tpc = tf->tf_tnpc;
+				tf->tf_tnpc += 4;
+				return 1;
+			} else if (IR == 1) {
+				/* Change the instruction into plain fmovq */
+				insn = (insn & 0x3e00001f) | 0x81a00060;
+				REGINFO(3,3,0,3,0,0,0); 
+			}
+		}
+	}
+
+}
diff --git a/sys/sun4v/sun4v/gdb_machdep.c b/sys/sun4v/sun4v/gdb_machdep.c
new file mode 100644
index 000000000000..97e613560511
--- /dev/null
+++ b/sys/sun4v/sun4v/gdb_machdep.c
@@ -0,0 +1,70 @@
+/*-
+ * Copyright (c) 2004 Marcel Moolenaar
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/kdb.h>
+#include <sys/kernel.h>
+#include <sys/signal.h>
+
+#include <machine/asm.h>
+#include <machine/gdb_machdep.h>
+#include <machine/pcb.h>
+#include <machine/reg.h>
+
+#include <gdb/gdb.h>
+
+void *
+gdb_cpu_getreg(int regnum, size_t *regsz)
+{
+	static uint64_t synth;
+
+	*regsz = gdb_cpu_regsz(regnum);
+	switch (regnum) {
+		/* 0-7: g0-g7 */
+		/* 8-15: o0-o7 */
+	case 14:
+		synth = kdb_thrctx->pcb_sp - CCFSZ;
+		return (&synth);
+		/* 16-23: l0-l7 */
+		/* 24-31: i0-i7 */
+	case 30: return (&kdb_thrctx->pcb_sp);
+		/* 32-63: f0-f31 */
+		/* 64-79: f32-f62 (16 double FP) */	
+	case 80: return (&kdb_thrctx->pcb_pc);
+	}
+	return (NULL);
+}
+
+void
+gdb_cpu_setreg(int regnum, void *val)
+{
+	switch (regnum) {
+	}
+}
diff --git a/sys/sun4v/sun4v/genassym.c b/sys/sun4v/sun4v/genassym.c
new file mode 100644
index 000000000000..3ca7dfcd2111
--- /dev/null
+++ b/sys/sun4v/sun4v/genassym.c
@@ -0,0 +1,292 @@
+/*-
+ * Copyright (c) 2001 Jake Burkholder.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	from: @(#)genassym.c	5.11 (Berkeley) 5/10/91
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include "opt_kstack_pages.h"
+
+#include <sys/param.h>
+#include <sys/assym.h>
+#include <sys/errno.h>
+#include <sys/ktr.h>
+#include <sys/proc.h>
+#include <sys/queue.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
+#include <sys/signal.h>
+#include <sys/smp.h>
+#include <sys/systm.h>
+#include <sys/ucontext.h>
+#include <sys/ucontext.h>
+#include <sys/vmmeter.h>
+
+#include <vm/vm.h>
+#include <vm/vm_param.h>
+#include <vm/vm_kern.h>
+#include <vm/vm_page.h>
+#include <vm/vm_map.h>
+
+#include <machine/asi.h>
+#include <machine/cache.h>
+#include <machine/vmparam.h>
+#include <machine/cpufunc.h>
+#include <machine/fp.h>
+#include <machine/frame.h>
+#include <machine/fsr.h>
+#include <machine/intr_machdep.h>
+#include <machine/lsu.h>
+#include <machine/pcb.h>
+#include <machine/pstate.h>
+#include <machine/setjmp.h>
+#include <machine/sigframe.h>
+#include <machine/smp.h>
+#include <machine/tte.h>
+#include <machine/tlb.h>
+#include <machine/tsb.h>
+#include <machine/tstate.h>
+#include <machine/utrap.h>
+
+ASSYM(KERNBASE, KERNBASE);
+ASSYM(VM_MIN_PROM_ADDRESS, VM_MIN_PROM_ADDRESS);
+
+ASSYM(EFAULT, EFAULT);
+ASSYM(ENAMETOOLONG, ENAMETOOLONG);
+
+ASSYM(KSTACK_PAGES, KSTACK_PAGES);
+ASSYM(KSTACK_GUARD_PAGES, KSTACK_GUARD_PAGES);
+ASSYM(PCPU_PAGES, PCPU_PAGES);
+
+ASSYM(PIL_TICK, PIL_TICK);
+
+ASSYM(FPRS_DL, FPRS_DL);
+ASSYM(FPRS_DU, FPRS_DU);
+ASSYM(FPRS_FEF, FPRS_FEF);
+
+ASSYM(LSU_VW, LSU_VW);
+ASSYM(LSU_IC, LSU_IC);
+ASSYM(LSU_DC, LSU_DC);
+
+ASSYM(TAR_VPN_SHIFT, TAR_VPN_SHIFT);
+
+ASSYM(TLB_DAR_SLOT_SHIFT, TLB_DAR_SLOT_SHIFT);
+ASSYM(TLB_DEMAP_NUCLEUS, TLB_DEMAP_NUCLEUS);
+ASSYM(TLB_DEMAP_PRIMARY, TLB_DEMAP_PRIMARY);
+ASSYM(TLB_DEMAP_CONTEXT, TLB_DEMAP_CONTEXT);
+ASSYM(TLB_DEMAP_PAGE, TLB_DEMAP_PAGE);
+ASSYM(TLB_DIRECT_TO_TTE_MASK, TLB_DIRECT_TO_TTE_MASK);
+
+ASSYM(TSB_BUCKET_MASK, TSB_BUCKET_MASK);
+ASSYM(TSB_BUCKET_SHIFT, TSB_BUCKET_SHIFT);
+
+ASSYM(INT_SHIFT, INT_SHIFT);
+ASSYM(PTR_SHIFT, PTR_SHIFT);
+
+ASSYM(PAGE_SHIFT, PAGE_SHIFT);
+ASSYM(PAGE_SHIFT_8K, PAGE_SHIFT_8K);
+ASSYM(PAGE_SHIFT_4M, PAGE_SHIFT_4M);
+ASSYM(PAGE_SIZE, PAGE_SIZE);
+
+ASSYM(CPU_CLKSYNC, CPU_CLKSYNC);
+ASSYM(CPU_INIT, CPU_INIT);
+
+ASSYM(CSA_MID, offsetof(struct cpu_start_args, csa_mid));
+ASSYM(CSA_PCPU, offsetof(struct cpu_start_args, csa_pcpu));
+ASSYM(CSA_STATE, offsetof(struct cpu_start_args, csa_state));
+ASSYM(CSA_TICK, offsetof(struct cpu_start_args, csa_tick));
+ASSYM(CSA_VER, offsetof(struct cpu_start_args, csa_ver));
+ASSYM(CSA_TTES, offsetof(struct cpu_start_args, csa_ttes));
+
+ASSYM(DC_TAG_SHIFT, DC_TAG_SHIFT);
+ASSYM(DC_TAG_MASK, DC_TAG_MASK);
+ASSYM(DC_VALID_SHIFT, DC_VALID_SHIFT);
+ASSYM(DC_VALID_MASK, DC_VALID_MASK);
+ASSYM(IC_TAG_SHIFT, IC_TAG_SHIFT);
+ASSYM(IC_TAG_MASK, IC_TAG_MASK);
+ASSYM(IC_VALID_SHIFT, IC_VALID_SHIFT);
+ASSYM(IC_VALID_MASK, IC_VALID_MASK);
+
+ASSYM(DC_SIZE, offsetof(struct cacheinfo, dc_size));
+ASSYM(DC_LINESIZE, offsetof(struct cacheinfo, dc_linesize));
+ASSYM(IC_SIZE, offsetof(struct cacheinfo, ic_size));
+ASSYM(IC_LINESIZE, offsetof(struct cacheinfo, ic_linesize));
+
+ASSYM(ICA_PA, offsetof(struct ipi_cache_args, ica_pa));
+
+ASSYM(KTR_SIZEOF, sizeof(struct ktr_entry));
+ASSYM(KTR_LINE, offsetof(struct ktr_entry, ktr_line));
+ASSYM(KTR_FILE, offsetof(struct ktr_entry, ktr_file));
+ASSYM(KTR_DESC, offsetof(struct ktr_entry, ktr_desc));
+ASSYM(KTR_CPU, offsetof(struct ktr_entry, ktr_cpu));
+ASSYM(KTR_TIMESTAMP, offsetof(struct ktr_entry, ktr_timestamp));
+ASSYM(KTR_PARM1, offsetof(struct ktr_entry, ktr_parms[0]));
+ASSYM(KTR_PARM2, offsetof(struct ktr_entry, ktr_parms[1]));
+ASSYM(KTR_PARM3, offsetof(struct ktr_entry, ktr_parms[2]));
+ASSYM(KTR_PARM4, offsetof(struct ktr_entry, ktr_parms[3]));
+ASSYM(KTR_PARM5, offsetof(struct ktr_entry, ktr_parms[4]));
+ASSYM(KTR_PARM6, offsetof(struct ktr_entry, ktr_parms[5]));
+
+ASSYM(TTE_VPN, offsetof(struct tte, tte_vpn));
+ASSYM(TTE_DATA, offsetof(struct tte, tte_data));
+ASSYM(TTE_SHIFT, TTE_SHIFT);
+
+ASSYM(TD_EXEC, TD_EXEC);
+ASSYM(TD_REF, TD_REF);
+ASSYM(TD_SW, TD_SW);
+ASSYM(TD_V, TD_V);
+ASSYM(TD_8K, TD_8K);
+ASSYM(TD_CP, TD_CP);
+ASSYM(TD_CV, TD_CV);
+ASSYM(TD_L, TD_L);
+ASSYM(TD_W, TD_W);
+
+ASSYM(TS_MIN, TS_MIN);
+ASSYM(TS_MAX, TS_MAX);
+
+ASSYM(TV_SIZE_BITS, TV_SIZE_BITS);
+
+ASSYM(V_INTR, offsetof(struct vmmeter, v_intr));
+
+ASSYM(PC_CURTHREAD, offsetof(struct pcpu, pc_curthread));
+ASSYM(PC_CURPCB, offsetof(struct pcpu, pc_curpcb));
+ASSYM(PC_CPUID, offsetof(struct pcpu, pc_cpuid));
+ASSYM(PC_CPUMASK, offsetof(struct pcpu, pc_cpumask));
+ASSYM(PC_IRHEAD, offsetof(struct pcpu, pc_irhead));
+ASSYM(PC_IRTAIL, offsetof(struct pcpu, pc_irtail));
+ASSYM(PC_IRFREE, offsetof(struct pcpu, pc_irfree));
+ASSYM(PC_MID, offsetof(struct pcpu, pc_mid));
+ASSYM(PC_TLB_CTX, offsetof(struct pcpu, pc_tlb_ctx));
+ASSYM(PC_TLB_CTX_MAX, offsetof(struct pcpu, pc_tlb_ctx_max));
+ASSYM(PC_TLB_CTX_MIN, offsetof(struct pcpu, pc_tlb_ctx_min));
+ASSYM(PC_PMAP, offsetof(struct pcpu, pc_pmap));
+ASSYM(PC_CNT, offsetof(struct pcpu, pc_cnt));
+ASSYM(PC_SIZEOF, sizeof(struct pcpu));
+
+ASSYM(IH_SHIFT, IH_SHIFT);
+
+ASSYM(IRSR_BUSY, IRSR_BUSY);
+
+ASSYM(IR_NEXT, offsetof(struct intr_request, ir_next));
+ASSYM(IR_FUNC, offsetof(struct intr_request, ir_func));
+ASSYM(IR_ARG, offsetof(struct intr_request, ir_arg));
+ASSYM(IR_PRI, offsetof(struct intr_request, ir_pri));
+ASSYM(IR_VEC, offsetof(struct intr_request, ir_vec));
+
+ASSYM(ITA_MASK, offsetof(struct ipi_tlb_args, ita_mask));
+ASSYM(ITA_PMAP, offsetof(struct ipi_tlb_args, ita_pmap));
+ASSYM(ITA_START, offsetof(struct ipi_tlb_args, ita_start));
+ASSYM(ITA_END, offsetof(struct ipi_tlb_args, ita_end));
+ASSYM(ITA_VA, offsetof(struct ipi_tlb_args, ita_va));
+
+ASSYM(IV_SHIFT, IV_SHIFT);
+ASSYM(IV_FUNC, offsetof(struct intr_vector, iv_func));
+ASSYM(IV_ARG, offsetof(struct intr_vector, iv_arg));
+ASSYM(IV_PRI, offsetof(struct intr_vector, iv_pri));
+
+ASSYM(IV_NAMLEN, IV_NAMLEN);
+ASSYM(IV_MAX, IV_MAX);
+
+ASSYM(TDF_ASTPENDING, TDF_ASTPENDING);
+ASSYM(TDF_NEEDRESCHED, TDF_NEEDRESCHED);
+
+ASSYM(MD_UTRAP, offsetof(struct mdproc, md_utrap));
+
+ASSYM(MTX_LOCK, offsetof(struct mtx, mtx_lock));
+
+ASSYM(P_COMM, offsetof(struct proc, p_comm));
+ASSYM(P_MD, offsetof(struct proc, p_md));
+ASSYM(P_PID, offsetof(struct proc, p_pid));
+ASSYM(P_SFLAG, offsetof(struct proc, p_sflag));
+ASSYM(P_VMSPACE, offsetof(struct proc, p_vmspace));
+
+ASSYM(RW_SHIFT, RW_SHIFT);
+
+ASSYM(TD_FLAGS, offsetof(struct thread, td_flags));
+
+ASSYM(TD_FRAME, offsetof(struct thread, td_frame));
+ASSYM(TD_KSTACK, offsetof(struct thread, td_kstack));
+ASSYM(TD_PCB, offsetof(struct thread, td_pcb));
+ASSYM(TD_PROC, offsetof(struct thread, td_proc));
+
+ASSYM(PCB_SIZEOF, sizeof(struct pcb));
+ASSYM(PCB_RW, offsetof(struct pcb, pcb_rw));
+ASSYM(PCB_KFP, offsetof(struct pcb, pcb_kfp));
+ASSYM(PCB_UFP, offsetof(struct pcb, pcb_ufp));
+ASSYM(PCB_RWSP, offsetof(struct pcb, pcb_rwsp));
+ASSYM(PCB_FLAGS, offsetof(struct pcb, pcb_flags));
+ASSYM(PCB_NSAVED, offsetof(struct pcb, pcb_nsaved));
+ASSYM(PCB_PC, offsetof(struct pcb, pcb_pc));
+ASSYM(PCB_SP, offsetof(struct pcb, pcb_sp));
+ASSYM(PCB_FEF, PCB_FEF);
+
+ASSYM(VM_PMAP, offsetof(struct vmspace, vm_pmap));
+ASSYM(PM_ACTIVE, offsetof(struct pmap, pm_active));
+ASSYM(PM_CONTEXT, offsetof(struct pmap, pm_context));
+ASSYM(PM_TSB, offsetof(struct pmap, pm_tsb));
+
+ASSYM(SF_UC, offsetof(struct sigframe, sf_uc));
+
+ASSYM(_JB_FP, offsetof(struct _jmp_buf, _jb[_JB_FP]));
+ASSYM(_JB_PC, offsetof(struct _jmp_buf, _jb[_JB_PC]));
+ASSYM(_JB_SP, offsetof(struct _jmp_buf, _jb[_JB_SP]));
+ASSYM(_JB_SIGFLAG, offsetof(struct _jmp_buf, _jb[_JB_SIGFLAG]));
+ASSYM(_JB_SIGMASK, offsetof(struct _jmp_buf, _jb[_JB_SIGMASK]));
+
+ASSYM(TF_G0, offsetof(struct trapframe, tf_global[0]));
+ASSYM(TF_G1, offsetof(struct trapframe, tf_global[1]));
+ASSYM(TF_G2, offsetof(struct trapframe, tf_global[2]));
+ASSYM(TF_G3, offsetof(struct trapframe, tf_global[3]));
+ASSYM(TF_G4, offsetof(struct trapframe, tf_global[4]));
+ASSYM(TF_G5, offsetof(struct trapframe, tf_global[5]));
+ASSYM(TF_G6, offsetof(struct trapframe, tf_global[6]));
+ASSYM(TF_G7, offsetof(struct trapframe, tf_global[7]));
+ASSYM(TF_O0, offsetof(struct trapframe, tf_out[0]));
+ASSYM(TF_O1, offsetof(struct trapframe, tf_out[1]));
+ASSYM(TF_O2, offsetof(struct trapframe, tf_out[2]));
+ASSYM(TF_O3, offsetof(struct trapframe, tf_out[3]));
+ASSYM(TF_O4, offsetof(struct trapframe, tf_out[4]));
+ASSYM(TF_O5, offsetof(struct trapframe, tf_out[5]));
+ASSYM(TF_O6, offsetof(struct trapframe, tf_out[6]));
+ASSYM(TF_O7, offsetof(struct trapframe, tf_out[7]));
+ASSYM(TF_FPRS, offsetof(struct trapframe, tf_fprs));
+ASSYM(TF_FSR, offsetof(struct trapframe, tf_fsr));
+ASSYM(TF_GSR, offsetof(struct trapframe, tf_gsr));
+ASSYM(TF_LEVEL, offsetof(struct trapframe, tf_level));
+ASSYM(TF_PIL, offsetof(struct trapframe, tf_pil));
+ASSYM(TF_SFAR, offsetof(struct trapframe, tf_sfar));
+ASSYM(TF_SFSR, offsetof(struct trapframe, tf_sfsr));
+ASSYM(TF_TAR, offsetof(struct trapframe, tf_tar));
+ASSYM(TF_TNPC, offsetof(struct trapframe, tf_tnpc));
+ASSYM(TF_TPC, offsetof(struct trapframe, tf_tpc));
+ASSYM(TF_TSTATE, offsetof(struct trapframe, tf_tstate));
+ASSYM(TF_TYPE, offsetof(struct trapframe, tf_type));
+ASSYM(TF_Y, offsetof(struct trapframe, tf_y));
+ASSYM(TF_WSTATE, offsetof(struct trapframe, tf_wstate));
+ASSYM(TF_SIZEOF, sizeof(struct trapframe));
+
+ASSYM(UT_MAX, UT_MAX);
diff --git a/sys/sun4v/sun4v/hcall.S b/sys/sun4v/sun4v/hcall.S
new file mode 100644
index 000000000000..0d1219458c3c
--- /dev/null
+++ b/sys/sun4v/sun4v/hcall.S
@@ -0,0 +1,1437 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License").  You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#pragma ident	"@(#)hcall.s	1.10	05/09/28 SMI"
+
+/*
+ * Hypervisor calls
+ */
+#define _ASM
+
+#include <machine/asm.h>
+__FBSDID("$FreeBSD$")				
+
+#include <machine/asi.h>
+#include <machine/asmacros.h>
+#include <machine/hypervisor_api.h>
+#include <machine/pstate.h>
+
+#include "assym.s"
+
+#if defined(lint)
+#error lint
+#endif
+#if defined(__lint)
+#error __lint
+#endif	
+#if defined(lint) || defined(__lint)
+
+/*ARGSUSED*/
+int64_t
+hv_cnputchar(uint8_t ch)
+{ return (0); }
+
+/*ARGSUSED*/
+int64_t
+hv_cngetchar(uint8_t *ch)
+{ return (0); }
+
+/*ARGSUSED*/
+uint64_t
+hv_tod_get(uint64_t *seconds)
+{ return (0); }
+
+/*ARGSUSED*/
+uint64_t
+hv_tod_set(uint64_t seconds)
+{ return (0);}
+
+/*ARGSUSED*/
+uint64_t
+hv_mmu_map_perm_addr(void *vaddr, int ctx, uint64_t tte, int flags)
+{ return (0); }
+
+/*ARGSUSED*/
+uint64_t
+hv_mmu_unmap_perm_addr(void *vaddr, int ctx, int flags)
+{ return (0); }
+
+/*ARGSUSED*/
+uint64_t
+hv_set_ctx0(uint64_t ntsb_descriptor, uint64_t desc_ra)
+{ return (0); }
+
+/*ARGSUSED*/
+uint64_t
+hv_set_ctxnon0(uint64_t ntsb_descriptor, uint64_t desc_ra)
+{ return (0); }
+
+#ifdef SET_MMU_STATS
+/*ARGSUSED*/
+uint64_t
+hv_mmu_set_stat_area(uint64_t rstatarea, uint64_t size)
+{ return (0); }
+#endif /* SET_MMU_STATS */
+
+/*ARGSUSED*/
+uint64_t
+hv_cpu_qconf(int queue, uint64_t paddr, int size)
+{ return (0); }
+
+/*ARGSUSED*/
+uint64_t
+hvio_config_get(devhandle_t dev_hdl, pci_device_t bdf,
+    pci_config_offset_t off, pci_config_size_t size, pci_cfg_data_t *data_p)
+{ return (0); }
+
+/*ARGSUSED*/
+uint64_t
+hvio_config_put(devhandle_t dev_hdl, pci_device_t bdf,
+    pci_config_offset_t off, pci_config_size_t size, pci_cfg_data_t data)
+{ return (0); }
+
+/*ARGSUSED*/
+uint64_t
+hvio_intr_devino_to_sysino(uint64_t dev_hdl, uint32_t devino, uint64_t *sysino)
+{ return (0); }
+
+/*ARGSUSED*/
+uint64_t
+hvio_intr_getvalid(uint64_t sysino, int *intr_valid_state)
+{ return (0); }
+
+/*ARGSUSED*/
+uint64_t
+hvio_intr_setvalid(uint64_t sysino, int intr_valid_state)
+{ return (0); }
+
+/*ARGSUSED*/
+uint64_t
+hvio_intr_getstate(uint64_t sysino, int *intr_state)
+{ return (0); }
+
+/*ARGSUSED*/
+uint64_t
+hvio_intr_setstate(uint64_t sysino, int intr_state)
+{ return (0); }
+
+/*ARGSUSED*/
+uint64_t
+hvio_intr_gettarget(uint64_t sysino, uint32_t *cpuid)
+{ return (0); }
+
+/*ARGSUSED*/
+uint64_t
+hvio_intr_settarget(uint64_t sysino, uint32_t cpuid)
+{ return (0); }
+
+/*ARGSUSED*/
+uint64_t
+hvio_iommu_map(devhandle_t dev_hdl, tsbid_t tsbid,
+    pages_t pages, io_attributes_t io_attributes,
+    io_page_list_t *io_page_list_p, pages_t *pages_mapped)
+{ return (0); }
+
+/*ARGSUSED*/
+uint64_t
+hvio_iommu_demap(devhandle_t dev_hdl, tsbid_t tsbid,
+    pages_t pages, pages_t *pages_demapped)
+{ return (0); }
+
+/*ARGSUSED*/
+uint64_t
+hvio_iommu_getmap(devhandle_t dev_hdl, tsbid_t tsbid,
+    io_attributes_t *attributes_p, r_addr_t *r_addr_p)
+{ return (0); }
+
+/*ARGSUSED*/
+uint64_t
+hvio_iommu_getbypass(devhandle_t dev_hdl, r_addr_t ra,
+    io_attributes_t io_attributes, io_addr_t *io_addr_p)
+{ return (0); }
+
+/*ARGSUSED*/
+uint64_t
+hvio_peek(devhandle_t dev_hdl, r_addr_t ra, size_t size, uint32_t *status,
+    uint64_t *data_p)
+{ return (0); }
+
+/*ARGSUSED*/
+uint64_t
+hvio_poke(devhandle_t dev_hdl, r_addr_t ra, uint64_t sizes, uint64_t data,
+    r_addr_t ra2, uint32_t *rdbk_status)
+{ return (0); }
+
+/*ARGSUSED*/
+uint64_t
+hvio_dma_sync(devhandle_t dev_hdl, r_addr_t ra, size_t num_bytes,
+    int io_sync_direction, size_t *bytes_synched)
+{ return (0); }
+
+/*ARGSUSED*/
+uint64_t
+hvio_msiq_conf(devhandle_t dev_hdl, msiqid_t msiq_id, r_addr_t ra,
+    uint_t msiq_rec_cnt)
+{ return (0); }
+
+/*ARGSUSED*/
+uint64_t
+hvio_msiq_info(devhandle_t dev_hdl, msiqid_t msiq_id, r_addr_t *r_addr_p,
+    uint_t *msiq_rec_cnt_p)
+{ return (0); }
+	
+/*ARGSUSED*/
+uint64_t
+hvio_msiq_getvalid(devhandle_t dev_hdl, msiqid_t msiq_id,
+    pci_msiq_valid_state_t *msiq_valid_state)
+{ return (0); }
+
+/*ARGSUSED*/
+uint64_t
+hvio_msiq_setvalid(devhandle_t dev_hdl, msiqid_t msiq_id,
+    pci_msiq_valid_state_t msiq_valid_state)
+{ return (0); }
+
+/*ARGSUSED*/
+uint64_t
+hvio_msiq_getstate(devhandle_t dev_hdl, msiqid_t msiq_id,
+    pci_msiq_state_t *msiq_state)
+{ return (0); }
+
+/*ARGSUSED*/
+uint64_t
+hvio_msiq_setstate(devhandle_t dev_hdl, msiqid_t msiq_id,
+    pci_msiq_state_t msiq_state)
+{ return (0); }
+
+/*ARGSUSED*/
+uint64_t
+hvio_msiq_gethead(devhandle_t dev_hdl, msiqid_t msiq_id,
+    msiqhead_t *msiq_head)
+{ return (0); }
+
+/*ARGSUSED*/
+uint64_t
+hvio_msiq_sethead(devhandle_t dev_hdl, msiqid_t msiq_id,
+    msiqhead_t msiq_head)
+{ return (0); }
+
+/*ARGSUSED*/
+uint64_t
+hvio_msiq_gettail(devhandle_t dev_hdl, msiqid_t msiq_id,
+    msiqtail_t *msiq_tail)
+{ return (0); }
+
+/*ARGSUSED*/
+uint64_t
+hvio_msi_getmsiq(devhandle_t dev_hdl, msinum_t msi_num,
+    msiqid_t *msiq_id)
+{ return (0); }
+
+/*ARGSUSED*/
+uint64_t
+hvio_msi_setmsiq(devhandle_t dev_hdl, msinum_t msi_num,
+    msiqid_t msiq_id, msi_type_t msitype)
+{ return (0); }
+
+/*ARGSUSED*/
+uint64_t
+hvio_msi_getvalid(devhandle_t dev_hdl, msinum_t msi_num,
+    pci_msi_valid_state_t *msi_valid_state)
+{ return (0); }
+
+/*ARGSUSED*/
+uint64_t
+hvio_msi_setvalid(devhandle_t dev_hdl, msinum_t msi_num,
+    pci_msi_valid_state_t msi_valid_state)
+{ return (0); }
+
+/*ARGSUSED*/
+uint64_t
+hvio_msi_getstate(devhandle_t dev_hdl, msinum_t msi_num,
+    pci_msi_state_t *msi_state)
+{ return (0); }
+
+/*ARGSUSED*/
+uint64_t
+hvio_msi_setstate(devhandle_t dev_hdl, msinum_t msi_num,
+    pci_msi_state_t msi_state)
+{ return (0); }
+
+/*ARGSUSED*/
+uint64_t
+hvio_msg_getmsiq(devhandle_t dev_hdl, pcie_msg_type_t msg_type,
+    msiqid_t *msiq_id)
+{ return (0); }
+
+/*ARGSUSED*/
+uint64_t
+hvio_msg_setmsiq(devhandle_t dev_hdl, pcie_msg_type_t msg_type,
+    msiqid_t msiq_id)
+{ return (0); }
+
+/*ARGSUSED*/
+uint64_t
+hvio_msg_getvalid(devhandle_t dev_hdl, pcie_msg_type_t msg_type,
+    pcie_msg_valid_state_t *msg_valid_state)
+{ return (0); }
+
+/*ARGSUSED*/
+uint64_t
+hvio_msg_setvalid(devhandle_t dev_hdl, pcie_msg_type_t msg_type,
+    pcie_msg_valid_state_t msg_valid_state)
+{ return (0); }
+
+uint64_t
+hv_cpu_yield(void)
+{ return (0); }
+
+/*ARGSUSED*/
+uint64_t
+hv_service_recv(uint64_t s_id, uint64_t buf_pa, uint64_t size,
+    uint64_t *recv_bytes)
+{ return (0); }
+
+/*ARGSUSED*/
+uint64_t
+hv_service_send(uint64_t s_id, uint64_t buf_pa, uint64_t size,
+    uint64_t *send_bytes)
+{ return (0); }
+
+/*ARGSUSED*/
+uint64_t
+hv_service_getstatus(uint64_t s_id, uint64_t *vreg)
+{ return (0); }
+
+/*ARGSUSED*/
+uint64_t
+hv_service_setstatus(uint64_t s_id, uint64_t bits)
+{ return (0); }
+
+/*ARGSUSED*/
+uint64_t
+hv_service_clrstatus(uint64_t s_id, uint64_t bits)
+{ return (0); }
+
+/*ARGSUSED*/
+uint64_t
+hv_cpu_state(uint64_t cpuid, uint64_t *cpu_state)
+{ return (0); }
+
+/*ARGSUSED*/
+uint64_t
+hv_dump_buf_update(uint64_t paddr, uint64_t size, uint64_t *minsize)
+{ return (0); }
+
+/*ARGSUSED*/
+uint64_t
+hv_mem_scrub(uint64_t real_addr, uint64_t length, uint64_t *scrubbed_len)
+{ return (0); }
+
+/*ARGSUSED*/
+uint64_t
+hv_mem_sync(uint64_t real_addr, uint64_t length, uint64_t *flushed_len)
+{ return (0); }
+
+/*ARGSUSED*/
+uint64_t
+hv_ttrace_buf_conf(uint64_t paddr, uint64_t size, uint64_t *size1)
+{ return (0); }
+
+/*ARGSUSED*/
+uint64_t
+hv_ttrace_buf_info(uint64_t *paddr, uint64_t *size)
+{ return (0); }
+
+/*ARGSUSED*/
+uint64_t
+hv_ttrace_enable(uint64_t enable, uint64_t *prev_enable)
+{ return (0); }
+
+/*ARGSUSED*/
+uint64_t
+hv_ttrace_freeze(uint64_t freeze, uint64_t *prev_freeze)
+{ return (0); }
+
+/*ARGSUSED*/
+uint64_t
+hv_mach_desc(uint64_t buffer_ra, uint64_t *buffer_sizep)
+{ return (0); }
+	
+/*ARGSUSED*/	
+uint64_t
+hv_ncs_request(int cmd, uint64_t realaddr, size_t sz)
+{ return (0); }
+
+/*ARGSUSED*/	
+uint64_t
+hv_ra2pa(uint64_t ra)
+{ return (0); }
+
+/*ARGSUSED*/	
+uint64_t
+hv_hpriv(void *func, uint64_t arg1, uint64_t arg2, uint64_t arg3)
+{ return (0); }
+
+#else	/* lint || __lint */
+
+	/*
+	 * %o0 - character
+	 */
+	ENTRY(hv_cnputchar)
+	mov	CONS_WRITE, %o5
+	ta	FAST_TRAP
+	retl
+	  nop
+	SET_SIZE(hv_cnputchar)
+
+	/*
+	 * %o0 pointer to character buffer
+	 * return values:
+	 * 0 success
+	 * hv_errno failure
+	 */
+	ENTRY(hv_cngetchar)
+	mov	%o0, %o2
+	mov	CONS_READ, %o5
+	ta	FAST_TRAP
+	brnz,a	%o0, 1f		! failure, just return error
+	mov	1, %o0
+
+	cmp	%o1, H_BREAK
+	be	1f
+	mov	%o1, %o0
+
+	cmp	%o1, H_HUP
+	be	1f
+	mov	%o1, %o0
+
+	stb	%o1, [%o2]	! success, save character and return 0
+	mov	0, %o0
+1:
+	retl
+	nop
+	SET_SIZE(hv_cngetchar)
+
+	ENTRY(hv_tod_get)
+	mov	%o0, %o4
+	mov	TOD_GET, %o5
+	ta	FAST_TRAP
+	retl
+	  stx	%o1, [%o4] 
+	SET_SIZE(hv_tod_get)
+
+	ENTRY(hv_tod_set)
+	mov	TOD_SET, %o5
+	ta	FAST_TRAP
+	retl
+	nop
+	SET_SIZE(hv_tod_set)
+
+	/*
+	 * Map permanent address
+	 * arg0 vaddr (%o0)
+	 * arg1 context (%o1)
+	 * arg2 tte (%o2)
+	 * arg3 flags (%o3)  0x1=d 0x2=i
+	 */
+	ENTRY(hv_mmu_map_perm_addr)
+	mov	MAP_PERM_ADDR, %o5
+	ta	FAST_TRAP
+	retl
+	nop
+	SET_SIZE(hv_mmu_map_perm_addr)
+
+	/*
+	 * Unmap permanent address
+	 * arg0 vaddr (%o0)
+	 * arg1 context (%o1)
+	 * arg2 flags (%o2)  0x1=d 0x2=i
+	 */
+	ENTRY(hv_mmu_unmap_perm_addr)
+	mov	UNMAP_PERM_ADDR, %o5
+	ta	FAST_TRAP
+	retl
+	nop
+	SET_SIZE(hv_mmu_unmap_perm_addr)
+
+	/*
+	 * Set TSB for context 0
+	 * arg0 ntsb_descriptor (%o0)
+	 * arg1 desc_ra (%o1)
+	 */
+	ENTRY(hv_set_ctx0)
+	mov	MMU_TSB_CTX0, %o5
+	ta	FAST_TRAP
+	retl
+	nop
+	SET_SIZE(hv_set_ctx0)
+
+	/*
+	 * Set TSB for context non0
+	 * arg0 ntsb_descriptor (%o0)
+	 * arg1 desc_ra (%o1)
+	 */
+	ENTRY(hv_set_ctxnon0)
+	mov	MMU_TSB_CTXNON0, %o5
+	ta	FAST_TRAP
+	retl
+	nop
+	SET_SIZE(hv_set_ctxnon0)
+
+#ifdef SET_MMU_STATS
+	/*
+	 * Returns old stat area on success
+	 */
+	ENTRY(hv_mmu_set_stat_area)
+	mov	MMU_STAT_AREA, %o5
+	ta	FAST_TRAP
+	retl
+	nop
+	SET_SIZE(hv_mmu_set_stat_area)
+#endif /* SET_MMU_STATS */
+
+	/*
+	 * CPU Q Configure
+	 * arg0 queue (%o0)
+	 * arg1 Base address RA (%o1)
+	 * arg2 Size (%o2)
+	 */
+	ENTRY(hv_cpu_qconf)
+	mov	CPU_QCONF, %o5
+	ta	FAST_TRAP
+	retl
+	nop
+	SET_SIZE(hv_cpu_qconf)
+
+	/*
+	 * CPU Mondo Send
+	 * arg0-1 cpulist (%o0, %o1)
+	 * arg2 data (%o2)
+	 *
+	 */
+	ENTRY(hv_cpu_mondo_send)
+	ldx	[PCPU(MONDO_DATA_RA)], %o2
+	mov	HV_INTR_SEND, %o5
+	ta	FAST_TRAP
+	retl
+	 membar	#Sync
+	END(hv_cpu_mondo_send)
+		
+	/*
+	 * arg0 - devhandle
+	 * arg1 - pci_device
+	 * arg2 - pci_config_offset
+	 * arg3 - pci_config_size
+	 *
+	 * ret0 - status
+	 * ret1 - error_flag
+	 * ret2 - pci_cfg_data
+	 */
+	ENTRY(hvio_config_get)
+	mov	HVIO_CONFIG_GET, %o5
+	ta	FAST_TRAP
+	brnz	%o0, 1f
+	movrnz	%o1, -1, %o2
+	brz,a	%o1, 1f
+	stuw	%o2, [%o4]
+1:	retl
+	nop
+	SET_SIZE(hvio_config_get)
+
+	/*
+	 * arg0 - devhandle
+	 * arg1 - pci_device
+	 * arg2 - pci_config_offset
+	 * arg3 - pci_config_size
+	 * arg4 - pci_cfg_data
+	 *
+	 * ret0 - status
+	 * ret1 - error_flag
+	 */
+	ENTRY(hvio_config_put)
+	mov	HVIO_CONFIG_PUT, %o5
+	ta	FAST_TRAP
+	retl
+	nop
+	SET_SIZE(hvio_config_put)
+
+	/*
+	 * arg0 - devhandle
+	 * arg1 - devino
+	 *
+	 * ret0 - status
+	 * ret1 - sysino
+	 */
+	ENTRY(hvio_intr_devino_to_sysino)
+	mov	HVIO_INTR_DEVINO2SYSINO, %o5
+	ta	FAST_TRAP
+	brz,a	%o0, 1f
+	stx	%o1, [%o2]
+1:	retl
+	nop
+	SET_SIZE(hvio_intr_devino_to_sysino)
+
+	/*
+	 * arg0 - sysino
+	 *
+	 * ret0 - status
+	 * ret1 - intr_valid_state
+	 */
+	ENTRY(hvio_intr_getvalid)
+	mov	%o1, %o2
+	mov	HVIO_INTR_GETENABLED, %o5
+	ta	FAST_TRAP
+	brz,a	%o0, 1f
+	stuw	%o1, [%o2]
+1:	retl
+	nop
+	SET_SIZE(hvio_intr_getvalid)
+
+	/*
+	 * arg0 - sysino
+	 * arg1 - intr_valid_state
+	 *
+	 * ret0 - status
+	 */
+	ENTRY(hvio_intr_setvalid)
+	mov	HVIO_INTR_SETENABLED, %o5
+	ta	FAST_TRAP
+	retl
+	nop
+	SET_SIZE(hvio_intr_setvalid)
+
+	/*
+	 * arg0 - sysino
+	 *
+	 * ret0 - status
+	 * ret1 - intr_state
+	 */
+	ENTRY(hvio_intr_getstate)
+	mov	%o1, %o2
+	mov	HVIO_INTR_GETSTATE, %o5
+	ta	FAST_TRAP
+	brz,a	%o0, 1f
+	stuw	%o1, [%o2]
+1:	retl
+	nop
+	SET_SIZE(hvio_intr_getstate)
+
+	/*
+	 * arg0 - sysino
+	 * arg1 - intr_state
+	 *
+	 * ret0 - status
+	 */
+	ENTRY(hvio_intr_setstate)
+	mov	HVIO_INTR_SETSTATE, %o5
+	ta	FAST_TRAP
+	retl
+	nop
+	SET_SIZE(hvio_intr_setstate)
+
+	/*
+	 * arg0 - sysino
+	 *
+	 * ret0 - status
+	 * ret1 - cpu_id
+	 */
+	ENTRY(hvio_intr_gettarget)
+	mov	%o1, %o2
+	mov	HVIO_INTR_GETTARGET, %o5
+	ta	FAST_TRAP
+	brz,a	%o0, 1f
+	stuw	%o1, [%o2]
+1:	retl
+	nop
+	SET_SIZE(hvio_intr_gettarget)
+
+	/*
+	 * arg0 - sysino
+	 * arg1 - cpu_id
+	 *
+	 * ret0 - status
+	 */
+	ENTRY(hvio_intr_settarget)
+	mov	HVIO_INTR_SETTARGET, %o5
+	ta	FAST_TRAP
+	retl
+	nop
+	SET_SIZE(hvio_intr_settarget)
+
+	/*
+	 * arg0 - devhandle
+	 * arg1 - tsbid
+	 * arg2 - pages
+	 * arg3 - io_attributes
+	 * arg4 - io_page_list_p
+	 *
+	 * ret1 - pages_mapped
+	 */
+	ENTRY(hvio_iommu_map)
+	save	%sp, -SA(MINFRAME64), %sp
+	mov	%i0, %o0
+	mov	%i1, %o1
+	mov	%i2, %o2
+	mov	%i3, %o3
+	mov	%i4, %o4
+	mov	HVIO_IOMMU_MAP, %o5
+	ta	FAST_TRAP
+	brnz	%o0, 1f
+	mov	%o0, %i0
+	stuw	%o1, [%i5]
+1:
+	ret
+	restore
+	SET_SIZE(hvio_iommu_map)
+
+	/*
+	 * arg0 - devhandle
+	 * arg1 - tsbid
+	 * arg2 - pages
+	 *
+	 * ret1 - pages_demapped
+	 */
+	ENTRY(hvio_iommu_demap)
+	mov	HVIO_IOMMU_DEMAP, %o5
+	ta	FAST_TRAP
+	brz,a	%o0, 1f
+	stuw	%o1, [%o3]
+1:	retl
+	nop
+	SET_SIZE(hvio_iommu_demap)
+
+	/*
+	 * arg0 - devhandle
+	 * arg1 - tsbid
+	 *
+	 *
+	 * ret0 - status
+	 * ret1 - io_attributes
+	 * ret2 - r_addr
+	 */
+	ENTRY(hvio_iommu_getmap)
+	mov	%o2, %o4
+	mov	HVIO_IOMMU_GETMAP, %o5
+	ta	FAST_TRAP
+	brnz	%o0, 1f
+	nop
+	stx	%o2, [%o3]
+	st	%o1, [%o4]
+1:
+	retl
+	nop
+	SET_SIZE(hvio_iommu_getmap)
+
+	/*
+	 * arg0 - devhandle
+	 * arg1 - r_addr
+	 * arg2 - io_attributes
+	 *
+	 *
+	 * ret0 - status
+	 * ret1 - io_addr
+	 */
+	ENTRY(hvio_iommu_getbypass)
+	mov	HVIO_IOMMU_GETBYPASS, %o5
+	ta	FAST_TRAP
+	brz,a	%o0, 1f
+	stx	%o1, [%o3]
+1:	retl
+	nop
+	SET_SIZE(hvio_iommu_getbypass)
+
+	/*
+	 * arg0 - devhandle
+	 * arg1 - r_addr
+	 * arg2 - size
+	 *
+	 * ret1 - error_flag
+	 * ret2 - data
+	 */
+	ENTRY(hvio_peek)
+	mov	HVIO_PEEK, %o5
+	ta	FAST_TRAP
+	brnz	%o0, 1f
+	nop
+	stx	%o2, [%o4]
+	st	%o1, [%o3]
+1:
+	retl
+	nop
+	SET_SIZE(hvio_peek)
+
+	/*
+	 * arg0 - devhandle
+	 * arg1 - r_addr
+	 * arg2 - sizes
+	 * arg3 - data
+	 * arg4 - r_addr2
+	 *
+	 * ret1 - error_flag
+	 */
+	ENTRY(hvio_poke)
+	save	%sp, -SA(MINFRAME64), %sp
+	mov	%i0, %o0
+	mov	%i1, %o1
+	mov	%i2, %o2
+	mov	%i3, %o3
+	mov	%i4, %o4
+	mov	HVIO_POKE, %o5
+	ta	FAST_TRAP
+	brnz	%o0, 1f
+	mov	%o0, %i0
+	stuw	%o1, [%i5]
+1:
+	ret
+	restore
+	SET_SIZE(hvio_poke)
+
+	/*
+	 * arg0 - devhandle
+	 * arg1 - r_addr
+	 * arg2 - num_bytes
+	 * arg3 - io_sync_direction
+	 *
+	 * ret0 - status
+	 * ret1 - bytes_synched
+	 */
+	ENTRY(hvio_dma_sync)
+	mov	HVIO_DMA_SYNC, %o5
+	ta	FAST_TRAP
+	brz,a	%o0, 1f
+	stx	%o1, [%o4]
+1:	retl
+	nop
+	SET_SIZE(hvio_dma_sync)
+
+	/*
+	 * arg0 - devhandle
+	 * arg1 - msiq_id
+	 * arg2 - r_addr
+	 * arg3 - nentries
+	 *
+	 * ret0 - status
+	 */
+	ENTRY(hvio_msiq_conf)
+	mov	HVIO_MSIQ_CONF, %o5
+	ta	FAST_TRAP
+	retl
+	nop
+	SET_SIZE(hvio_msiq_conf)
+
+	/*
+	 * arg0 - devhandle
+	 * arg1 - msiq_id
+	 *
+	 * ret0 - status
+	 * ret1 - r_addr
+	 * ret1 - nentries
+	 */
+	ENTRY(hvio_msiq_info)
+	mov     %o2, %o4
+	mov     HVIO_MSIQ_INFO, %o5
+	ta      FAST_TRAP
+	brnz    %o0, 1f
+	nop
+	stx     %o1, [%o4]
+	stuw    %o2, [%o3]
+1:      retl
+	nop
+	SET_SIZE(hvio_msiq_info)
+
+	/*
+	 * arg0 - devhandle
+	 * arg1 - msiq_id
+	 *
+	 * ret0 - status
+	 * ret1 - msiq_valid_state
+	 */
+	ENTRY(hvio_msiq_getvalid)
+	mov	HVIO_MSIQ_GETVALID, %o5
+	ta	FAST_TRAP
+	brz,a	%o0, 1f
+	stuw	%o1, [%o2]
+1:	retl
+	nop
+	SET_SIZE(hvio_msiq_getvalid)
+
+	/*
+	 * arg0 - devhandle
+	 * arg1 - msiq_id
+	 * arg2 - msiq_valid_state
+	 *
+	 * ret0 - status
+	 */
+	ENTRY(hvio_msiq_setvalid)
+	mov	HVIO_MSIQ_SETVALID, %o5
+	ta	FAST_TRAP
+	retl
+	nop
+	SET_SIZE(hvio_msiq_setvalid)
+
+	/*
+	 * arg0 - devhandle
+	 * arg1 - msiq_id
+	 *
+	 * ret0 - status
+	 * ret1 - msiq_state
+	 */
+	ENTRY(hvio_msiq_getstate)
+	mov	HVIO_MSIQ_GETSTATE, %o5
+	ta	FAST_TRAP
+	brz,a	%o0, 1f
+	stuw	%o1, [%o2]
+1:	retl
+	nop
+	SET_SIZE(hvio_msiq_getstate)
+
+	/*
+	 * arg0 - devhandle
+	 * arg1 - msiq_id
+	 * arg2 - msiq_state
+	 *
+	 * ret0 - status
+	 */
+	ENTRY(hvio_msiq_setstate)
+	mov	HVIO_MSIQ_SETSTATE, %o5
+	ta	FAST_TRAP
+	retl
+	nop
+	SET_SIZE(hvio_msiq_setstate)
+
+	/*
+	 * arg0 - devhandle
+	 * arg1 - msiq_id
+	 *
+	 * ret0 - status
+	 * ret1 - msiq_head
+	 */
+	ENTRY(hvio_msiq_gethead)
+	mov	HVIO_MSIQ_GETHEAD, %o5
+	ta	FAST_TRAP
+	brz,a	%o0, 1f
+	stx	%o1, [%o2]
+1:	retl
+	nop
+	SET_SIZE(hvio_msiq_gethead)
+
+	/*
+	 * arg0 - devhandle
+	 * arg1 - msiq_id
+	 * arg2 - msiq_head
+	 *
+	 * ret0 - status
+	 */
+	ENTRY(hvio_msiq_sethead)
+	mov	HVIO_MSIQ_SETHEAD, %o5
+	ta	FAST_TRAP
+	retl
+	nop
+	SET_SIZE(hvio_msiq_sethead)
+
+	/*
+	 * arg0 - devhandle
+	 * arg1 - msiq_id
+	 *
+	 * ret0 - status
+	 * ret1 - msiq_tail
+	 */
+	ENTRY(hvio_msiq_gettail)
+	mov	HVIO_MSIQ_GETTAIL, %o5
+	ta	FAST_TRAP
+	brz,a	%o0, 1f
+	stx	%o1, [%o2]
+1:	retl
+	nop
+	SET_SIZE(hvio_msiq_gettail)
+
+	/*
+	 * arg0 - devhandle
+	 * arg1 - msi_num
+	 *
+	 * ret0 - status
+	 * ret1 - msiq_id
+	 */
+	ENTRY(hvio_msi_getmsiq)
+	mov	HVIO_MSI_GETMSIQ, %o5
+	ta	FAST_TRAP
+	brz,a	%o0, 1f
+	stuw	%o1, [%o2]
+1:	retl
+	nop
+	SET_SIZE(hvio_msi_getmsiq)
+
+	/*
+	 * arg0 - devhandle
+	 * arg1 - msi_num
+	 * arg2 - msiq_id
+	 * arg2 - msitype
+	 *
+	 * ret0 - status
+	 */
+	ENTRY(hvio_msi_setmsiq)
+	mov	HVIO_MSI_SETMSIQ, %o5
+	ta	FAST_TRAP
+	retl
+	nop
+	SET_SIZE(hvio_msi_setmsiq)
+
+	/*
+	 * arg0 - devhandle
+	 * arg1 - msi_num
+	 *
+	 * ret0 - status
+	 * ret1 - msi_valid_state
+	 */
+	ENTRY(hvio_msi_getvalid)
+	mov	HVIO_MSI_GETVALID, %o5
+	ta	FAST_TRAP
+	brz,a	%o0, 1f
+	stuw	%o1, [%o2]
+1:	retl
+	nop
+	SET_SIZE(hvio_msi_getvalid)
+
+	/*
+	 * arg0 - devhandle
+	 * arg1 - msi_num
+	 * arg2 - msi_valid_state
+	 *
+	 * ret0 - status
+	 */
+	ENTRY(hvio_msi_setvalid)
+	mov	HVIO_MSI_SETVALID, %o5
+	ta	FAST_TRAP
+	retl
+	nop
+	SET_SIZE(hvio_msi_setvalid)
+
+	/*
+	 * arg0 - devhandle
+	 * arg1 - msi_num
+	 *
+	 * ret0 - status
+	 * ret1 - msi_state
+	 */
+	ENTRY(hvio_msi_getstate)
+	mov	HVIO_MSI_GETSTATE, %o5
+	ta	FAST_TRAP
+	brz,a	%o0, 1f
+	stuw	%o1, [%o2]
+1:	retl
+	nop
+	SET_SIZE(hvio_msi_getstate)
+
+	/*
+	 * arg0 - devhandle
+	 * arg1 - msi_num
+	 * arg2 - msi_state
+	 *
+	 * ret0 - status
+	 */
+	ENTRY(hvio_msi_setstate)
+	mov	HVIO_MSI_SETSTATE, %o5
+	ta	FAST_TRAP
+	retl
+	nop
+	SET_SIZE(hvio_msi_setstate)
+
+	/*
+	 * arg0 - devhandle
+	 * arg1 - msg_type
+	 *
+	 * ret0 - status
+	 * ret1 - msiq_id
+	 */
+	ENTRY(hvio_msg_getmsiq)
+	mov	HVIO_MSG_GETMSIQ, %o5
+	ta	FAST_TRAP
+	brz,a	%o0, 1f
+	stuw	%o1, [%o2]
+1:	retl
+	nop
+	SET_SIZE(hvio_msg_getmsiq)
+
+	/*
+	 * arg0 - devhandle
+	 * arg1 - msg_type
+	 * arg2 - msiq_id
+	 *
+	 * ret0 - status
+	 */
+	ENTRY(hvio_msg_setmsiq)
+	mov	HVIO_MSG_SETMSIQ, %o5
+	ta	FAST_TRAP
+	retl
+	nop
+	SET_SIZE(hvio_msg_setmsiq)
+
+	/*
+	 * arg0 - devhandle
+	 * arg1 - msg_type
+	 *
+	 * ret0 - status
+	 * ret1 - msg_valid_state
+	 */
+	ENTRY(hvio_msg_getvalid)
+	mov	HVIO_MSG_GETVALID, %o5
+	ta	FAST_TRAP
+	brz,a	%o0, 1f
+	stuw	%o1, [%o2]
+1:	retl
+	nop
+	SET_SIZE(hvio_msg_getvalid)
+
+	/*
+	 * arg0 - devhandle
+	 * arg1 - msg_type
+	 * arg2 - msg_valid_state
+	 *
+	 * ret0 - status
+	 */
+	ENTRY(hvio_msg_setvalid)
+	mov	HVIO_MSG_SETVALID, %o5
+	ta	FAST_TRAP
+	retl
+	nop
+	SET_SIZE(hvio_msg_setvalid)
+
+	/*
+	 * hv_cpu_yield(void)
+	 */
+	ENTRY(hv_cpu_yield)
+	mov	HV_CPU_YIELD, %o5
+	ta	FAST_TRAP
+	retl
+	nop
+	SET_SIZE(hv_cpu_yield)
+
+	/*
+	 * hv_service_recv(uint64_t s_id, uint64_t buf_pa,
+	 *     uint64_t size, uint64_t *recv_bytes);
+	 */
+	ENTRY(hv_service_recv)
+	save	%sp, -SA(MINFRAME), %sp
+	mov	%i0, %o0
+	mov	%i1, %o1
+	mov	%i2, %o2
+	mov	%i3, %o3
+	mov	SVC_RECV, %o5
+	ta	FAST_TRAP
+	brnz	%o0, 1f
+	mov	%o0, %i0
+	stx	%o1, [%i3]
+1:
+	ret
+	restore
+	SET_SIZE(hv_service_recv)
+
+	/*
+	 * hv_service_send(uint64_t s_id, uint64_t buf_pa,
+	 *     uint64_t size, uint64_t *recv_bytes);
+	 */
+	ENTRY(hv_service_send)
+	save	%sp, -SA(MINFRAME), %sp
+	mov	%i0, %o0
+	mov	%i1, %o1
+	mov	%i2, %o2
+	mov	%i3, %o3
+	mov	SVC_SEND, %o5
+	ta	FAST_TRAP
+	brnz	%o0, 1f
+	mov	%o0, %i0
+	stx	%o1, [%i3]
+1:
+	ret
+	restore	
+	SET_SIZE(hv_service_send)
+	
+	/*
+	 * hv_service_getstatus(uint64_t s_id, uint64_t *vreg);
+	 */
+	ENTRY(hv_service_getstatus)
+	mov	%o1, %o4			! save datap
+	mov	SVC_GETSTATUS, %o5
+	ta	FAST_TRAP
+	brz,a	%o0, 1f
+	stx	%o1, [%o4]
+1:
+	retl
+	nop
+	SET_SIZE(hv_service_getstatus)
+	
+	/*
+	 * hv_service_setstatus(uint64_t s_id, uint64_t bits);
+	 */
+	ENTRY(hv_service_setstatus)
+	mov	SVC_SETSTATUS, %o5
+	ta	FAST_TRAP
+	retl
+	nop
+	SET_SIZE(hv_service_setstatus)
+
+	/*
+	 * hv_service_clrstatus(uint64_t s_id, uint64_t bits);
+	 */
+	ENTRY(hv_service_clrstatus)
+	mov	SVC_CLRSTATUS, %o5
+	ta	FAST_TRAP
+	retl
+	nop
+	SET_SIZE(hv_service_clrstatus)
+
+	/*
+	 * int hv_cpu_state(uint64_t cpuid, uint64_t *cpu_state);
+	 */
+	ENTRY(hv_cpu_state)
+	mov	%o1, %o4			! save datap
+	mov	HV_CPU_STATE, %o5
+	ta	FAST_TRAP
+	brz,a	%o0, 1f
+	stx	%o1, [%o4]
+1:
+	retl
+	nop
+	SET_SIZE(hv_cpu_state)
+
+	/*
+	 * HV state dump zone Configure
+	 * arg0 real adrs of dump buffer (%o0)
+	 * arg1 size of dump buffer (%o1)
+	 * ret0 status (%o0)
+	 * ret1 size of buffer on success and min size on EINVAL (%o1)
+	 * hv_dump_buf_update(uint64_t paddr, uint64_t size, uint64_t *ret_size)
+	 */
+	ENTRY(hv_dump_buf_update)
+	mov	DUMP_BUF_UPDATE, %o5
+	ta	FAST_TRAP
+	retl
+	stx	%o1, [%o2]
+	SET_SIZE(hv_dump_buf_update)
+
+
+	/*
+	 * For memory scrub
+	 * int hv_mem_scrub(uint64_t real_addr, uint64_t length,
+	 * 	uint64_t *scrubbed_len);
+	 * Retun %o0 -- status
+	 *       %o1 -- bytes scrubbed
+	 */
+	ENTRY(hv_mem_scrub)
+	mov	%o2, %o4
+	mov	HV_MEM_SCRUB, %o5
+	ta	FAST_TRAP
+	retl
+	stx	%o1, [%o4]
+	SET_SIZE(hv_mem_scrub)
+
+	/*
+	 * Flush ecache 
+	 * int hv_mem_sync(uint64_t real_addr, uint64_t length,
+	 * 	uint64_t *flushed_len);
+	 * Retun %o0 -- status
+	 *       %o1 -- bytes flushed
+	 */
+	ENTRY(hv_mem_sync)
+	mov	%o2, %o4
+	mov	HV_MEM_SYNC, %o5
+	ta	FAST_TRAP
+	retl
+	stx	%o1, [%o4]
+	SET_SIZE(hv_mem_sync)
+
+	/*
+	 * TTRACE_BUF_CONF Configure
+	 * arg0 RA base of buffer (%o0)
+	 * arg1 buf size in no. of entries (%o1)
+	 * ret0 status (%o0)
+	 * ret1 minimum size in no. of entries on failure,
+	 * actual size in no. of entries on success (%o1)
+	 */
+	ENTRY(hv_ttrace_buf_conf)
+	mov	TTRACE_BUF_CONF, %o5
+	ta	FAST_TRAP
+	retl
+	stx	%o1, [%o2]
+	SET_SIZE(hv_ttrace_buf_conf)
+
+	 /*
+	 * TTRACE_BUF_INFO
+	 * ret0 status (%o0)
+	 * ret1 RA base of buffer (%o1)
+	 * ret2 size in no. of entries (%o2)
+	 */
+	ENTRY(hv_ttrace_buf_info)
+	mov	%o0, %o3
+	mov	%o1, %o4
+	mov	TTRACE_BUF_INFO, %o5
+	ta	FAST_TRAP
+	stx	%o1, [%o3]
+	retl
+	stx	%o2, [%o4]
+	SET_SIZE(hv_ttrace_buf_info)
+
+	/*
+	 * TTRACE_ENABLE
+	 * arg0 enable/ disable (%o0)
+	 * ret0 status (%o0)
+	 * ret1 previous enable state (%o1)
+	 */
+	ENTRY(hv_ttrace_enable)
+	mov	%o1, %o2
+	mov	TTRACE_ENABLE, %o5
+	ta	FAST_TRAP
+	retl
+	stx	%o1, [%o2]
+	SET_SIZE(hv_ttrace_enable)
+
+	/*
+	 * TTRACE_FREEZE
+	 * arg0 enable/ freeze (%o0)
+	 * ret0 status (%o0)
+	 * ret1 previous freeze state (%o1)
+	*/
+	ENTRY(hv_ttrace_freeze)
+	mov	%o1, %o2
+	mov	TTRACE_FREEZE, %o5
+	ta	FAST_TRAP
+	retl
+	stx	%o1, [%o2]
+	SET_SIZE(hv_ttrace_freeze)
+
+	/*
+	 * MACH_DESC
+	 * arg0 buffer real address
+	 * arg1 pointer to uint64_t for size of buffer
+	 * ret0 status
+	 * ret1 return required size of buffer / returned data size
+	 */
+	ENTRY(hv_mach_desc)
+	mov     %o1, %o4                ! save datap
+	ldx     [%o1], %o1
+	mov     HV_MACH_DESC, %o5
+	ta      FAST_TRAP
+	retl
+	stx   %o1, [%o4]
+	SET_SIZE(hv_mach_desc)
+
+	/*
+	 * hv_ncs_request(int cmd, uint64_t realaddr, size_t sz)
+	 */
+	ENTRY(hv_ncs_request)
+	mov	HV_NCS_REQUEST, %o5
+	ta	FAST_TRAP
+	retl
+	nop
+	SET_SIZE(hv_ncs_request)
+
+	/*
+	 * hv_ra2pa(uint64_t ra)
+	 *
+	 * MACH_DESC
+	 * arg0 Real address to convert
+	 * ret0 Returned physical address or -1 on error
+	 */
+	ENTRY(hv_ra2pa)
+	mov	HV_RA2PA, %o5
+	ta	FAST_TRAP
+	cmp	%o0, 0
+	move	%xcc, %o1, %o0
+	movne	%xcc, -1, %o0
+	retl
+	nop
+	SET_SIZE(hv_ra2pa)
+
+	/*
+	 * hv_hpriv(void *func, uint64_t arg1, uint64_t arg2, uint64_t arg3)
+	 *
+	 * MACH_DESC
+	 * arg0 OS function to call
+	 * arg1 First arg to OS function
+	 * arg2 Second arg to OS function
+	 * arg3 Third arg to OS function
+	 * ret0 Returned value from function
+	 */
+	
+	ENTRY(hv_hpriv)
+	mov	HV_HPRIV, %o5
+	ta	FAST_TRAP
+	retl
+	nop
+	SET_SIZE(hv_hpriv)
+
+	/*
+	 * panic_bad_hcall is called when a hcall returns
+	 * unexpected error
+	 * %o0 error number
+	 * %o1 hcall number
+	 */
+
+	.text
+bad_hcall_error:
+	.asciz	"hypervisor call 0x%x returned an unexpected error %d"
+
+	
+	ENTRY(panic_bad_hcall)
+	mov	%o0, %o2
+	sethi	%hi(bad_hcall_error), %o0
+	or	%o0, %lo(bad_hcall_error), %o0
+	mov	%o7, %o3
+	call	panic
+	  mov	%o3, %o7
+	SET_SIZE(panic_bad_hcall)
+
+	ENTRY(hv_magic_trap_on)
+	ta	0x77
+	retl
+	  nop
+	END(hv_magic_trap_on)
+
+	ENTRY(hv_magic_trap_off)
+	ta	0x78
+	retl
+	  nop
+	END(hv_magic_trap_off)
+
+
+	ENTRY(hv_sim_read)
+	mov	HVIO_SIM_READ, %o5
+	ta	FAST_TRAP
+	retl
+	  nop
+	END(hv_read)
+
+	ENTRY(hv_sim_write)
+	mov	HVIO_SIM_WRITE, %o5
+	ta	FAST_TRAP
+	retl
+	  nop
+	END(hv_write)
+	
+		
+		
+#endif	/* lint || __lint */
diff --git a/sys/sun4v/sun4v/hv_pci.c b/sys/sun4v/sun4v/hv_pci.c
new file mode 100644
index 000000000000..67aeaa515ce8
--- /dev/null
+++ b/sys/sun4v/sun4v/hv_pci.c
@@ -0,0 +1,531 @@
+/*-
+ * Copyright 2006 John-Mark Gurney.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+/*
+ * Support for the Hypervisor PCI bus.
+ */
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/bus.h>
+#include <sys/kernel.h>
+#include <sys/module.h>
+#include <sys/pcpu.h>
+#include <sys/endian.h>
+#include <sys/rman.h>
+#include <dev/pci/pcireg.h>
+#include <dev/pci/pcivar.h>
+
+#include <machine/bus.h>
+
+#include <machine/hypervisor_api.h>
+
+#include <dev/ofw/ofw_bus.h>
+#include <dev/ofw/ofw_bus_subr.h>
+#include <dev/ofw/ofw_pci.h>
+#include <dev/ofw/openfirm.h>
+#include <sparc64/pci/ofw_pci.h>
+
+#include <machine/hv_pcivar.h>
+#include <machine/hviommu.h>
+#include <machine/vmparam.h>
+#include <machine/tlb.h>
+#include <machine/nexusvar.h>
+
+#include "pcib_if.h"
+
+/*
+ * XXX - should get this through the bus, but Sun overloaded the reg OFW
+ * property, so there isn't normal resources associated w/ this device.
+ */
+extern struct bus_space_tag nexus_bustag;
+/*
+ * Methods
+ */
+static device_probe_t hvpci_probe;
+static device_attach_t hvpci_attach;
+static bus_read_ivar_t hvpci_read_ivar;
+static bus_write_ivar_t hvpci_write_ivar;
+static bus_alloc_resource_t hvpci_alloc_resource;
+static bus_activate_resource_t hvpci_activate_resource;
+static bus_deactivate_resource_t hvpci_deactivate_resource;
+static bus_release_resource_t hvpci_release_resource;
+static bus_get_dma_tag_t hvpci_get_dma_tag;
+static pcib_maxslots_t hvpci_maxslots;
+static pcib_read_config_t hvpci_read_config;
+static pcib_write_config_t hvpci_write_config;
+static pcib_route_interrupt_t hvpci_route_interrupt;
+static ofw_bus_get_node_t hvpci_get_node;
+static ofw_pci_intr_pending_t hvpci_intr_pending;
+static ofw_pci_get_bus_handle_t hvpci_get_bus_handle;
+
+static device_method_t hv_pcib_methods[] = {
+	/* Device interface */
+	DEVMETHOD(device_probe,		hvpci_probe),
+	DEVMETHOD(device_attach,	hvpci_attach),
+	DEVMETHOD(device_shutdown,	bus_generic_shutdown),
+	DEVMETHOD(device_suspend,	bus_generic_suspend),
+	DEVMETHOD(device_resume,	bus_generic_resume),
+
+	/* Bus interface */
+	DEVMETHOD(bus_print_child,	bus_generic_print_child),
+	DEVMETHOD(bus_read_ivar,	hvpci_read_ivar),
+	DEVMETHOD(bus_write_ivar,	hvpci_write_ivar),
+	DEVMETHOD(bus_setup_intr,	bus_generic_setup_intr),
+	DEVMETHOD(bus_teardown_intr,	bus_generic_teardown_intr),
+	DEVMETHOD(bus_alloc_resource,	hvpci_alloc_resource),
+	DEVMETHOD(bus_activate_resource,	hvpci_activate_resource),
+	DEVMETHOD(bus_deactivate_resource,	hvpci_deactivate_resource),
+	DEVMETHOD(bus_release_resource,	hvpci_release_resource),
+	DEVMETHOD(bus_get_dma_tag,	hvpci_get_dma_tag),
+
+	/* pcib interface */
+	DEVMETHOD(pcib_maxslots,	hvpci_maxslots),
+	DEVMETHOD(pcib_read_config,	hvpci_read_config),
+	DEVMETHOD(pcib_write_config,	hvpci_write_config),
+	DEVMETHOD(pcib_route_interrupt,	hvpci_route_interrupt),
+
+	/* ofw_bus interface */
+	DEVMETHOD(ofw_bus_get_node,	hvpci_get_node),
+
+	/* ofw_pci interface */
+	DEVMETHOD(ofw_pci_intr_pending,	hvpci_intr_pending),
+	DEVMETHOD(ofw_pci_get_bus_handle,	hvpci_get_bus_handle),
+
+	{ 0, 0 }
+};
+
+static driver_t hvpci_driver = {
+	"pcib",
+	hv_pcib_methods,
+	sizeof(struct hvpci_softc),
+};
+
+static devclass_t hvpci_devclass;
+
+DRIVER_MODULE(hvpci, nexus, hvpci_driver, hvpci_devclass, 0, 0);
+
+static int
+hvpci_probe(device_t dev)
+{
+
+	if (strcmp(ofw_bus_get_name(dev), "pci") != 0)
+		return (ENXIO);
+
+	device_set_desc(dev, "Hypervisor PCI Bridge");
+	return (0);
+}
+
+static int
+hvpci_attach(device_t dev)
+{
+	struct ofw_pci_ranges *range;
+	struct rman *rmanp;
+	struct hvpci_softc *sc;
+	struct hviommu *himp;
+	bus_space_tag_t *btp;
+	phandle_t node;
+	uint32_t *dvma;
+	int br[2];
+	int n, type;
+	int i, nrange;
+
+	sc = device_get_softc(dev);
+
+	node = ofw_bus_get_node(dev);
+	if (node == -1)
+		panic("%s: ofw_bus_get_node failed.", __func__);
+
+	sc->hs_node = node;
+
+	/* Setup the root bus number for this bus */
+	n = OF_getprop(node, "bus-range", &br[0], sizeof br);
+	if (n == -1)
+		panic("%s: could not get bus-range", __func__);
+	if (n != sizeof br)
+		panic("%s: broken bus-range (%d)", __func__, n);
+	sc->hs_busnum = br[0];
+
+	/* Setup the HyperVisor devhandle for this bus */
+	sc->hs_devhandle = nexus_get_devhandle(dev);
+
+	/* Pull in the ra addresses out of OFW */
+	nrange = OF_getprop_alloc(node, "ranges", sizeof *range,
+	    (void **)&range);
+
+	/* Initialize memory and I/O rmans. */
+	for (i = 0; i < nrange; i++) {
+/* XXX - from sun4v/io/px/px_lib4v.c: px_ranges_phi_mask */
+#define PHYS_MASK	((1ll << (28 + 32)) - 1)
+		switch (OFW_PCI_RANGE_CS(&range[i])) {
+		case OFW_PCI_CS_IO:
+			rmanp = &sc->hs_pci_io_rman;
+			rmanp->rm_descr = "HyperVisor PCI I/O Ports";
+			btp = &sc->hs_pci_iot;
+			sc->hs_pci_ioh = OFW_PCI_RANGE_PHYS(&range[i]) &
+			    PHYS_MASK;
+			type = PCI_IO_BUS_SPACE;
+#ifdef DEBUG
+			printf("io handle: %#lx\n", sc->hs_pci_ioh);
+#endif
+			break;
+
+		case OFW_PCI_CS_MEM32:
+			rmanp = &sc->hs_pci_mem_rman;
+			rmanp->rm_descr = "HyperVisor PCI Memory";
+			btp = &sc->hs_pci_memt;
+			sc->hs_pci_memh = OFW_PCI_RANGE_PHYS(&range[i]) &
+			    PHYS_MASK;
+			type = PCI_MEMORY_BUS_SPACE;
+			break;
+
+		case OFW_PCI_CS_MEM64:
+			continue;
+
+		default:
+			panic("%s: unknown range type: %d", __func__,
+			    OFW_PCI_RANGE_CS(&range[i]));
+		}
+		rmanp->rm_type = RMAN_ARRAY;
+		if (rman_init(rmanp) != 0 || rman_manage_region(rmanp, 0,
+		    OFW_PCI_RANGE_SIZE(&range[i])) != 0)
+			panic("%s: failed to set up rman type: %d", __func__,
+			    OFW_PCI_RANGE_CS(&range[i]));
+
+		*btp = (bus_space_tag_t)malloc(sizeof **btp, M_DEVBUF,
+		    M_WAITOK|M_ZERO);
+		(*btp)->bst_parent = &nexus_bustag;
+		(*btp)->bst_type = type;
+	}
+	free(range, M_OFWPROP);
+
+	nrange = OF_getprop_alloc(node, "virtual-dma", sizeof *dvma,
+	    (void **)&dvma);
+	KASSERT(nrange == 2, ("virtual-dma propery invalid"));
+
+	/* Setup bus_dma_tag */
+	himp = hviommu_init(sc->hs_devhandle, dvma[0], dvma[1]);
+	sc->hs_dmatag.dt_cookie = himp;
+	sc->hs_dmatag.dt_mt = &hviommu_dma_methods;
+	sc->hs_dmatag.dt_lowaddr = ~0;
+	sc->hs_dmatag.dt_highaddr = ~0;
+	sc->hs_dmatag.dt_boundary = BUS_SPACE_MAXADDR_32BIT + 1;
+
+	free(dvma, M_OFWPROP);
+
+	/* Setup ofw imap */
+	ofw_bus_setup_iinfo(node, &sc->hs_pci_iinfo, sizeof(ofw_pci_intr_t));
+
+	device_add_child(dev, "pci", -1);
+
+	return (bus_generic_attach(dev));
+}
+
+static int
+hvpci_maxslots(device_t dev)
+{
+
+	return (0);
+}
+
+#define HVPCI_BDF(b, d, f)	\
+		((b & 0xff) << 16) | ((d & 0x1f) << 11) | ((f & 0x7) << 8)
+#if 0
+hvio_config_get(devhandle_t dev_hdl, pci_device_t bdf,
+    pci_config_offset_t off, pci_config_size_t size, pci_cfg_data_t *data_p)
+{ return (0); }
+#endif
+static uint32_t
+hvpci_read_config(device_t dev, u_int bus, u_int slot, u_int func, u_int reg,
+    int width)
+{
+	struct hvpci_softc *sc;
+	uint32_t data = -1;
+	uint64_t r;
+	uint32_t ret;
+
+	sc = device_get_softc(dev);
+
+	r = hvio_config_get(sc->hs_devhandle, HVPCI_BDF(bus, slot, func),
+			    reg, width, (pci_cfg_data_t *)&data);
+		
+	if (r == H_EOK) {
+		switch (width) {
+		case 1:
+			ret = data & 0xff;
+			if (ret == 0 && reg == PCIR_INTLINE)
+				ret = PCI_INVALID_IRQ;
+			break;
+		case 2:
+			ret = data & 0xffff;
+			break;
+		case 4:
+			ret = data;
+			break;
+		default:
+			ret = -1;
+		}
+		return ret;
+	}
+
+	return -1;
+}
+
+#if 0
+uint64_t
+hvio_config_put(devhandle_t dev_hdl, pci_device_t bdf,
+    pci_config_offset_t off, pci_config_size_t size, pci_cfg_data_t data)
+#endif
+static void
+hvpci_write_config(device_t dev, u_int bus, u_int slot, u_int func, u_int reg,
+     uint32_t val, int width)
+{
+	struct hvpci_softc *sc;
+	pci_cfg_data_t data = { 0 };
+	uint64_t r;
+
+	sc = device_get_softc(dev);
+	switch (width) {
+	case 1:
+		data.qw = (uint8_t)val;
+		break;
+	case 2:
+		data.qw = (uint16_t)(val & 0xffff);
+		break;
+	case 4:
+		data.qw = (uint32_t)val;
+		break;
+	default:
+		panic("unsupported width: %d", width);
+	}
+
+	r = hvio_config_put(sc->hs_devhandle, HVPCI_BDF(bus, slot, func), 
+			    reg, width, (pci_cfg_data_t)data);
+
+	if (r)
+		printf("put failed with: %ld\n", r);
+}
+
+static int
+hvpci_route_interrupt(device_t bridge, device_t dev, int pin)
+{
+	struct hvpci_softc *sc;
+	struct ofw_pci_register reg;
+	phandle_t node;
+	ofw_pci_intr_t pintr, mintr;
+	int obli;
+	uint8_t maskbuf[sizeof(reg) + sizeof(pintr)];
+
+	sc = device_get_softc(bridge);
+	node = ofw_bus_get_node(dev);
+	pintr = pin;
+	obli = ofw_bus_lookup_imap(node, &sc->hs_pci_iinfo, &reg, sizeof(reg),
+	    &pintr, sizeof(pintr), &mintr, sizeof(mintr), maskbuf);
+	device_printf(dev, "called hvpci_route_intr: %d, got: mintr: %#x\n",
+	    obli, mintr);
+	if (obli)
+		return (mintr);
+
+	panic("pin %d not found in imap of %s", pin, device_get_nameunit(bridge));
+}
+
+static phandle_t
+hvpci_get_node(device_t bus, device_t dev)
+{
+	struct hvpci_softc *sc;
+
+	sc = device_get_softc(bus);
+
+	return (sc->hs_node);
+}
+
+static int
+hvpci_intr_pending(device_t dev, ofw_pci_intr_t intr)
+{
+	/* XXX - implement */
+	panic("unimplemnted");
+}
+
+static bus_space_handle_t
+hvpci_get_bus_handle(device_t dev, int type, bus_space_handle_t childhdl,
+    bus_space_tag_t *tag)
+{
+	struct hvpci_softc *sc;
+
+	sc = device_get_softc(dev);
+	switch (type) {
+	case SYS_RES_IOPORT:
+		*tag = sc->hs_pci_iot;
+#ifdef DEBUG
+		printf("io handle: %#lx\n", sc->hs_pci_ioh + childhdl);
+#endif
+		return (sc->hs_pci_ioh + childhdl);
+		break;
+
+	case SYS_RES_MEMORY:
+		*tag = sc->hs_pci_memt;
+		return (sc->hs_pci_ioh + childhdl);
+		break;
+
+	default:
+		panic("%s: illegal space (%d)", __func__, type);
+	}
+}
+
+static int
+hvpci_read_ivar(device_t dev, device_t child, int which, uintptr_t *result)
+{
+	struct hvpci_softc *sc;
+
+	sc = device_get_softc(dev);
+
+	switch (which) {
+	case PCIB_IVAR_BUS:
+		*result = sc->hs_busnum;
+		return (0);
+	}
+
+	return (ENOENT);
+}
+
+static int
+hvpci_write_ivar(device_t dev, device_t child, int which, uintptr_t value)
+{
+	struct hvpci_softc *sc;
+
+	sc = device_get_softc(dev);
+	switch (which) {
+	case PCIB_IVAR_BUS:
+		sc->hs_busnum = value;
+		return (0);
+	}
+
+	return (ENOENT);
+}
+
+static struct resource *
+hvpci_alloc_resource(device_t bus, device_t child, int type, int *rid,
+    u_long start, u_long end, u_long count, u_int flags)
+{
+	struct hvpci_softc *sc;
+	struct resource *rv;
+	struct rman *rm;
+	bus_space_tag_t bt;
+	bus_space_handle_t bh;
+	int needactivate;
+
+	sc = device_get_softc(bus);
+
+	needactivate = flags & RF_ACTIVE;
+	flags &= ~RF_ACTIVE;
+
+	switch (type) {
+	case SYS_RES_IRQ:
+		return BUS_ALLOC_RESOURCE(device_get_parent(bus), child, type,
+		    rid, start, end, count, flags);
+		break;
+
+	case SYS_RES_MEMORY:
+		rm = &sc->hs_pci_mem_rman;
+		bt = sc->hs_pci_memt;
+		bh = sc->hs_pci_memh;
+		break;
+
+	case SYS_RES_IOPORT:
+#ifdef DEBUG
+		printf("alloc: start: %#lx, end: %#lx, count: %#lx\n", start, end, count);
+#endif
+		rm = &sc->hs_pci_io_rman;
+		bt = sc->hs_pci_iot;
+		bh = sc->hs_pci_ioh;
+		break;
+
+	default:
+		return (NULL);
+	}
+
+	rv = rman_reserve_resource(rm, start, end, count, flags, child);
+	if (rv == NULL)
+		return (NULL);
+
+	bh += rman_get_start(rv);
+	rman_set_bustag(rv, bt);
+	rman_set_bushandle(rv, bh);
+
+	if (needactivate) {
+		if (bus_activate_resource(child, type, *rid, rv)) {
+			rman_release_resource(rv);
+			return (NULL);
+		}
+	}
+
+	return (rv);
+}
+
+static int
+hvpci_activate_resource(device_t bus, device_t child, int type, int rid,
+    struct resource *r) 
+{
+	void *p;
+
+	if (type == SYS_RES_MEMORY) {
+		/* XXX - we may still need to set the IE bit on the mapping */
+		p = (void *)TLB_PHYS_TO_DIRECT(rman_get_bushandle(r));
+		rman_set_virtual(r, p);
+	}
+	return (rman_activate_resource(r));
+}
+
+static int
+hvpci_deactivate_resource(device_t bus, device_t child, int type, int rid,
+    struct resource *r) 
+{
+
+	return (0);
+}
+
+static int
+hvpci_release_resource(device_t bus, device_t child, int type, int rid,
+    struct resource *r)
+{
+
+	return (0);
+}
+
+static bus_dma_tag_t
+hvpci_get_dma_tag(device_t bus, device_t child)
+{
+	struct hvpci_softc *sc;
+
+	sc = device_get_softc(bus);
+
+	return &sc->hs_dmatag;
+}
diff --git a/sys/sun4v/sun4v/hvcons.c b/sys/sun4v/sun4v/hvcons.c
new file mode 100644
index 000000000000..e08916bbba7b
--- /dev/null
+++ b/sys/sun4v/sun4v/hvcons.c
@@ -0,0 +1,404 @@
+/*-
+ * Copyright (C) 2006 Kip Macy
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY Kip Macy ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL Kip Macy BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+ * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/types.h>
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/module.h>
+#include <sys/bus.h>
+#include <sys/kdb.h>
+#include <sys/kernel.h>
+#include <sys/conf.h>
+#include <sys/cons.h>
+#include <sys/consio.h>
+#include <sys/rman.h>
+#include <sys/tty.h>
+
+#include <machine/mdesc_bus.h>
+#include <machine/cddl/mdesc.h>
+
+#include "mdesc_bus_if.h"
+
+#include "opt_simulator.h"
+
+#include <machine/resource.h>
+#include <machine/hypervisor_api.h>
+
+#define HVCN_POLL_FREQ 10
+
+
+static d_open_t  hvcn_open;
+static d_close_t hvcn_close;
+
+static struct cdevsw hvcn_cdevsw = {
+	.d_version =	D_VERSION,
+	.d_open =	hvcn_open,
+	.d_close =	hvcn_close,
+	.d_name =	"hvcn",
+	.d_flags =	D_TTY | D_NEEDGIANT,
+};
+
+#define PCBURST 16
+static struct tty		*hvcn_tp = NULL;
+static struct resource          *hvcn_irq;
+static void                     *hvcn_intrhand;
+
+static int bufindex;
+static int buflen;
+static u_char buf[PCBURST];
+static int			polltime;
+static struct callout_handle	hvcn_timeouthandle
+    = CALLOUT_HANDLE_INITIALIZER(&hvcn_timeouthandle);
+
+#if defined(KDB)
+static int			alt_break_state;
+#endif
+
+static void	hvcn_tty_start(struct tty *);
+static int	hvcn_tty_param(struct tty *, struct termios *);
+static void	hvcn_tty_stop(struct tty *, int);
+static void     hvcn_timeout(void *);
+
+static cn_probe_t	hvcnprobe;
+static cn_init_t	hvcninit;
+static cn_getc_t	hvcngetc;
+static cn_checkc_t 	hvcncheckc;
+static cn_putc_t	hvcnputc;
+
+
+CONS_DRIVER(hvcn, hvcnprobe, hvcninit, NULL, hvcngetc,
+    hvcncheckc, hvcnputc, NULL);
+
+
+static int
+hvcn_open(struct cdev *dev, int flag, int mode, struct thread *td)
+{
+	struct	tty *tp;
+	int	error, setuptimeout;
+
+	setuptimeout = 0;
+
+	if (dev->si_tty == NULL) {
+		hvcn_tp = ttyalloc();
+		dev->si_tty = hvcn_tp;
+		hvcn_tp->t_dev = dev;
+	}
+	tp = dev->si_tty;
+
+	tp->t_oproc = hvcn_tty_start;
+	tp->t_param = hvcn_tty_param;
+	tp->t_stop = hvcn_tty_stop;
+
+	if ((tp->t_state & TS_ISOPEN) == 0) {
+		tp->t_state |= TS_CARR_ON;
+		ttyconsolemode(tp, 0);
+
+		setuptimeout = 1;
+	} else if ((tp->t_state & TS_XCLUDE) && suser(td)) {
+		return (EBUSY);
+	}
+
+	error = ttyld_open(tp, dev);
+#if defined(SIMULATOR) || 1
+	if (error == 0 && setuptimeout) {
+		int polltime;
+
+		polltime = hz / HVCN_POLL_FREQ;
+		if (polltime < 1) {
+			polltime = 1;
+		}
+
+		hvcn_timeouthandle = timeout(hvcn_timeout, tp, polltime);
+	}
+#endif
+	return (error);
+}
+ 
+static int
+hvcn_close(struct cdev *dev, int flag, int mode, struct thread *td)
+{
+	int	unit;
+	struct	tty *tp;
+
+	unit = minor(dev);
+	tp = dev->si_tty;
+
+	if (unit != 0) 
+		return (ENXIO);
+	
+	untimeout(hvcn_timeout, tp, hvcn_timeouthandle);
+	ttyld_close(tp, flag);
+	tty_close(tp);
+
+	return (0);
+}
+
+static void
+hvcnprobe(struct consdev *cp)
+{
+
+#if 0
+	char name[64];
+
+	node = OF_peer(0);
+	if (node == -1)
+		panic("%s: OF_peer failed.", __func__);
+	
+	for (node = OF_child(node); node > 0; node = OF_peer(node)) {
+		OF_getprop(node, "name", name, sizeof(name));
+		if (!strcmp(name, "virtual-devices"))
+			break;
+	}
+	
+	if (node == 0)
+		goto done;
+
+	for (node = OF_child(node); node > 0; node = OF_peer(node)) {
+		OF_getprop(node, "name", name, sizeof(name));
+		if (!strcmp(name, "console"))
+			break;
+	}
+done:
+#endif	
+	cp->cn_pri = CN_NORMAL;
+
+}
+
+static void
+hvcninit(struct consdev *cp)
+{
+	sprintf(cp->cn_name, "hvcn");
+}
+
+static int
+hvcngetc(struct consdev *cp)
+{
+	unsigned char ch;
+	int l;
+
+	ch = '\0';
+
+	while ((l = hv_cngetchar(&ch)) != H_EOK) {
+#if defined(KDB)
+		if (l == H_BREAK || l ==  H_HUP)
+			kdb_enter("Break sequence on console");
+
+	if (kdb_alt_break(ch, &alt_break_state))
+		kdb_enter("Break sequence on console");
+#endif
+		if (l != -2 && l != 0) {
+			return (-1);
+		}
+	}
+
+
+
+	return (ch);
+}
+
+static int
+hvcncheckc(struct consdev *cp)
+{
+	unsigned char ch;
+	int l;
+
+	if ((l = hv_cngetchar(&ch)) == H_EOK) {
+#if defined(KDB)
+		if (l == H_BREAK || l ==  H_HUP)
+			kdb_enter("Break sequence on console");
+		if (kdb_alt_break(ch, &alt_break_state))
+			kdb_enter("Break sequence on console");
+#endif
+		return (ch);
+	}
+
+	return (-1);
+}
+
+
+static void
+hvcnputc(struct consdev *cp, int c)
+{
+
+	int error;
+
+	error = 0;
+	do {
+		if (c == '\n') 
+			error = hv_cnputchar('\r');
+	} while (error == H_EWOULDBLOCK);
+	do {
+		error = hv_cnputchar(c);
+	} while (error == H_EWOULDBLOCK);
+}
+
+static int
+hvcn_tty_param(struct tty *tp, struct termios *t)
+{
+        tp->t_ispeed = t->c_ispeed;
+        tp->t_ospeed = t->c_ospeed;
+        tp->t_cflag = t->c_cflag;
+
+	return (0);
+}
+
+static void
+hvcn_tty_start(struct tty *tp)
+{
+
+        if (!(tp->t_state & (TS_TIMEOUT | TS_BUSY | TS_TTSTOP))) {
+		tp->t_state |= TS_BUSY;
+
+		do {
+			if (buflen == 0) {
+				buflen = q_to_b(&tp->t_outq, buf, PCBURST);
+				bufindex = 0;
+			}
+			while (buflen) {
+				if (hv_cnputchar(buf[bufindex]) == H_EWOULDBLOCK)
+					goto done;
+				bufindex++;
+				buflen--;
+			}
+		} while (tp->t_outq.c_cc != 0);
+	done:
+		tp->t_state &= ~TS_BUSY;
+		ttwwakeup(tp);
+	}
+}
+
+static void
+hvcn_tty_stop(struct tty *tp, int flag)
+{
+	if ((tp->t_state & TS_BUSY) && !(tp->t_state & TS_TTSTOP)) 
+		tp->t_state |= TS_FLUSH;
+		
+	
+}
+
+static void
+hvcn_intr(void *v)
+{
+	struct tty *tp;
+	int c;
+
+	tp = (struct tty *)v;
+	
+	while ((c = hvcncheckc(NULL)) != -1) 
+		if (tp->t_state & TS_ISOPEN) 
+			ttyld_rint(tp, c);
+
+	if (tp->t_outq.c_cc != 0 || buflen != 0) 
+		hvcn_tty_start(tp);
+}
+
+static void
+hvcn_timeout(void *v)
+{
+	hvcn_intr(v);
+
+	hvcn_timeouthandle = timeout(hvcn_timeout, v, polltime);
+}
+
+
+static int
+hvcn_probe(device_t dev)
+{
+
+	if (strcmp(mdesc_bus_get_name(dev), "console"))
+		return (ENXIO);
+	
+	device_set_desc(dev, "sun4v virtual console");	
+
+	return (0);
+}
+
+
+static int
+hvcn_attach(device_t dev)
+{
+      
+	struct cdev *cdev;
+	char output[32];
+	int error, rid;
+
+	/* belongs in attach - but attach is getting called multiple times
+	 * for reasons I have not delved into
+	 */
+
+	if (hvcn_consdev.cn_pri == CN_DEAD || 
+	    hvcn_consdev.cn_name[0] == '\0') 
+		return (ENXIO);
+#if 0
+	if ((options = OF_finddevice("/options")) == -1 ||
+	    OF_getprop(options, "output-device", output,
+		       sizeof(output)) == -1)
+		return (ENXIO);
+
+#endif
+	cdev = make_dev(&hvcn_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600, "%s", output);
+	make_dev_alias(cdev, "hvcn");
+	
+	rid = 0;
+
+	if ((hvcn_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
+					RF_SHAREABLE | RF_ACTIVE)) == NULL) {
+		device_printf(dev, "couldn't map interrupt\n");
+		error = ENXIO;
+		goto fail;
+		
+	}
+	error = bus_setup_intr(dev, hvcn_irq, INTR_TYPE_TTY, hvcn_intr, hvcn_tp, 
+			       hvcn_intrhand);
+	
+	if (error)
+		device_printf(dev, "couldn't set up irq\n");
+	
+		
+fail:
+	return (error);
+      
+}
+
+static device_method_t hvcn_methods[] = {
+        DEVMETHOD(device_probe, hvcn_probe),
+        DEVMETHOD(device_attach, hvcn_attach),
+        {0, 0}
+};
+
+
+static driver_t hvcn_driver = {
+        "hvcn",
+        hvcn_methods,
+	0,
+};
+
+
+static devclass_t hvcn_devclass;
+
+DRIVER_MODULE(hvcn, vnex, hvcn_driver, hvcn_devclass, 0, 0);
diff --git a/sys/sun4v/sun4v/hviommu.c b/sys/sun4v/sun4v/hviommu.c
new file mode 100644
index 000000000000..e9d5533696fe
--- /dev/null
+++ b/sys/sun4v/sun4v/hviommu.c
@@ -0,0 +1,977 @@
+/*-
+ * Copyright (c) 1999, 2000 Matthew R. Green
+ * Copyright (c) 2001-2003 Thomas Moestl
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The name of the author may not be used to endorse or promote products
+ *    derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+/*-
+ * Copyright (c) 1998 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The NetBSD Foundation
+ * by Paul Kranenburg.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ *    must display the following acknowledgement:
+ *        This product includes software developed by the NetBSD
+ *        Foundation, Inc. and its contributors.
+ * 4. Neither the name of The NetBSD Foundation nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+/*-
+ * Copyright (c) 1992, 1993
+ *	The Regents of the University of California.  All rights reserved.
+ *
+ * This software was developed by the Computer Systems Engineering group
+ * at Lawrence Berkeley Laboratory under DARPA contract BG 91-66 and
+ * contributed to Berkeley.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 4. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	from: NetBSD: sbus.c,v 1.13 1999/05/23 07:24:02 mrg Exp
+ *	from: @(#)sbus.c	8.1 (Berkeley) 6/11/93
+ *	from: NetBSD: iommu.c,v 1.42 2001/08/06 22:02:58 eeh Exp
+ *
+ * $FreeBSD$
+ */
+
+#include <sys/param.h>
+#include <sys/kernel.h>
+#include <sys/lock.h>
+#include <sys/malloc.h>
+#include <sys/mbuf.h>
+#include <sys/mutex.h>
+#include <sys/proc.h>
+#include <sys/uio.h>
+
+#include <vm/vm.h>
+#include <vm/pmap.h>
+#include <vm/vm_map.h>
+
+#include <machine/bus.h>
+#include <machine/bus_private.h>
+#include <machine/hviommu.h>
+#include <machine/iommureg.h>
+#include <machine/pmap.h>
+#include <machine/resource.h>
+
+#include <sys/rman.h>
+
+/*
+ * Tuning constants.
+ */
+#define	IOMMU_MAX_PRE		(32 * 1024)
+#define	IOMMU_MAX_PRE_SEG	3
+
+#define	IO_PAGE_SIZE		PAGE_SIZE_8K
+#define	IO_PAGE_MASK		PAGE_MASK_8K
+#define	IO_PAGE_SHIFT		PAGE_SHIFT_8K
+#define	round_io_page(x)	round_page(x)
+#define	trunc_io_page(x)	trunc_page(x)
+
+
+MALLOC_DEFINE(M_HVIOMMU, "hviommu", "HyperVisor IOMMU");
+
+TAILQ_HEAD(hviommu_maplruq_head, bus_dmamap);
+
+struct hviommu {
+	struct mtx	him_mtx;
+
+	devhandle_t	him_handle;
+	u_long		him_dvmabase;
+	u_long		him_dvmasize;
+
+	struct hviommu_maplruq_head him_maplruq;
+	struct rman	him_rman;
+};
+
+#define VA_TO_TSBID(him, va)	((va - (him)->him_dvmabase) >> IO_PAGE_SHIFT)
+
+#ifdef IOMMU_DEBUG
+#define DPRINTF printf
+#else 
+#define DPRINTF(...)
+#endif
+
+/*
+ * Always overallocate one page; this is needed to handle alignment of the
+ * buffer, so it makes sense using a lazy allocation scheme.
+ */
+#define	IOMMU_SIZE_ROUNDUP(sz)						\
+	(round_io_page(sz) + IO_PAGE_SIZE)
+
+/* Resource helpers */
+#define IOMMU_RES_TO(v)	((v) >> IO_PAGE_SHIFT)
+#define	IOMMU_RES_START(res)						\
+	((bus_addr_t)rman_get_start(res) << IO_PAGE_SHIFT)
+#define	IOMMU_RES_END(res)						\
+	((bus_addr_t)(rman_get_end(res) + 1) << IO_PAGE_SHIFT)
+#define	IOMMU_RES_SIZE(res)						\
+	((bus_size_t)rman_get_size(res) << IO_PAGE_SHIFT)
+
+/* Helpers for struct bus_dmamap_res */
+#define	BDR_START(r)	IOMMU_RES_START((r)->dr_res)
+#define	BDR_END(r)	IOMMU_RES_END((r)->dr_res)
+#define	BDR_SIZE(r)	IOMMU_RES_SIZE((r)->dr_res)
+
+/* Locking macros. */
+#define	HIM_LOCK(him)	mtx_lock(&him->him_mtx)
+#define	HIM_LOCK_ASSERT(him)	mtx_assert(&him->him_mtx, MA_OWNED)
+#define	HIM_UNLOCK(him)	mtx_unlock(&him->him_mtx)
+
+/* LRU queue handling for lazy resource allocation. */
+static __inline void
+hviommu_map_insq(struct hviommu *him, bus_dmamap_t map)
+{
+
+	HIM_LOCK_ASSERT(him);
+	if (!SLIST_EMPTY(&map->dm_reslist)) {
+		if (map->dm_onq)
+			TAILQ_REMOVE(&him->him_maplruq, map, dm_maplruq);
+		TAILQ_INSERT_TAIL(&him->him_maplruq, map, dm_maplruq);
+		map->dm_onq = 1;
+	}
+}
+
+static __inline void
+hviommu_map_remq(struct hviommu *him, bus_dmamap_t map)
+{
+
+	HIM_LOCK_ASSERT(him);
+	if (map->dm_onq)
+		TAILQ_REMOVE(&him->him_maplruq, map, dm_maplruq);
+	map->dm_onq = 0;
+}
+
+struct hviommu *
+hviommu_init(devhandle_t dh, u_long dvmabase, u_long dvmasize)
+{
+	struct hviommu *him;
+	u_long end;
+
+	him = malloc(sizeof *him, M_HVIOMMU, M_WAITOK|M_ZERO);
+
+	mtx_init(&him->him_mtx, "hviommu", NULL, MTX_DEF);
+	him->him_handle = dh;
+	him->him_dvmabase = dvmabase;
+	him->him_dvmasize = dvmasize;
+
+	TAILQ_INIT(&him->him_maplruq);
+	him->him_rman.rm_type = RMAN_ARRAY;
+	him->him_rman.rm_descr = "HyperVisor IOMMU Memory";
+	end = him->him_dvmabase + him->him_dvmasize - 1;
+	if (rman_init(&him->him_rman) != 0 ||
+	    rman_manage_region(&him->him_rman, him->him_dvmabase >>
+	    IO_PAGE_SHIFT, end >> IO_PAGE_SHIFT) != 0)
+		panic("%s: can't initalize rman", __func__);
+
+	return him;
+}
+
+static void
+hviommu_remove(struct hviommu *him, vm_offset_t va, vm_size_t len)
+{
+	uint64_t error;
+	pages_t demapped;
+
+	KASSERT(va >= him->him_dvmabase,
+	    ("%s: va 0x%lx not in DVMA space", __func__, (u_long)va));
+	KASSERT(va + len >= va,
+	    ("%s: va 0x%lx + len 0x%lx wraps", __func__, (long)va, (long)len));
+	KASSERT((va & IO_PAGE_MASK) == 0 && (len & IO_PAGE_MASK) == 0,
+	    ("%s: va %#lx or len %#lx not page aligned", __func__, va, len));
+	while (len > 0) {
+		if ((error = hvio_iommu_demap(him->him_handle,
+		    VA_TO_TSBID(him, va), len >> IO_PAGE_SHIFT, &demapped))) {
+			printf("%s: demap: va: %#lx, npages: %#lx, err: %ld\n",
+			    __func__, va, len >> IO_PAGE_SHIFT, error);
+			demapped = 1;
+		}
+		va += demapped << IO_PAGE_SHIFT;
+		len -= demapped << IO_PAGE_SHIFT;
+	}
+}
+
+/*
+ * Allocate DVMA virtual memory for a map. The map may not be on a queue, so
+ * that it can be freely modified.
+ */
+static int
+hviommu_dvma_valloc(bus_dma_tag_t t, struct hviommu *him, bus_dmamap_t map,
+    bus_size_t size)
+{
+	struct resource *res;
+	struct bus_dmamap_res *bdr;
+	bus_size_t align, sgsize;
+
+	KASSERT(!map->dm_onq, ("hviommu_dvma_valloc: map on queue!"));
+	if ((bdr = malloc(sizeof(*bdr), M_HVIOMMU, M_NOWAIT)) == NULL)
+		return (EAGAIN);
+
+	/*
+	 * If a boundary is specified, a map cannot be larger than it; however
+	 * we do not clip currently, as that does not play well with the lazy
+	 * allocation code.
+	 * Alignment to a page boundary is always enforced.
+	 */
+	align = (t->dt_alignment + IO_PAGE_MASK) >> IO_PAGE_SHIFT;
+	sgsize = IOMMU_RES_TO(round_io_page(size));
+	if (t->dt_boundary > 0 && t->dt_boundary < IO_PAGE_SIZE)
+		panic("hviommu_dvmamap_load: illegal boundary specified");
+	res = rman_reserve_resource_bound(&him->him_rman, 0L,
+	    IOMMU_RES_TO(t->dt_lowaddr), sgsize,
+	    IOMMU_RES_TO(t->dt_boundary),
+	    RF_ACTIVE | rman_make_alignment_flags(align), NULL);
+	if (res == NULL) {
+		free(bdr, M_HVIOMMU);
+		return (ENOMEM);
+	}
+
+	bdr->dr_res = res;
+	bdr->dr_used = 0;
+	SLIST_INSERT_HEAD(&map->dm_reslist, bdr, dr_link);
+	return (0);
+}
+
+/* Unload the map and mark all resources as unused, but do not free them. */
+static void
+hviommu_dvmamap_vunload(struct hviommu *him, bus_dmamap_t map)
+{
+	struct bus_dmamap_res *r;
+
+	SLIST_FOREACH(r, &map->dm_reslist, dr_link) {
+		hviommu_remove(him, BDR_START(r), BDR_SIZE(r));
+		r->dr_used = 0;
+	}
+}
+
+/* Free a DVMA virtual memory resource. */
+static __inline void
+hviommu_dvma_vfree_res(bus_dmamap_t map, struct bus_dmamap_res *r)
+{
+
+	KASSERT(r->dr_used == 0, ("hviommu_dvma_vfree_res: resource busy!"));
+	if (r->dr_res != NULL && rman_release_resource(r->dr_res) != 0)
+		printf("warning: DVMA space lost\n");
+	SLIST_REMOVE(&map->dm_reslist, r, bus_dmamap_res, dr_link);
+	free(r, M_HVIOMMU);
+}
+
+/* Free all DVMA virtual memory for a map. */
+static void
+hviommu_dvma_vfree(struct hviommu *him, bus_dmamap_t map)
+{
+
+	HIM_LOCK(him);
+	hviommu_map_remq(him, map);
+	hviommu_dvmamap_vunload(him, map);
+	HIM_UNLOCK(him);
+	while (!SLIST_EMPTY(&map->dm_reslist))
+		hviommu_dvma_vfree_res(map, SLIST_FIRST(&map->dm_reslist));
+}
+
+/* Prune a map, freeing all unused DVMA resources. */
+static bus_size_t
+hviommu_dvma_vprune(struct hviommu *him, bus_dmamap_t map)
+{
+	struct bus_dmamap_res *r, *n;
+	bus_size_t freed = 0;
+
+	HIM_LOCK_ASSERT(him);
+	for (r = SLIST_FIRST(&map->dm_reslist); r != NULL; r = n) {
+		n = SLIST_NEXT(r, dr_link);
+		if (r->dr_used == 0) {
+			freed += BDR_SIZE(r);
+			hviommu_dvma_vfree_res(map, r);
+		}
+	}
+	if (SLIST_EMPTY(&map->dm_reslist))
+		hviommu_map_remq(him, map);
+	return (freed);
+}
+
+/*
+ * Try to find a suitably-sized (and if requested, -aligned) slab of DVMA
+ * memory with IO page offset voffs.
+ */
+static bus_addr_t
+hviommu_dvma_vfindseg(bus_dmamap_t map, vm_offset_t voffs, bus_size_t size,
+    bus_addr_t amask)
+{
+	struct bus_dmamap_res *r;
+	bus_addr_t dvmaddr, dvmend;
+
+	KASSERT(!map->dm_onq, ("hviommu_dvma_vfindseg: map on queue!"));
+	SLIST_FOREACH(r, &map->dm_reslist, dr_link) {
+		dvmaddr = round_io_page(BDR_START(r) + r->dr_used);
+		/* Alignment can only work with voffs == 0. */
+		dvmaddr = (dvmaddr + amask) & ~amask;
+		dvmaddr += voffs;
+		dvmend = dvmaddr + size;
+		if (dvmend <= BDR_END(r)) {
+			r->dr_used = dvmend - BDR_START(r);
+			r->dr_offset = voffs;
+			return (dvmaddr);
+		}
+	}
+	return (0);
+}
+
+/*
+ * Try to find or allocate a slab of DVMA space; see above.
+ */
+static int
+hviommu_dvma_vallocseg(bus_dma_tag_t dt, struct hviommu *him, bus_dmamap_t map,
+    vm_offset_t voffs, bus_size_t size, bus_addr_t amask, bus_addr_t *addr)
+{
+	bus_dmamap_t tm, last;
+	bus_addr_t dvmaddr, freed;
+	int error, complete = 0;
+
+	dvmaddr = hviommu_dvma_vfindseg(map, voffs, size, amask);
+
+	/* Need to allocate. */
+	if (dvmaddr == 0) {
+		while ((error = hviommu_dvma_valloc(dt, him, map,
+			voffs + size)) == ENOMEM && !complete) {
+			/*
+			 * Free the allocated DVMA of a few maps until
+			 * the required size is reached. This is an
+			 * approximation to not have to call the allocation
+			 * function too often; most likely one free run
+			 * will not suffice if not one map was large enough
+			 * itself due to fragmentation.
+			 */
+			HIM_LOCK(him);
+			freed = 0;
+			last = TAILQ_LAST(&him->him_maplruq, hviommu_maplruq_head);
+			do {
+				tm = TAILQ_FIRST(&him->him_maplruq);
+				complete = tm == last;
+				if (tm == NULL)
+					break;
+				freed += hviommu_dvma_vprune(him, tm);
+				/* Move to the end. */
+				hviommu_map_insq(him, tm);
+			} while (freed < size && !complete);
+			HIM_UNLOCK(him);
+		}
+		if (error != 0)
+			return (error);
+		dvmaddr = hviommu_dvma_vfindseg(map, voffs, size, amask);
+		KASSERT(dvmaddr != 0,
+		    ("hviommu_dvma_vallocseg: allocation failed unexpectedly!"));
+	}
+	*addr = dvmaddr;
+	return (0);
+}
+
+static int
+hviommu_dvmamem_alloc(bus_dma_tag_t dt, void **vaddr, int flags,
+    bus_dmamap_t *mapp)
+{
+	struct hviommu *him = dt->dt_cookie;
+	int error, mflags;
+
+	/*
+	 * XXX: This will break for 32 bit transfers on machines with more than
+	 * 16G (1 << 34 bytes) of memory.
+	 */
+	if ((error = sparc64_dma_alloc_map(dt, mapp)) != 0)
+		return (error);
+
+	if ((flags & BUS_DMA_NOWAIT) != 0)
+		mflags = M_NOWAIT;
+	else
+		mflags = M_WAITOK;
+	if ((flags & BUS_DMA_ZERO) != 0)
+		mflags |= M_ZERO;
+
+	if ((*vaddr = malloc(dt->dt_maxsize, M_HVIOMMU, mflags)) == NULL) {
+		error = ENOMEM;
+		sparc64_dma_free_map(dt, *mapp);
+		return (error);
+	}
+	if ((flags & BUS_DMA_COHERENT) != 0)
+		(*mapp)->dm_flags |= DMF_COHERENT;
+	/*
+	 * Try to preallocate DVMA space. If this fails, it is retried at load
+	 * time.
+	 */
+	hviommu_dvma_valloc(dt, him, *mapp, IOMMU_SIZE_ROUNDUP(dt->dt_maxsize));
+	HIM_LOCK(him);
+	hviommu_map_insq(him, *mapp);
+	HIM_UNLOCK(him);
+	return (0);
+}
+
+static void
+hviommu_dvmamem_free(bus_dma_tag_t dt, void *vaddr, bus_dmamap_t map)
+{
+	struct hviommu *him = dt->dt_cookie;
+
+	hviommu_dvma_vfree(him, map);
+	sparc64_dma_free_map(dt, map);
+	free(vaddr, M_HVIOMMU);
+}
+
+static int
+hviommu_dvmamap_create(bus_dma_tag_t dt, int flags, bus_dmamap_t *mapp)
+{
+	struct hviommu *him = dt->dt_cookie;
+	bus_size_t totsz, presz, currsz;
+	int error, i, maxpre;
+
+	if ((error = sparc64_dma_alloc_map(dt, mapp)) != 0)
+		return (error);
+	if ((flags & BUS_DMA_COHERENT) != 0)
+		(*mapp)->dm_flags |= DMF_COHERENT;
+	/*
+	 * Preallocate DVMA space; if this fails now, it is retried at load
+	 * time. Through bus_dmamap_load_mbuf() and bus_dmamap_load_uio(), it
+	 * is possible to have multiple discontiguous segments in a single map,
+	 * which is handled by allocating additional resources, instead of
+	 * increasing the size, to avoid fragmentation.
+	 * Clamp preallocation to IOMMU_MAX_PRE. In some situations we can
+	 * handle more; that case is handled by reallocating at map load time.
+	 */
+	totsz = ulmin(IOMMU_SIZE_ROUNDUP(dt->dt_maxsize), IOMMU_MAX_PRE); 
+	error = hviommu_dvma_valloc(dt, him, *mapp, totsz);
+	if (error != 0)
+		return (0);
+	/*
+	 * Try to be smart about preallocating some additional segments if
+	 * needed.
+	 */
+	maxpre = imin(dt->dt_nsegments, IOMMU_MAX_PRE_SEG);
+	presz = dt->dt_maxsize / maxpre;
+	KASSERT(presz != 0, ("hviommu_dvmamap_create: bogus preallocation size "
+	    ", nsegments = %d, maxpre = %d, maxsize = %lu", dt->dt_nsegments,
+	    maxpre, dt->dt_maxsize));
+	for (i = 1; i < maxpre && totsz < IOMMU_MAX_PRE; i++) {
+		currsz = round_io_page(ulmin(presz, IOMMU_MAX_PRE - totsz));
+		error = hviommu_dvma_valloc(dt, him, *mapp, currsz);
+		if (error != 0)
+			break;
+		totsz += currsz;
+	}
+	HIM_LOCK(him);
+	hviommu_map_insq(him, *mapp);
+	HIM_UNLOCK(him);
+	return (0);
+}
+
+static int
+hviommu_dvmamap_destroy(bus_dma_tag_t dt, bus_dmamap_t map)
+{
+	struct hviommu *him = dt->dt_cookie;
+
+	hviommu_dvma_vfree(him, map);
+	sparc64_dma_free_map(dt, map);
+	return (0);
+}
+
+#define IOTTE_CNT	64
+
+static void
+hviommu_map_pages(struct hviommu *him, bus_addr_t dvmaddr, uint64_t *iottes, pages_t iottecnt)
+{
+	uint64_t err;
+#ifdef IOMMU_DEBUG
+	bus_addr_t ra;
+	io_attributes_t ioattr;
+#endif
+	pages_t mapcnt;
+	int cntdone;
+	int i;
+
+	DPRINTF("mapping: dh: %#lx, dvmaddr: %#lx, tsbid: %#lx, cnt: %d\n",
+	    him->him_handle, dvmaddr, VA_TO_TSBID(him, dvmaddr), iottecnt);
+	for (i = 0; i < iottecnt; i++) {
+		DPRINTF("iotte:%#lx\n", iottes[i]);
+	}
+
+	/* push tte's */
+	cntdone = 0;
+	while (cntdone < iottecnt) {
+		if ((err = hvio_iommu_map(him->him_handle, VA_TO_TSBID(him,
+		    dvmaddr), iottecnt, PCI_MAP_ATTR_READ | PCI_MAP_ATTR_WRITE,
+		    (io_page_list_t *)pmap_kextract((vm_offset_t)&iottes[0]),
+		    &mapcnt))) {
+			DPRINTF("iommu_map: err: %ld\n", err);
+			mapcnt = 1;
+		}
+		cntdone += mapcnt;
+	}
+	for (i = 0; i < iottecnt; i++) {
+		DPRINTF("err: %ld", hvio_iommu_getmap(him->him_handle,
+		    VA_TO_TSBID(him, dvmaddr + i * IO_PAGE_SIZE),
+						     &ioattr, &ra));
+		DPRINTF(", ioattr: %d, raddr: %#lx\n", ioattr, ra);
+	}
+}
+
+/*
+ * IOMMU DVMA operations, common to SBUS and PCI.
+ */
+static int
+hviommu_dvmamap_load_buffer(bus_dma_tag_t dt, struct hviommu *him,
+    bus_dmamap_t map, void *buf, bus_size_t buflen, struct thread *td,
+    int flags, bus_dma_segment_t *segs, int *segp, int align)
+{
+	uint64_t iottes[IOTTE_CNT];
+	bus_addr_t amask, dvmaddr, iottebase;
+	bus_size_t sgsize, esize;
+	vm_offset_t vaddr, voffs;
+	vm_paddr_t curaddr;
+	int error, sgcnt, firstpg;
+	pmap_t pmap = NULL;
+	pages_t iottecnt;
+
+	KASSERT(buflen != 0, ("hviommu_dvmamap_load_buffer: buflen == 0!"));
+	if (buflen > dt->dt_maxsize)
+		return (EINVAL);
+
+	if (td != NULL)
+		pmap = vmspace_pmap(td->td_proc->p_vmspace);
+
+	vaddr = (vm_offset_t)buf;
+	voffs = vaddr & IO_PAGE_MASK;
+	amask = align ? dt->dt_alignment - 1 : 0;
+
+	/* Try to find a slab that is large enough. */
+	error = hviommu_dvma_vallocseg(dt, him, map, voffs, buflen, amask,
+	    &dvmaddr);
+	if (error != 0)
+		return (error);
+
+	DPRINTF("vallocseg: dvmaddr: %#lx, voffs: %#lx, buflen: %#lx\n",
+	    dvmaddr, voffs, buflen);
+	sgcnt = *segp;
+	firstpg = 1;
+	iottecnt = 0;
+	iottebase = 0;	/* shutup gcc */
+	for (; buflen > 0; ) {
+		/*
+		 * Get the physical address for this page.
+		 */
+		if (pmap != NULL)
+			curaddr = pmap_extract(pmap, vaddr);
+		else
+			curaddr = pmap_kextract(vaddr);
+
+		/*
+		 * Compute the segment size, and adjust counts.
+		 */
+		sgsize = IO_PAGE_SIZE - ((u_long)vaddr & IO_PAGE_MASK);
+		if (buflen < sgsize)
+			sgsize = buflen;
+
+		buflen -= sgsize;
+		vaddr += sgsize;
+
+#if 0
+		hviommu_enter(him, trunc_io_page(dvmaddr), trunc_io_page(curaddr),
+		    flags);
+#else
+		if (iottecnt == 0)
+			iottebase = trunc_io_page(dvmaddr);
+		DPRINTF("adding: %#lx\n", trunc_io_page(curaddr));
+		iottes[iottecnt++] = trunc_io_page(curaddr);
+
+		if (iottecnt >= IOTTE_CNT) {
+			hviommu_map_pages(him, iottebase, iottes, iottecnt);
+			iottecnt = 0;
+		}
+#endif
+
+		/*
+		 * Chop the chunk up into segments of at most maxsegsz, but try
+		 * to fill each segment as well as possible.
+		 */
+		if (!firstpg) {
+			esize = ulmin(sgsize,
+			    dt->dt_maxsegsz - segs[sgcnt].ds_len);
+			segs[sgcnt].ds_len += esize;
+			sgsize -= esize;
+			dvmaddr += esize;
+		}
+		while (sgsize > 0) {
+			sgcnt++;
+			if (sgcnt >= dt->dt_nsegments)
+				return (EFBIG);
+			/*
+			 * No extra alignment here - the common practice in the
+			 * busdma code seems to be that only the first segment
+			 * needs to satisfy the alignment constraints (and that
+			 * only for bus_dmamem_alloc()ed maps). It is assumed
+			 * that such tags have maxsegsize >= maxsize.
+			 */
+			esize = ulmin(sgsize, dt->dt_maxsegsz);
+			segs[sgcnt].ds_addr = dvmaddr;
+			segs[sgcnt].ds_len = esize;
+			sgsize -= esize;
+			dvmaddr += esize;
+		}
+
+		firstpg = 0;
+	}
+	hviommu_map_pages(him, iottebase, iottes, iottecnt);
+	*segp = sgcnt;
+	return (0);
+}
+
+static int
+hviommu_dvmamap_load(bus_dma_tag_t dt, bus_dmamap_t map, void *buf,
+    bus_size_t buflen, bus_dmamap_callback_t *cb, void *cba,
+    int flags)
+{
+	struct hviommu *him = dt->dt_cookie;
+	int error, seg = -1;
+
+	if ((map->dm_flags & DMF_LOADED) != 0) {
+#ifdef DIAGNOSTIC
+		printf("hviommu_dvmamap_load: map still in use\n");
+#endif
+		bus_dmamap_unload(dt, map);
+	}
+
+	/*
+	 * Make sure that the map is not on a queue so that the resource list
+	 * may be safely accessed and modified without needing the lock to
+	 * cover the whole operation.
+	 */
+	HIM_LOCK(him);
+	hviommu_map_remq(him, map);
+	HIM_UNLOCK(him);
+
+	error = hviommu_dvmamap_load_buffer(dt, him, map, buf, buflen, NULL,
+	    flags, dt->dt_segments, &seg, 1);
+
+	HIM_LOCK(him);
+	hviommu_map_insq(him, map);
+	if (error != 0) {
+		hviommu_dvmamap_vunload(him, map);
+		HIM_UNLOCK(him);
+		(*cb)(cba, dt->dt_segments, 0, error);
+	} else {
+		HIM_UNLOCK(him);
+		map->dm_flags |= DMF_LOADED;
+		(*cb)(cba, dt->dt_segments, seg + 1, 0);
+	}
+
+	return (error);
+}
+
+static int
+hviommu_dvmamap_load_mbuf(bus_dma_tag_t dt, bus_dmamap_t map, struct mbuf *m0,
+    bus_dmamap_callback2_t *cb, void *cba, int flags)
+{
+	struct hviommu *him = dt->dt_cookie;
+	struct mbuf *m;
+	int error = 0, first = 1, nsegs = -1;
+
+	M_ASSERTPKTHDR(m0);
+
+	if ((map->dm_flags & DMF_LOADED) != 0) {
+#ifdef DIAGNOSTIC
+		printf("hviommu_dvmamap_load_mbuf: map still in use\n");
+#endif
+		bus_dmamap_unload(dt, map);
+	}
+
+	HIM_LOCK(him);
+	hviommu_map_remq(him, map);
+	HIM_UNLOCK(him);
+
+	if (m0->m_pkthdr.len <= dt->dt_maxsize) {
+		for (m = m0; m != NULL && error == 0; m = m->m_next) {
+			if (m->m_len == 0)
+				continue;
+			error = hviommu_dvmamap_load_buffer(dt, him, map,
+			    m->m_data, m->m_len, NULL, flags, dt->dt_segments,
+			    &nsegs, first);
+			first = 0;
+		}
+	} else
+		error = EINVAL;
+
+	HIM_LOCK(him);
+	hviommu_map_insq(him, map);
+	if (error != 0) {
+		hviommu_dvmamap_vunload(him, map);
+		HIM_UNLOCK(him);
+		/* force "no valid mappings" in callback */
+		(*cb)(cba, dt->dt_segments, 0, 0, error);
+	} else {
+		HIM_UNLOCK(him);
+		map->dm_flags |= DMF_LOADED;
+		(*cb)(cba, dt->dt_segments, nsegs + 1, m0->m_pkthdr.len, 0);
+	}
+	return (error);
+}
+
+static int
+hviommu_dvmamap_load_mbuf_sg(bus_dma_tag_t dt, bus_dmamap_t map,
+    struct mbuf *m0, bus_dma_segment_t *segs, int *nsegs, int flags)
+{
+	struct hviommu *him = dt->dt_cookie;
+	struct mbuf *m;
+	int error = 0, first = 1;
+
+	M_ASSERTPKTHDR(m0);
+
+	*nsegs = -1;
+	if ((map->dm_flags & DMF_LOADED) != 0) {
+#ifdef DIAGNOSTIC
+		printf("hviommu_dvmamap_load_mbuf: map still in use\n");
+#endif
+		bus_dmamap_unload(dt, map);
+	}
+
+	HIM_LOCK(him);
+	hviommu_map_remq(him, map);
+	HIM_UNLOCK(him);
+
+	if (m0->m_pkthdr.len <= dt->dt_maxsize) {
+		for (m = m0; m != NULL && error == 0; m = m->m_next) {
+			if (m->m_len == 0)
+				continue;
+			error = hviommu_dvmamap_load_buffer(dt, him, map,
+			    m->m_data, m->m_len, NULL, flags, segs,
+			    nsegs, first);
+			first = 0;
+		}
+	} else
+		error = EINVAL;
+
+	HIM_LOCK(him);
+	hviommu_map_insq(him, map);
+	if (error != 0) {
+		hviommu_dvmamap_vunload(him, map);
+	} else {
+		map->dm_flags |= DMF_LOADED;
+		++*nsegs;
+	}
+	HIM_UNLOCK(him);
+	return (error);
+}
+
+static int
+hviommu_dvmamap_load_uio(bus_dma_tag_t dt, bus_dmamap_t map, struct uio *uio,
+    bus_dmamap_callback2_t *cb,  void *cba, int flags)
+{
+	struct hviommu *him = dt->dt_cookie;
+	struct iovec *iov;
+	struct thread *td = NULL;
+	bus_size_t minlen, resid;
+	int nsegs = -1, error = 0, first = 1, i;
+
+	if ((map->dm_flags & DMF_LOADED) != 0) {
+#ifdef DIAGNOSTIC
+		printf("hviommu_dvmamap_load_uio: map still in use\n");
+#endif
+		bus_dmamap_unload(dt, map);
+	}
+
+	HIM_LOCK(him);
+	hviommu_map_remq(him, map);
+	HIM_UNLOCK(him);
+
+	resid = uio->uio_resid;
+	iov = uio->uio_iov;
+
+	if (uio->uio_segflg == UIO_USERSPACE) {
+		td = uio->uio_td;
+		KASSERT(td != NULL,
+		    ("%s: USERSPACE but no proc", __func__));
+	}
+
+	for (i = 0; i < uio->uio_iovcnt && resid != 0 && error == 0; i++) {
+		/*
+		 * Now at the first iovec to load.  Load each iovec
+		 * until we have exhausted the residual count.
+		 */
+		minlen = resid < iov[i].iov_len ? resid : iov[i].iov_len;
+		if (minlen == 0)
+			continue;
+
+		error = hviommu_dvmamap_load_buffer(dt, him, map,
+		    iov[i].iov_base, minlen, td, flags, dt->dt_segments, 
+		    &nsegs, first);
+		first = 0;
+
+		resid -= minlen;
+	}
+
+	HIM_LOCK(him);
+	hviommu_map_insq(him, map);
+	if (error) {
+		hviommu_dvmamap_vunload(him, map);
+		HIM_UNLOCK(him);
+		/* force "no valid mappings" in callback */
+		(*cb)(cba, dt->dt_segments, 0, 0, error);
+	} else {
+		HIM_UNLOCK(him);
+		map->dm_flags |= DMF_LOADED;
+		(*cb)(cba, dt->dt_segments, nsegs + 1, uio->uio_resid, 0);
+	}
+	return (error);
+}
+
+static void
+hviommu_dvmamap_unload(bus_dma_tag_t dt, bus_dmamap_t map)
+{
+	struct hviommu *him = dt->dt_cookie;
+
+	if ((map->dm_flags & DMF_LOADED) == 0)
+		return;
+	HIM_LOCK(him);
+	hviommu_dvmamap_vunload(him, map);
+	hviommu_map_insq(him, map);
+	HIM_UNLOCK(him);
+	map->dm_flags &= ~DMF_LOADED;
+}
+
+static void
+hviommu_dvmamap_sync(bus_dma_tag_t dt, bus_dmamap_t map, bus_dmasync_op_t op)
+{
+	struct hviommu *him = dt->dt_cookie;
+	struct bus_dmamap_res *r;
+	vm_offset_t va;
+	vm_size_t len;
+	size_t synced;
+	bus_addr_t ra;
+	uint64_t err;
+	io_attributes_t ioattr;
+	vm_paddr_t raddr;
+	io_sync_direction_t iodir;
+
+	if ((map->dm_flags & DMF_LOADED) == 0)
+		return;
+
+	iodir = 0;
+
+	if (op & (BUS_DMASYNC_POSTREAD))
+		iodir |= IO_SYNC_CPU;
+ 	if (op & (BUS_DMASYNC_PREWRITE))
+		iodir |= IO_SYNC_DEVICE;
+
+	if ((op & (BUS_DMASYNC_PREREAD|BUS_DMASYNC_POSTWRITE)) != 0)
+		membar(Sync);
+
+        /* nothing to be done */
+	if (!iodir) 
+		return;
+
+	HIM_LOCK(him);
+	SLIST_FOREACH(r, &map->dm_reslist, dr_link) {
+		va = (vm_offset_t)BDR_START(r) + r->dr_offset ;
+		len = r->dr_used;
+		while (len > 0) {
+			if ((err = hvio_iommu_getmap(him->him_handle,
+			    VA_TO_TSBID(him, va), &ioattr, &ra))) {
+				if (err != H_ENOMAP)
+					printf("failed to _g=etmap: err: %ld, handle: %#lx, tsbid: %#lx\n", 
+					       err, him->him_handle, VA_TO_TSBID(him, va));
+				continue;
+			}
+			if ((err = hvio_dma_sync(him->him_handle, ra,
+			    ulmin(len, (trunc_io_page(ra) + IO_PAGE_SIZE) - ra),
+			    iodir, &synced))) {
+				printf("failed to dma_sync: err: %ld, handle: %#lx, ra: %#lx, len: %#lx, dir: %d\n",
+				    err, him->him_handle, ra, ulmin(len,
+				    (trunc_io_page(ra) + IO_PAGE_SIZE) - ra),
+				    iodir);
+				synced = ulmin(len, (trunc_io_page(ra) + IO_PAGE_SIZE) - ra);
+				printf("err: %ld", hvio_iommu_getmap(him->him_handle, VA_TO_TSBID(him, va),
+								     &ioattr, &raddr));
+				printf(", ioattr: %d, raddr: %#lx\n", ioattr, raddr);
+			}
+			va += synced;
+			len -= synced;
+		}
+	}
+	HIM_UNLOCK(him);
+
+	if ((op & BUS_DMASYNC_PREWRITE) != 0)
+		membar(Sync);
+}
+
+struct bus_dma_methods hviommu_dma_methods = {
+	.dm_dmamap_create = hviommu_dvmamap_create,
+	.dm_dmamap_destroy = hviommu_dvmamap_destroy,
+	.dm_dmamap_load = hviommu_dvmamap_load,
+	.dm_dmamap_load_mbuf = hviommu_dvmamap_load_mbuf,
+	.dm_dmamap_load_mbuf_sg = hviommu_dvmamap_load_mbuf_sg,
+	.dm_dmamap_load_uio = hviommu_dvmamap_load_uio,
+	.dm_dmamap_unload = hviommu_dvmamap_unload,
+	.dm_dmamap_sync = hviommu_dvmamap_sync,
+	.dm_dmamem_alloc = hviommu_dvmamem_alloc,
+	.dm_dmamem_free = hviommu_dvmamem_free,
+};
diff --git a/sys/sun4v/sun4v/identcpu.c b/sys/sun4v/sun4v/identcpu.c
new file mode 100644
index 000000000000..d5dd6e4d07f1
--- /dev/null
+++ b/sys/sun4v/sun4v/identcpu.c
@@ -0,0 +1,106 @@
+/*-
+ * Initial implementation:
+ * Copyright (c) 2001 Robert Drehmel
+ * All rights reserved.
+ *
+ * As long as the above copyright statement and this notice remain
+ * unchanged, you can do what ever you want with this file.
+ *
+ * $FreeBSD$
+ */
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/kernel.h>
+#include <sys/sysctl.h>
+
+#include <machine/cpufunc.h>
+#include <machine/md_var.h>
+#include <machine/ver.h>
+
+char machine[] = MACHINE;
+SYSCTL_STRING(_hw, HW_MACHINE, machine, CTLFLAG_RD,
+    machine, 0, "Machine class");
+
+static char cpu_model[128];
+SYSCTL_STRING(_hw, HW_MODEL, model, CTLFLAG_RD,
+    cpu_model, 0, "Machine model");
+
+void setPQL2(int *const size, int *const ways);
+
+void
+setPQL2(int *const size, int *const ways)
+{
+	/* XXX SUN4V_FIXME - this is hardcoded for the T1
+	 * this info should really be pulled from the machine description
+	 */
+	*size = 3*1024; 
+	*ways = 12;
+}
+
+void
+cpu_identify(u_long vers, u_int freq, u_int id)
+{
+	const char *manus;
+	const char *impls;
+
+	switch (VER_MANUF(vers)) {
+	case 0x04:
+		manus = "HAL";
+		break;
+	case 0x13:
+	case 0x17:
+	case 0x22:
+	case 0x3e:
+		manus = "Sun Microsystems";
+		break;
+	default:
+		manus = NULL;
+		break;
+	}
+	switch (VER_IMPL(vers)) {
+	case CPU_IMPL_SPARC64:
+		impls = "SPARC64";
+		break;
+	case CPU_IMPL_ULTRASPARCI:
+		impls = "UltraSparc-I";
+		break;
+	case CPU_IMPL_ULTRASPARCII:
+		impls = "UltraSparc-II";
+		break;
+	case CPU_IMPL_ULTRASPARCIIi:
+		impls = "UltraSparc-IIi";
+		break;
+	case CPU_IMPL_ULTRASPARCIIe:
+		/* V9 Manual says `UltraSparc-e'.  I assume this is wrong. */
+		impls = "UltraSparc-IIe";
+		break;
+	case CPU_IMPL_ULTRASPARCIII:
+		impls = "UltraSparc-III";
+		break;
+	case CPU_IMPL_ULTRASPARCIIIp:
+		impls = "UltraSparc-III+";
+		break;
+	case CPU_IMPL_ULTRASPARCIIIi:
+		impls = "UltraSparc-IIIi";
+		break;
+	default:
+		impls = NULL;
+		break;
+	}
+	if (manus == NULL || impls == NULL) {
+		printf(
+		    "CPU: unknown; please e-mail the following value together\n"
+		    "     with the exact name of your processor to "
+		    "<freebsd-sparc64@FreeBSD.org>.\n"
+		    "     version register: <0x%lx>\n", vers);
+		return;
+	}
+
+	snprintf(cpu_model, sizeof(cpu_model), "%s %s", manus, impls);
+	printf("cpu%d: %s %s Processor (%d.%02d MHz CPU)\n", id, manus, impls,
+	    (freq + 4999) / 1000000, ((freq + 4999) / 10000) % 100);
+	if (bootverbose) {
+		printf("  mask=0x%lx maxtl=%ld maxwin=%ld\n", VER_MASK(vers),
+		    VER_MAXTL(vers), VER_MAXWIN(vers));
+	}
+}
diff --git a/sys/sun4v/sun4v/in_cksum.c b/sys/sun4v/sun4v/in_cksum.c
new file mode 100644
index 000000000000..8239b110e772
--- /dev/null
+++ b/sys/sun4v/sun4v/in_cksum.c
@@ -0,0 +1,249 @@
+/*-
+ * Copyright (c) 1990 The Regents of the University of California.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 4. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+/*-
+ * Copyright (c) 2001 by Thomas Moestl <tmm@FreeBSD.org>.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+ * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ *	from tahoe:	in_cksum.c	1.2	86/01/05
+ *	from:		@(#)in_cksum.c	1.3 (Berkeley) 1/19/91
+ * 	from: FreeBSD: src/sys/i386/i386/in_cksum.c,v 1.22 2000/11/25
+ *
+ * $FreeBSD$
+ */
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/mbuf.h>
+
+#include <netinet/in.h>
+#include <netinet/in_systm.h>
+#include <netinet/ip.h>
+
+#include <machine/in_cksum.h>
+
+/*
+ * Checksum routine for Internet Protocol family headers.
+ *
+ * This routine is very heavily used in the network
+ * code and should be modified for each CPU to be as fast as possible.
+ *
+ * This implementation is a sparc64 version. Most code was taken over and
+ * adapted from the i386. Some optimizations were changed to achieve (hopefully)
+ * better performance.
+ * This uses 64 bit loads, but 32 bit additions due to the lack of a 64-bit
+ * add-with-carry operation.
+ */
+
+/*
+ * REDUCE() is actually not used that frequently... maybe a C implementation
+ * would suffice.
+ */
+#define REDUCE(sum, tmp) __asm __volatile( \
+	"sll %2, 16, %1\n" \
+	"addcc %2, %1, %0\n" \
+	"srl %0, 16, %0\n" \
+	"addc %0, 0, %0" : "=r" (sum), "=r" (tmp) : "0" (sum))
+
+/*
+ * Note that some of these macros depend on the flags being preserved between
+ * calls, so they should not be intermixed with other C statements.
+ */
+#define LD64_ADD32(sum, tmp, addr, n, mod) __asm __volatile( \
+	"ldx [%3 + " #n "], %1\n" \
+	"add" #mod " %2, %1, %0\n" \
+	"srlx %1, 32, %1\n" \
+	"addccc %0, %1, %0" : "=r" (sum), "=r" (tmp) : "0" (sum), "r" (addr))
+
+#define LD32_ADD32(sum, tmp, addr, n, mod) __asm __volatile( \
+	"lduw [%3 + " #n "], %1\n" \
+	"add" #mod " %2, %1, %0\n" \
+	: "=r" (sum), "=r" (tmp) : "0" (sum), "r" (addr))
+
+#define MOP(sum) __asm __volatile( \
+	"addc %1, 0, %0" : "=r" (sum) : "0" (sum))
+
+u_short
+in_cksum_skip(struct mbuf *m, int len, int skip)
+{
+	u_short *w;
+	unsigned long tmp, sum = 0;
+	int mlen = 0;
+	int byte_swapped = 0;
+	u_short	su = 0;
+
+	len -= skip;
+	for (; skip > 0 && m != NULL; m = m->m_next) {
+		if (m->m_len > skip) {
+			mlen = m->m_len - skip;
+			w = (u_short *)(mtod(m, u_char *) + skip);
+			goto skip_start;
+		} else
+			skip -= m->m_len;
+	}
+
+	for (; m != NULL && len > 0; m = m->m_next) {
+		if (m->m_len == 0)
+			continue;
+		w = mtod(m, u_short *);
+		if (mlen == -1) {
+			/*
+			 * The first byte of this mbuf is the continuation
+			 * of a word spanning between this mbuf and the
+			 * last mbuf.
+			 *
+			 * The high order byte of su is already saved when
+			 * scanning previous mbuf.  sum was REDUCEd when we
+			 * found mlen == -1
+			 */
+			sum += su | *(u_char *)w;
+			w = (u_short *)((u_char *)w + 1);
+			mlen = m->m_len - 1;
+			len--;
+		} else
+			mlen = m->m_len;
+skip_start:
+		if (len < mlen)
+			mlen = len;
+		len -= mlen;
+		/*
+		 * Force to a 8-byte boundary first so that we can use
+		 * LD64_ADD32.
+		 */
+		if (((u_long)w & 7) != 0) {
+			REDUCE(sum, tmp);
+			if (((u_long)w & 1) != 0 && mlen >= 1) {
+				sum <<= 8;
+				su = *(u_char *)w << 8;
+				w = (u_short *)((u_char *)w + 1);
+				mlen--;
+				byte_swapped = 1;
+			}
+			if (((u_long)w & 2) != 0 && mlen >= 2) {
+				sum += *w++;
+				mlen -= 2;
+			}
+			if (((u_long)w & 4) != 0 && mlen >= 4) {
+				LD32_ADD32(sum, tmp, w, 0, cc);
+				MOP(sum);
+				w += 2;
+				mlen -= 4;
+			}
+		}
+		/*
+		 * Do as much of the checksum as possible 64 bits at at time.
+		 * In fact, this loop is unrolled to make overhead from
+		 * branches &c small.
+		 */
+		for (; mlen >= 64; mlen -= 64) {
+			LD64_ADD32(sum, tmp, w, 0, cc);
+			LD64_ADD32(sum, tmp, w, 8, ccc);
+			LD64_ADD32(sum, tmp, w, 16, ccc);
+			LD64_ADD32(sum, tmp, w, 24, ccc);
+			LD64_ADD32(sum, tmp, w, 32, ccc);
+			LD64_ADD32(sum, tmp, w, 40, ccc);
+			LD64_ADD32(sum, tmp, w, 48, ccc);
+			LD64_ADD32(sum, tmp, w, 56, ccc);
+			MOP(sum);
+			w += 32;
+		}
+		if (mlen >= 32) {
+			LD64_ADD32(sum, tmp, w, 0, cc);
+			LD64_ADD32(sum, tmp, w, 8, ccc);
+			LD64_ADD32(sum, tmp, w, 16, ccc);
+			LD64_ADD32(sum, tmp, w, 24, ccc);
+			MOP(sum);
+			w += 16;
+			mlen -= 32;
+		}
+		if (mlen >= 16) {
+			LD64_ADD32(sum, tmp, w, 0, cc);
+			LD64_ADD32(sum, tmp, w, 8, ccc);
+			MOP(sum);
+			w += 8;
+			mlen -= 16;
+		}
+		if (mlen >= 8) {
+			LD64_ADD32(sum, tmp, w, 0, cc);
+			MOP(sum);
+			w += 4;
+			mlen -= 8;
+		}
+		REDUCE(sum, tmp);
+		while ((mlen -= 2) >= 0)
+			sum += *w++;
+		if (byte_swapped) {
+			sum <<= 8;
+			byte_swapped = 0;
+			if (mlen == -1) {
+				su |= *(u_char *)w;
+				sum += su;
+				mlen = 0;
+			} else
+				mlen = -1;
+		} else if (mlen == -1) {
+			/*
+			 * This mbuf has odd number of bytes.
+			 * There could be a word split betwen
+			 * this mbuf and the next mbuf.
+			 * Save the last byte (to prepend to next mbuf).
+			 */
+			su = *(u_char *)w << 8;
+		}
+	}
+
+	if (len)
+		printf("%s: out of data by %d\n", __func__, len);
+	if (mlen == -1) {
+		/* The last mbuf has odd # of bytes. Follow the
+		   standard (the odd byte is shifted left by 8 bits) */
+		sum += su & 0xff00;
+	}
+	REDUCE(sum, tmp);
+	return (~sum & 0xffff);
+}
diff --git a/sys/sun4v/sun4v/interrupt.S b/sys/sun4v/sun4v/interrupt.S
new file mode 100644
index 000000000000..9fc67f15a14c
--- /dev/null
+++ b/sys/sun4v/sun4v/interrupt.S
@@ -0,0 +1,477 @@
+/*-
+ * Copyright (c) 2002 Jake Burkholder.
+ * Copyright (c) 2006 Kip Macy kmacy@FreeBSD.org
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <machine/asm.h>
+__FBSDID("$FreeBSD$");
+
+#include "opt_simulator.h"
+#include "opt_trap_trace.h"
+#include <machine/hypervisorvar.h>
+#include <machine/asi.h>
+#include <machine/asmacros.h>
+#include <machine/ktr.h>
+#include <machine/pstate.h>
+
+#include "assym.s"
+
+
+#define PUTCHAR(x) \
+	mov	x, %o0		; \
+	mov	CONS_WRITE, %o5	; \
+	ta	FAST_TRAP
+	
+ENTRY(intr_fast)
+	save	%sp, -CCFSZ, %sp
+1:	ldx	[PCPU(IRHEAD)], %l0
+	brnz,a,pt %l0, 2f
+	 nop
+
+	ret
+	 restore
+
+2:	wrpr	%g0, PSTATE_NORMAL, %pstate
+
+	ldx	[%l0 + IR_NEXT], %l1
+	brnz,pt	%l1, 3f
+	 stx	%l1, [PCPU(IRHEAD)]
+	PCPU_ADDR(IRHEAD, %l1)
+	stx	%l1, [PCPU(IRTAIL)]
+
+3:	ldx	[%l0 + IR_FUNC], %o0
+	ldx	[%l0 + IR_ARG], %o1
+	lduw	[%l0 + IR_VEC], %o2
+
+	/* intrcnt[intr_countp[%o2]]++ */
+	SET(intrcnt, %l7, %l2)		/* %l2 = intrcnt */
+	prefetcha [%l2] ASI_N, 1
+	SET(intr_countp, %l7, %l3)	/* %l3 = intr_countp */
+	sllx	%o2, 1, %l4		/* %l4 = vec << 1 */
+	lduh	[%l4 + %l3], %l5	/* %l5 = intr_countp[%o2] */
+	sllx	%l5, 3, %l6		/* %l6 = intr_countp[%o2] << 3 */
+	add	%l6, %l2, %l7		/* %l7 = intrcnt[intr_countp[%o2]] */
+	ldx	[%l7], %l2
+	inc	%l2
+	stx	%l2, [%l7]
+
+	ldx	[PCPU(IRFREE)], %l1
+	stx	%l1, [%l0 + IR_NEXT]
+	stx	%l0, [PCPU(IRFREE)]
+
+	wrpr	%g0, PSTATE_KERNEL, %pstate
+
+	call	%o0
+	 mov	%o1, %o0
+
+	lduw	[%l0 + IR_VEC], %o0
+	call	hvio_intr_setstate
+	  mov	HV_INTR_IDLE_STATE, %o1
+	ba,a	%xcc, 1b
+	 nop
+END(intr_fast)
+
+/*
+ * Running tally of invalid CPU mondo interrupts
+ */
+#if defined(lint)
+uint64_t cpu_mondo_invalid;
+#else /* lint */
+        .data
+        .globl cpu_mondo_invalid
+        .align  8
+cpu_mondo_invalid:
+        .skip   8
+
+        .text
+#endif  /* lint */
+
+#if defined(lint)
+void
+cpu_mondo(void)
+{}
+#else   /* lint */
+
+/*
+ * (TT 0x7c, TL>0) CPU Mondo Queue Handler
+ *      Globals are the Interrupt Globals.
+ *
+ * Interrupts in sun4v are delivered to privileged code in the form 
+ * of interrupt reports.  Each interrupt report is 64-bytes long, 
+ * consisting of 8 64-bit words.  Each interrupt report is appended 
+ * onto the appropriate interrupt queue of which there are currently two,
+ * one for CPU mondos (formerly referred to as cross trap function
+ * requests) and one for device mondos.  Each queue has its own 
+ * trap vector.
+ *
+ * New reports are appended onto the tail of the queue, and privileged
+ * code reads the report from the queue's head.  The head pointer is stored
+ * in a register and is equal to the current offset from the base address 
+ * of its associated queue.  
+ *
+ * The structure of the cpu mondo report follows sun4u conventions:
+ *
+ * word 0:	address of cross-trap handler
+ * word 1:	first argument to handler
+ * word 2:	second argument to handler
+ * word 3-7:	unused
+ *
+ * Privileged code is responsible for incrementing the head pointer 
+ * to remove each entry.  The pointers are updated using modulo
+ * arithmetic such that the queue is empty when head == tail and is full
+ * when the addition of an entry would make head == tail. 
+ * 
+ * This trap handler is called when the cpu mondo queue becomes non-empty
+ * and will continue to be called while interrupts enabled until the 
+ * queue becomes empty.
+ *
+ */
+ENTRY(cpu_mondo)
+        !
+        !       Register Usage:-
+        !       %g5     PC for fasttrap TL>0 handler
+        !       %g1     arg 1   
+        !       %g2     arg 2
+        !       %g3     queue base RA, ackmask
+        !       %g4     queue size mask
+        !       %g6     head ptr
+        mov     CPU_MONDO_QUEUE_HEAD, %g1     
+        ldxa    [%g1]ASI_QUEUE, %g6     ! %g6 = head ptr 
+        mov     CPU_MONDO_QUEUE_TAIL, %g2     
+        ldxa    [%g2]ASI_QUEUE, %g4     ! %g4 = tail ptr 
+        cmp     %g6, %g4
+        be,pn   %xcc, 0f                ! head == tail
+         nop
+
+	/*
+	 * Get the address of the current CPU and index into
+	 * the pcpu structure for the Q values.
+	 */
+	GET_PCPU_SCRATCH
+	ldx	[PCPU(CPU_Q_RA)], %g3		! %g3 = queue base PA
+	ldx	[PCPU(CPU_Q_SIZE)], %g4		! %g4 = queue size
+        sub     %g4, 1, %g4             	! %g4 = queue size mask 
+
+        ! Load interrupt receive data registers 1 and 2 to fetch
+        ! the arguments for the fast trap handler.
+        !
+        ! Since the data words in the interrupt report are not defined yet 
+        ! we assume that the consective words contain valid data and preserve
+        ! sun4u's xcall mondo arguments. 
+        ! Register usage:
+        !       %g5     PC for fasttrap TL>0 handler
+        !       %g1     arg 1   
+        !       %g2     arg 2   
+
+        ldxa    [%g3 + %g6]ASI_REAL, %g5 	! get PC from q base + head
+        add     %g6, 0x8, %g6           	! inc head 8 bytes
+        ldxa    [%g3 + %g6]ASI_REAL, %g1 	! read data word 1
+        add     %g6, 0x8, %g6           	! inc head 8 bytes
+        ldxa    [%g3 + %g6]ASI_REAL, %g2 	! read data word 2
+        add     %g6, 0x8, %g6           	! inc head 8 bytes
+        ldxa    [%g3 + %g6]ASI_REAL, %g7 	! read data word 3
+        add     %g6, (INTR_REPORT_SIZE - 24) , %g6 ! inc head to next record
+
+        and     %g6, %g4, %g6           ! and size mask for wrap around 
+        mov     CPU_MONDO_QUEUE_HEAD, %g3     
+        stxa    %g6, [%g3]ASI_QUEUE     ! store head pointer 
+        membar  #Sync
+	
+	mov	%g7, %g3		! ackmask
+        /*
+         * For now catch invalid PC being passed via cpu_mondo queue
+         */
+        set     KERNBASE, %g4
+        cmp     %g5, %g4
+        bl,a,pn %xcc, 1f                ! branch if bad %pc
+          nop
+
+        jmp     %g5                     ! jump to traphandler
+          nop
+1:
+        ! invalid trap handler, discard it for now
+        set     cpu_mondo_invalid, %g4
+        ldx     [%g4], %g5
+        inc     %g5
+        stx     %g5, [%g4]
+0:
+        retry
+        /* NOTREACHED */
+END(cpu_mondo)
+
+#endif /* lint */
+
+
+#if defined(lint)
+void
+dev_mondo(void)
+{}
+#else   /* lint */
+
+/*
+ * (TT 0x7d, TL>0) Dev Mondo Queue Handler
+ *      Globals are the Interrupt Globals.
+ *
+ * Device interrupt reports take the following form:
+ * Bits:   	63		11 10		6 5		0
+ * word 0: 	|   zero	 | |  bus port  | | device ID 	|
+ * word 1-7: 			unused
+ *
+ */
+ENTRY(dev_mondo)
+        /*
+         *       Register Usage:-
+         *       %g5     inum
+         *       %g1     ptr to intrerrupt vector entry for inum
+         *       %g3     queue base PA 
+         *       %g4     queue size mask
+         *       %g6     head ptr
+	 */
+!	mov %o0, %g1
+!	mov %o5, %g6
+!	PUTCHAR(0x2b)	
+!	PUTCHAR(0x2b)	
+!	mov %g1, %o0
+!	mov %g6, %o5	
+		
+        mov     DEV_MONDO_QUEUE_HEAD, %g1     
+        ldxa    [%g1]ASI_QUEUE, %g6     	! %g6 = head ptr 
+        mov     DEV_MONDO_QUEUE_TAIL, %g2     
+        ldxa    [%g2]ASI_QUEUE, %g4     	! %g4 = tail ptr 
+        cmp     %g6, %g4
+        be,pn   %xcc, 0f                	! head == tail
+        nop
+
+	/*
+	 * Get CPU address and index to Q base
+	 */
+	GET_PCPU_SCRATCH
+        ldx     [PCPU(DEV_Q_RA)], %g3  		! %g3 = queue base PA
+
+        /*
+	 *  Register usage:
+         *       %g5 - inum
+         */
+        ldxa    [%g3 + %g6]ASI_REAL, %g5 	! %g5 = inum from q base + head
+
+        /*
+         * We verify that inum is valid ( < IV_MAX). If it is greater
+         * than IV_MAX, we let the software interrupt handler take care
+	 * of it.
+	 */
+        set     IV_MAX, %g4
+        cmp     %g5, %g4
+        bgeu,a,pn       %xcc, 1f
+          ldx     [PCPU(DEV_Q_SIZE)], %g4		! queue size (delay slot)
+
+        /*
+	 * Find the function, argument and desired priority from the
+	 * intr_vectors table
+	 */
+        set     intr_vectors, %g1
+        sll     %g5, IV_SHIFT, %g2
+        add     %g1, %g2, %g1                   ! %g1 = &intr_vectors[inum]
+
+        /*
+         * Get an intr_request from the free list.  There should always be one
+         * unless we are getting an interrupt storm from stray interrupts, in
+         * which case the we will dereference a NULL pointer and panic.
+         */
+	ldx	[PCPU(IRFREE)], %g2		! %g2 = ptr to tail element
+
+	/*
+	 * Sun4v-fixme: It might be better to have the software interrupt
+	 * handler deal with this scenario rather than just panic.
+	 */
+	!brz,a,pt        %g2, 1f
+	!ldx     [PCPU(DEV_Q_SIZE)], %g4	! queue size (delay slot)
+
+	ldx	[%g2 + IR_NEXT], %g3		! %g3 = next element in list
+	stx	%g3, [PCPU(IRFREE)]		! %g3 is now the tail
+
+	/*
+	 * Store the vector number, function, argument, and priority.
+	 */
+	stw	%g5, [%g2 + IR_VEC]
+	ldx     [%g1 + IV_FUNC], %g3		! %g3 = function ptr
+	stx	%g3, [%g2 + IR_FUNC]
+	ldx	[%g1 + IV_ARG], %g3		! %g3 = argument ptr
+	stx	%g3, [%g2 + IR_ARG]
+	lduw	[%g1 + IV_PRI], %g1		! %g1 = priority
+	stw	%g1, [%g2 + IR_PRI]
+
+	/*
+	 * Link it onto the end of the active list
+	 */
+	stx	%g0, [%g2 + IR_NEXT]
+	ldx	[PCPU(IRTAIL)], %g3		! %g3 = ptr to tail
+	stx	%g2, [%g3]
+	add	%g2, IR_NEXT, %g2
+	stx	%g2, [PCPU(IRTAIL)]
+	
+ 	/* 
+	 * Unlike Solaris, FreeBSD throws away the rest of the 
+	 * 64-byte packet.  We may want to revisit this philosophy
+	 * at some point since information in the packet is defined by 
+	 * the device and MAY be meaningful.
+	 *
+	 * For now, we retain FreeBSD's sun4u semantics.
+	 */
+        ldx     [PCPU(DEV_Q_SIZE)], %g4		! queue size
+
+1:      sub     %g4, 1, %g4             	! %g4 = queue size mask 
+        add     %g6, INTR_REPORT_SIZE, %g6 	! inc head to next record   
+        and     %g6, %g4, %g6           	! and mask for wrap around      
+        mov     DEV_MONDO_QUEUE_HEAD, %g3     
+        stxa    %g6, [%g3]ASI_QUEUE     	! increment head offset 
+        membar  #Sync
+
+	/*
+	 * Trigger a softint at the level indicated by the priority
+	 */
+	mov	1, %g6
+	sllx	%g6, %g1, %g6
+	wr	%g6, 0, %set_softint
+#if 0
+	mov %o0, %g1
+	mov %o5, %g6
+	PUTCHAR(0x2b)	
+	PUTCHAR(0x2b)	
+	mov %g1, %o0
+	mov %g6, %o5	
+#endif
+	/*
+	 * Done, retry the instruction	
+	 */
+0:      retry 
+
+	
+        /* NOTREACHED */
+END(dev_mondo)
+#endif /* lint */
+
+ENTRY(tl_invlctx)
+	mov	%o0, %g2
+	mov	%o1, %g4
+	mov	%o2, %g5
+	mov	%o3, %g6
+	mov	%o5, %g7
+	mov	%g0, %o0
+	mov	%g0, %o1
+	mov	%g1, %o2
+	mov	MAP_ITLB|MAP_DTLB, %o3
+	mov	MMU_DEMAP_CTX, %o5
+	ta	FAST_TRAP
+	brz,pt	%o0, 1f
+	  nop
+	ba	panic_bad_hcall
+	  mov	MMU_DEMAP_CTX, %o1
+1:	
+	mov	%g2, %o0
+	mov	%g4, %o1
+	mov	%g5, %o2
+	mov	%g6, %o3
+	mov	%g7, %o5
+	ba,a,pt	%xcc, set_ackmask
+	  nop
+END(tl_invlctx)
+
+ENTRY(tl_invltlb)
+	mov	%o0, %g1
+	mov	%o1, %g2
+	mov	%o2, %g4
+	mov	%o5, %g5
+	
+	mov	%g0, %o0
+	mov	%g0, %o1
+	mov	MAP_ITLB | MAP_DTLB, %o2
+	mov	MMU_DEMAP_ALL, %o5
+	ta	FAST_TRAP
+	brz,pt	%o0, 1f
+	  nop
+	ba	panic_bad_hcall
+	   mov	MMU_DEMAP_ALL, %o1
+1:
+	mov	%g1, %o0
+	mov	%g2, %o1
+	mov	%g4, %o2
+	mov	%g5, %o5
+	
+	ba,a,pt	%xcc, set_ackmask
+	  nop
+END(tl_invltlb)
+
+ENTRY(tl_invlpg)
+	mov	%o0, %g5
+	mov	%o1, %g6
+	mov	%o2, %g7
+	mov	MAP_ITLB|MAP_DTLB, %o2
+	mov	%g1, %o0
+	mov	%g2, %o1							
+	ta	MMU_UNMAP_ADDR
+	brz,pt	%o0, 1f
+	  nop
+	ba	panic_bad_hcall
+	   mov	MMU_UNMAP_ADDR, %o1
+1:
+	mov	%g5, %o0
+	mov	%g6, %o1
+	mov	%g7, %o2
+	ba,a,pt	%xcc, set_ackmask
+	  nop
+END(tl_invlpg)
+	
+ENTRY(set_ackmask)
+	membar	#Sync
+	GET_PCPU_PHYS_SCRATCH(%g6)
+	wr	%g0, ASI_REAL, %asi
+
+#ifdef TRAP_TRACING
+	/* pcpu->pad[0] = %pc */
+	rd	%pc, %g4
+	stxa	%g4, [PCPU(PAD)]%asi
+
+	/* pcpu->pad[1] = %tick */
+	rdpr	%tick, %g4
+	stxa    %g4, [PCPU(PAD) + 8]%asi
+
+	/*
+	 * Increment a counter which might help us notice if we're
+	 * stuck in a loop. pcpu->pad[2] = count
+	 */
+	ldxa	[PCPU(PAD) + 16]%asi, %g4
+	add	%g4, 1, %g4
+	stxa    %g4, [PCPU(PAD) + 16]%asi
+#endif
+
+	lda	[PCPU(CPUMASK)]%asi, %g4
+	lda	[%g3]%asi, %g1
+1:	or	%g1, %g4, %g2
+	casa	[%g3]%asi, %g1, %g2
+	cmp	%g1, %g2
+	bne,a,pn %icc, 1b
+	  lda	[%g3]%asi, %g1
+	retry
+END(set_ackmask)
diff --git a/sys/sun4v/sun4v/intr_machdep.c b/sys/sun4v/sun4v/intr_machdep.c
new file mode 100644
index 000000000000..8ce3989f5d8d
--- /dev/null
+++ b/sys/sun4v/sun4v/intr_machdep.c
@@ -0,0 +1,470 @@
+/*-
+ * Copyright (c) 1991 The Regents of the University of California.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to Berkeley by
+ * William Jolitz.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 4. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+/*-
+ * Copyright (c) 2001 Jake Burkholder.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	from: @(#)isa.c	7.2 (Berkeley) 5/13/91
+ *	form: src/sys/i386/isa/intr_machdep.c,v 1.57 2001/07/20
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/types.h>
+#include <sys/malloc.h>
+#include <sys/bus.h>
+#include <sys/errno.h>
+#include <sys/interrupt.h>
+#include <sys/kernel.h>
+#include <sys/ktr.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
+#include <sys/pcpu.h>
+#include <sys/proc.h>
+#include <sys/smp.h>
+#include <sys/vmmeter.h>
+
+#include <machine/frame.h>
+#include <machine/intr_machdep.h>
+#include <machine/hypervisor_api.h>
+#include <machine/cpu.h>
+#include <machine/smp.h>
+
+#include <vm/vm.h>
+#include <vm/pmap.h>
+
+#define PANIC_IF(exp) if (unlikely(exp)) {panic("%s: %s:%d", #exp, __FILE__, __LINE__);}
+
+#define	MAX_STRAY_LOG	5
+
+CTASSERT((1 << IV_SHIFT) == sizeof(struct intr_vector));
+
+ih_func_t *intr_handlers[PIL_MAX];
+uint16_t pil_countp[PIL_MAX];
+
+struct intr_vector intr_vectors[IV_MAX];
+uint16_t intr_countp[IV_MAX];
+static u_long intr_stray_count[IV_MAX];
+
+static char *pil_names[] = {
+	"stray",
+	"low",		/* PIL_LOW */
+	"ithrd",	/* PIL_ITHREAD */
+	"rndzvs",	/* PIL_RENDEZVOUS */
+	"ast",		/* PIL_AST */
+	"stop",		/* PIL_STOP */
+	"stray", "stray", "stray", "stray", "stray", "stray", "stray",
+	"fast",		/* PIL_FAST */
+	"tick",		/* PIL_TICK */
+};
+	
+ 
+/*
+ * XXX SUN4V_FIXME - the queue size values should
+ * really be calculated based on the size of the partition
+ *
+ */
+
+int cpu_q_entries = 128;
+int dev_q_entries = 128;
+
+static vm_offset_t *mondo_data_array;
+static vm_offset_t *cpu_list_array;
+static vm_offset_t *cpu_q_array;
+static vm_offset_t *dev_q_array;
+static vm_offset_t *rq_array;
+static vm_offset_t *nrq_array;
+static int cpu_list_size;
+
+
+
+/* protect the intr_vectors table */
+static struct mtx intr_table_lock;
+
+static void intr_execute_handlers(void *);
+static void intr_stray_level(struct trapframe *);
+static void intr_stray_vector(void *);
+static int  intrcnt_setname(const char *, int);
+static void intrcnt_updatename(int, const char *, int);
+static void cpu_intrq_alloc(void);
+
+/*
+ * not MPSAFE
+ */
+static void
+intrcnt_updatename(int vec, const char *name, int ispil)
+{
+	static int intrcnt_index, stray_pil_index, stray_vec_index;
+	int name_index;
+
+	if (intrnames[0] == '\0') {
+		/* for bitbucket */
+		if (bootverbose)
+			printf("initalizing intr_countp\n");
+		intrcnt_setname("???", intrcnt_index++);
+
+		stray_vec_index = intrcnt_index++;
+		intrcnt_setname("stray", stray_vec_index);
+		for (name_index = 0; name_index < IV_MAX; name_index++)
+			intr_countp[name_index] = stray_vec_index;
+
+		stray_pil_index = intrcnt_index++;
+		intrcnt_setname("pil", stray_pil_index);
+		for (name_index = 0; name_index < PIL_MAX; name_index++)
+			pil_countp[name_index] = stray_pil_index;
+	}
+
+	if (name == NULL)
+		name = "???";
+
+	if (!ispil && intr_countp[vec] != stray_vec_index)
+		name_index = intr_countp[vec];
+	else if (ispil && pil_countp[vec] != stray_pil_index)
+		name_index = pil_countp[vec];
+	else
+		name_index = intrcnt_index++;
+
+	if (intrcnt_setname(name, name_index))
+		name_index = 0;
+
+	if (!ispil)
+		intr_countp[vec] = name_index;
+	else
+		pil_countp[vec] = name_index;
+}
+
+static int
+intrcnt_setname(const char *name, int index)
+{
+
+	if (intrnames + (MAXCOMLEN + 1) * index >= eintrnames)
+		return (E2BIG);
+	snprintf(intrnames + (MAXCOMLEN + 1) * index, MAXCOMLEN + 1, "%-*s",
+	    MAXCOMLEN, name);
+	return (0);
+}
+
+void
+intr_setup(int pri, ih_func_t *ihf, int vec, iv_func_t *ivf, void *iva)
+{
+	char pilname[MAXCOMLEN + 1];
+	u_long ps;
+
+	ps = intr_disable();
+	if (vec != -1) {
+		if ((char *)intr_vectors[vec].iv_func == (char *)intr_stray_level) {
+			intr_vectors[vec].iv_func = ivf;
+			intr_vectors[vec].iv_arg = iva;
+		} else {
+			intr_vectors[vec].iv_func = intr_execute_handlers;
+			intr_vectors[vec].iv_arg = &intr_vectors[vec];
+		}
+		if (pri > intr_vectors[vec].iv_pri)
+			intr_vectors[vec].iv_pri = pri;
+		intr_vectors[vec].iv_vec = vec;
+	}
+	snprintf(pilname, MAXCOMLEN + 1, "pil%d: %s", pri, pil_names[pri]);
+	intrcnt_updatename(pri, pilname, 1);
+	intr_handlers[pri] = ihf;
+	intr_restore(ps);
+}
+
+static void
+intr_stray_level(struct trapframe *tf)
+{
+
+	printf("stray level interrupt - pil=%ld\n", tf->tf_pil);
+}
+
+static void
+intr_stray_vector(void *cookie)
+{
+	struct intr_vector *iv;
+
+	iv = cookie;
+	if (intr_stray_count[iv->iv_vec] < MAX_STRAY_LOG) {
+		printf("stray vector interrupt %d\n", iv->iv_vec);
+		intr_stray_count[iv->iv_vec]++;
+		if (intr_stray_count[iv->iv_vec] >= MAX_STRAY_LOG)
+			printf("got %d stray interrupt %d's: not logging "
+			    "anymore\n", MAX_STRAY_LOG, iv->iv_vec);
+	}
+}
+
+static void
+intr_init(void)
+{
+	int i;
+
+	/* Mark all interrupts as being stray. */
+	for (i = 0; i < PIL_MAX; i++)
+		intr_handlers[i] = intr_stray_level;
+	for (i = 0; i < IV_MAX; i++) {
+		intr_vectors[i].iv_func = intr_stray_vector;
+		intr_vectors[i].iv_arg = &intr_vectors[i];
+		intr_vectors[i].iv_pri = PIL_LOW;
+		intr_vectors[i].iv_vec = i;
+	}
+	intr_handlers[PIL_LOW] = intr_fast;
+	
+#ifdef SMP
+	intr_handlers[PIL_AST] = cpu_ipi_ast;
+	intr_handlers[PIL_RENDEZVOUS] = (ih_func_t *)smp_rendezvous_action;
+	intr_handlers[PIL_STOP]= cpu_ipi_stop;
+#endif
+	mtx_init(&intr_table_lock, "intr table", NULL, MTX_SPIN);
+	cpu_intrq_alloc();
+	cpu_intrq_init();
+
+}
+SYSINIT(intr_init, SI_SUB_INTR, SI_ORDER_FIRST, intr_init, NULL);
+
+
+static void
+intr_execute_handlers(void *cookie)
+{
+	struct intr_vector *iv;
+	struct intr_event *ie;
+	struct intr_handler *ih;
+	int error, thread;
+
+	iv = cookie;
+	ie = iv->iv_event;
+	if (ie == NULL) {
+		intr_stray_vector(iv);
+		return;
+	}
+
+	/* Execute fast interrupt handlers directly. */
+	thread = 0;
+	TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) {
+		if (!(ih->ih_flags & IH_FAST)) {
+			thread = 1;
+			continue;
+		}
+		MPASS(ih->ih_flags & IH_FAST && ih->ih_argument != NULL);
+		CTR3(KTR_INTR, "%s: executing handler %p(%p)", __func__,
+		    ih->ih_handler, ih->ih_argument);
+		ih->ih_handler(ih->ih_argument);
+	}
+
+	hvio_intr_setstate(iv->iv_vec, HV_INTR_IDLE_STATE);
+
+	/* Schedule a heavyweight interrupt process. */
+	if (thread)
+		error = intr_event_schedule_thread(ie);
+	else if (TAILQ_EMPTY(&ie->ie_handlers))
+		error = EINVAL;
+	else
+		error = 0;
+	if (error == EINVAL)
+		intr_stray_vector(iv);
+}
+
+int
+inthand_add(const char *name, int vec, void (*handler)(void *), void *arg,
+    int flags, void **cookiep)
+{
+	struct intr_vector *iv;
+	struct intr_event *ie;		/* descriptor for the IRQ */
+	struct intr_event *orphan;
+	int errcode, pil;
+	iv_func_t *ivf;
+	void *iva;
+	/*
+	 * Work around a race where more than one CPU may be registering
+	 * handlers on the same IRQ at the same time.
+	 */
+	iv = &intr_vectors[vec];
+	mtx_lock_spin(&intr_table_lock);
+	ie = iv->iv_event;
+	mtx_unlock_spin(&intr_table_lock);
+	if (ie == NULL) {
+		errcode = intr_event_create(&ie, (void *)(intptr_t)vec, 0, NULL,
+		    "vec%d:", vec);
+		if (errcode)
+			return (errcode);
+		mtx_lock_spin(&intr_table_lock);
+		if (iv->iv_event == NULL) {
+			iv->iv_event = ie;
+			mtx_unlock_spin(&intr_table_lock);
+		} else {
+			orphan = ie;
+			ie = iv->iv_event;
+			mtx_unlock_spin(&intr_table_lock);
+			intr_event_destroy(orphan);
+		}
+	}
+
+	errcode = intr_event_add_handler(ie, name, handler, arg,
+	    intr_priority(flags), flags, cookiep);
+	if (errcode)
+		return (errcode);
+	
+	pil = (flags & INTR_FAST) ? PIL_FAST : PIL_ITHREAD;
+	ivf = (flags & INTR_FAST) ? handler : intr_execute_handlers;
+	iva = (flags & INTR_FAST) ? arg : iv;
+
+	intr_setup(pil, intr_fast, vec, ivf, iva);
+
+	intr_stray_count[vec] = 0;
+
+	intrcnt_updatename(vec, ie->ie_fullname, 0);
+
+	return (0);
+}
+
+int
+inthand_remove(int vec, void *cookie)
+{
+	struct intr_vector *iv;
+	int error;
+	
+	error = intr_event_remove_handler(cookie);
+	if (error == 0) {
+		/*
+		 * XXX: maybe this should be done regardless of whether
+		 * intr_event_remove_handler() succeeded?
+		 * XXX: aren't the PIL's backwards below?
+		 */
+		iv = &intr_vectors[vec];
+		mtx_lock_spin(&intr_table_lock);
+		if (iv->iv_event == NULL)
+			intr_setup(PIL_ITHREAD, intr_fast, vec,
+			    intr_stray_vector, iv);
+		else
+			intr_setup(PIL_LOW, intr_fast, vec,
+			    intr_execute_handlers, iv);
+		mtx_unlock_spin(&intr_table_lock);
+	}
+	return (error);
+}
+
+/* 
+ * Allocate and register intrq fields
+ */
+static void 
+cpu_intrq_alloc(void)
+{
+	
+
+	
+	mondo_data_array = malloc(INTR_REPORT_SIZE*MAXCPU, M_DEVBUF, M_WAITOK | M_ZERO);
+	PANIC_IF(mondo_data_array == NULL);
+
+	cpu_list_size = CPU_LIST_SIZE > INTR_REPORT_SIZE ? CPU_LIST_SIZE : INTR_REPORT_SIZE;
+	cpu_list_array = malloc(cpu_list_size*MAXCPU, M_DEVBUF, M_WAITOK | M_ZERO);
+	PANIC_IF(cpu_list_array == NULL);
+	
+	cpu_q_array = malloc(INTR_CPU_Q_SIZE*MAXCPU, M_DEVBUF, M_WAITOK | M_ZERO);
+	PANIC_IF(cpu_q_array == NULL);
+	
+	dev_q_array = malloc(INTR_DEV_Q_SIZE*MAXCPU, M_DEVBUF, M_WAITOK | M_ZERO);
+	PANIC_IF(dev_q_array == NULL);
+
+	rq_array = malloc(2*CPU_RQ_SIZE*MAXCPU, M_DEVBUF, M_WAITOK | M_ZERO);
+	PANIC_IF(rq_array == NULL);
+	
+	nrq_array = malloc(2*CPU_NRQ_SIZE*MAXCPU, M_DEVBUF, M_WAITOK | M_ZERO);
+	PANIC_IF(nrq_array == NULL);
+
+}
+
+void 
+cpu_intrq_init()
+{
+
+	uint64_t error;
+
+	pcpup->pc_mondo_data = (vm_offset_t *) ((char *)mondo_data_array + curcpu*INTR_REPORT_SIZE);
+	pcpup->pc_mondo_data_ra = vtophys(pcpup->pc_mondo_data);
+
+	pcpup->pc_cpu_q = (vm_offset_t *)((char *)cpu_q_array + curcpu*INTR_CPU_Q_SIZE);
+
+	pcpup->pc_cpu_q_ra = vtophys(pcpup->pc_cpu_q);
+	pcpup->pc_cpu_q_size = INTR_CPU_Q_SIZE;
+
+	pcpup->pc_dev_q = (vm_offset_t *)((char *)dev_q_array + curcpu*INTR_DEV_Q_SIZE);
+	pcpup->pc_dev_q_ra = vtophys(pcpup->pc_dev_q);
+	pcpup->pc_dev_q_size = INTR_DEV_Q_SIZE;
+
+	pcpup->pc_rq = (vm_offset_t *)((char *)rq_array + curcpu*2*CPU_RQ_SIZE);
+	pcpup->pc_rq_ra = vtophys(pcpup->pc_rq);
+	pcpup->pc_rq_size = CPU_RQ_SIZE;
+
+	pcpup->pc_nrq = (vm_offset_t *)((char *)nrq_array + curcpu*2*CPU_NRQ_SIZE);
+	pcpup->pc_nrq_ra = vtophys(pcpup->pc_nrq);
+	pcpup->pc_nrq_size = CPU_NRQ_SIZE;
+
+
+	error = hv_cpu_qconf(Q(CPU_MONDO_QUEUE_HEAD), pcpup->pc_cpu_q_ra, cpu_q_entries);
+	if (error != H_EOK)
+		panic("cpu_mondo queue configuration failed: %lu va=%p ra=0x%lx", error,
+		      pcpup->pc_cpu_q, pcpup->pc_cpu_q_ra);
+
+	error = hv_cpu_qconf(Q(DEV_MONDO_QUEUE_HEAD), pcpup->pc_dev_q_ra, dev_q_entries);
+	if (error != H_EOK)
+		panic("dev_mondo queue configuration failed: %lu", error);
+
+	error = hv_cpu_qconf(Q(RESUMABLE_ERROR_QUEUE_HEAD), pcpup->pc_rq_ra, CPU_RQ_ENTRIES);
+	if (error != H_EOK)
+		panic("resumable error queue configuration failed: %lu", error);
+
+	error = hv_cpu_qconf(Q(NONRESUMABLE_ERROR_QUEUE_HEAD), pcpup->pc_nrq_ra, CPU_NRQ_ENTRIES);
+	if (error != H_EOK)
+		panic("non-resumable error queue configuration failed: %lu", error);
+
+}
diff --git a/sys/sun4v/sun4v/iommu.c b/sys/sun4v/sun4v/iommu.c
new file mode 100644
index 000000000000..178ac4e2b49e
--- /dev/null
+++ b/sys/sun4v/sun4v/iommu.c
@@ -0,0 +1,1276 @@
+/*-
+ * Copyright (c) 1999, 2000 Matthew R. Green
+ * Copyright (c) 2001-2003 Thomas Moestl
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The name of the author may not be used to endorse or promote products
+ *    derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+/*-
+ * Copyright (c) 1998 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The NetBSD Foundation
+ * by Paul Kranenburg.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ *    must display the following acknowledgement:
+ *        This product includes software developed by the NetBSD
+ *        Foundation, Inc. and its contributors.
+ * 4. Neither the name of The NetBSD Foundation nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+/*-
+ * Copyright (c) 1992, 1993
+ *	The Regents of the University of California.  All rights reserved.
+ *
+ * This software was developed by the Computer Systems Engineering group
+ * at Lawrence Berkeley Laboratory under DARPA contract BG 91-66 and
+ * contributed to Berkeley.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 4. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	from: NetBSD: sbus.c,v 1.13 1999/05/23 07:24:02 mrg Exp
+ *	from: @(#)sbus.c	8.1 (Berkeley) 6/11/93
+ *	from: NetBSD: iommu.c,v 1.42 2001/08/06 22:02:58 eeh Exp
+ *
+ * $FreeBSD$
+ */
+
+/*
+ * UltraSPARC IOMMU support; used by both the sbus and pci code.
+ * Currently, the IOTSBs are synchronized, because determining the bus the map
+ * is to be loaded for is not possible with the current busdma code.
+ * The code is structured so that the IOMMUs can be easily divorced when that
+ * is fixed.
+ *
+ * TODO:
+ * - As soon as there is a newbus way to get a parent dma tag, divorce the
+ *   IOTSBs.
+ * - Support sub-page boundaries.
+ * - Fix alignment handling for small allocations (the possible page offset
+ *   of malloc()ed memory is not handled at all). Revise interaction of
+ *   alignment with the load_mbuf and load_uio functions.
+ * - Handle lowaddr and highaddr in some way, and try to work out a way
+ *   for filter callbacks to work. Currently, only lowaddr is honored
+ *   in that no addresses above it are considered at all.
+ * - Implement BUS_DMA_ALLOCNOW in bus_dma_tag_create as far as possible.
+ * - Check the possible return values and callback error arguments;
+ *   the callback currently gets called in error conditions where it should
+ *   not be.
+ * - When running out of DVMA space, return EINPROGRESS in the non-
+ *   BUS_DMA_NOWAIT case and delay the callback until sufficient space
+ *   becomes available.
+ * - Use the streaming cache unless BUS_DMA_COHERENT is specified; do not
+ *   flush the streaming cache when coherent mappings are synced.
+ * - Add bounce buffers to support machines with more than 16GB of RAM.
+ */
+
+#include "opt_iommu.h"
+
+#include <sys/param.h>
+#include <sys/kernel.h>
+#include <sys/lock.h>
+#include <sys/malloc.h>
+#include <sys/mbuf.h>
+#include <sys/mutex.h>
+#include <sys/proc.h>
+#include <sys/systm.h>
+#include <sys/uio.h>
+
+#include <vm/vm.h>
+#include <vm/pmap.h>
+#include <vm/vm_map.h>
+
+#include <machine/bus.h>
+#include <machine/bus_private.h>
+#include <machine/iommureg.h>
+#include <machine/pmap.h>
+#include <machine/resource.h>
+
+#include <sys/rman.h>
+
+#include <machine/iommuvar.h>
+
+/*
+ * Tuning constants.
+ */
+#define	IOMMU_MAX_PRE		(32 * 1024)
+#define	IOMMU_MAX_PRE_SEG	3
+
+/* Threshold for using the streaming buffer. */
+#define	IOMMU_STREAM_THRESH	128
+
+MALLOC_DEFINE(M_IOMMU, "dvmamem", "IOMMU DVMA Buffers");
+
+static	int iommu_strbuf_flush_sync(struct iommu_state *);
+#ifdef IOMMU_DIAG
+static 	void iommu_diag(struct iommu_state *, vm_offset_t va);
+#endif
+
+/*
+ * Protects iommu_maplruq, dm_reslist of all maps on the queue and all
+ * iommu states as long as the TSBs are synchronized.
+ */
+struct mtx iommu_mtx;
+
+/*
+ * The following 4 variables need to be moved to the per-IOMMU state once
+ * the IOTSBs are divorced.
+ * LRU queue handling for lazy resource allocation.
+ */
+static TAILQ_HEAD(iommu_maplruq_head, bus_dmamap) iommu_maplruq =
+   TAILQ_HEAD_INITIALIZER(iommu_maplruq);
+
+/* DVMA space rman. */
+static struct rman iommu_dvma_rman;
+
+/* Virtual and physical address of the TSB. */
+static u_int64_t *iommu_tsb;
+static vm_offset_t iommu_ptsb;
+
+/* List of all IOMMUs. */
+static STAILQ_HEAD(, iommu_state) iommu_insts =
+   STAILQ_HEAD_INITIALIZER(iommu_insts);
+
+/*
+ * Helpers. Some of these take unused iommu states as parameters, to ease the
+ * transition to divorced TSBs.
+ */
+#define	IOMMU_READ8(is, reg, off) 					\
+	bus_space_read_8((is)->is_bustag, (is)->is_bushandle, 		\
+	    (is)->reg + (off))
+#define	IOMMU_WRITE8(is, reg, off, v)					\
+	bus_space_write_8((is)->is_bustag, (is)->is_bushandle, 		\
+	    (is)->reg + (off), (v))
+
+#define	IOMMU_HAS_SB(is)						\
+	((is)->is_sb[0] != 0 || (is)->is_sb[1] != 0)
+
+/*
+ * Always overallocate one page; this is needed to handle alignment of the
+ * buffer, so it makes sense using a lazy allocation scheme.
+ */
+#define	IOMMU_SIZE_ROUNDUP(sz)						\
+	(round_io_page(sz) + IO_PAGE_SIZE)
+
+#define	IOMMU_SET_TTE(is, va, tte)					\
+	(iommu_tsb[IOTSBSLOT(va)] = (tte))
+#define	IOMMU_GET_TTE(is, va)						\
+	iommu_tsb[IOTSBSLOT(va)]
+
+/* Resource helpers */
+#define	IOMMU_RES_START(res)						\
+	((bus_addr_t)rman_get_start(res) << IO_PAGE_SHIFT)
+#define	IOMMU_RES_END(res)						\
+	((bus_addr_t)(rman_get_end(res) + 1) << IO_PAGE_SHIFT)
+#define	IOMMU_RES_SIZE(res)						\
+	((bus_size_t)rman_get_size(res) << IO_PAGE_SHIFT)
+
+/* Helpers for struct bus_dmamap_res */
+#define	BDR_START(r)	IOMMU_RES_START((r)->dr_res)
+#define	BDR_END(r)	IOMMU_RES_END((r)->dr_res)
+#define	BDR_SIZE(r)	IOMMU_RES_SIZE((r)->dr_res)
+
+/* Locking macros. */
+#define	IS_LOCK(is)	mtx_lock(&iommu_mtx)
+#define	IS_LOCK_ASSERT(is)	mtx_assert(&iommu_mtx, MA_OWNED)
+#define	IS_UNLOCK(is)	mtx_unlock(&iommu_mtx)
+
+
+/* Flush a page from the TLB. No locking required, since this is atomic. */
+static __inline void
+iommu_tlb_flush(struct iommu_state *is, bus_addr_t va)
+{
+	struct iommu_state *it;
+
+	/*
+	 * Since the TSB is shared for now, the TLBs of all IOMMUs
+	 * need to be flushed.
+	 */
+	STAILQ_FOREACH(it, &iommu_insts, is_link)
+		IOMMU_WRITE8(it, is_iommu, IMR_FLUSH, va);
+}
+
+/*
+ * Flush a page from the streaming buffer. No locking required, since this is
+ * atomic.
+ */
+static __inline void
+iommu_strbuf_flushpg(struct iommu_state *is, bus_addr_t va)
+{
+	int i;
+
+	for (i = 0; i < 2; i++) {
+		if (is->is_sb[i] != 0)
+			IOMMU_WRITE8(is, is_sb[i], ISR_PGFLUSH, va);
+	}
+}
+
+/*
+ * Flush an address from the streaming buffer(s); this is an asynchronous
+ * operation. To make sure that it has completed, iommu_strbuf_sync() needs
+ * to be called. No locking required.
+ */
+static __inline void
+iommu_strbuf_flush(struct iommu_state *is, bus_addr_t va)
+{
+	struct iommu_state *it;
+
+	/*
+	 * Need to flush the streaming buffers of all IOMMUs, we cannot
+	 * determine which one was used for the transaction.
+	 */
+	STAILQ_FOREACH(it, &iommu_insts, is_link)
+		iommu_strbuf_flushpg(it, va);
+}
+
+/* Synchronize all outstanding flush operations. */
+static __inline void
+iommu_strbuf_sync(struct iommu_state *is)
+{
+	struct iommu_state *it;
+
+	IS_LOCK_ASSERT(is);
+	/*
+	 * Need to sync the streaming buffers of all IOMMUs, we cannot
+	 * determine which one was used for the transaction.
+	 */
+	STAILQ_FOREACH(it, &iommu_insts, is_link)
+		iommu_strbuf_flush_sync(it);
+}
+
+/* LRU queue handling for lazy resource allocation. */
+static __inline void
+iommu_map_insq(struct iommu_state *is, bus_dmamap_t map)
+{
+
+	IS_LOCK_ASSERT(is);
+	if (!SLIST_EMPTY(&map->dm_reslist)) {
+		if (map->dm_onq)
+			TAILQ_REMOVE(&iommu_maplruq, map, dm_maplruq);
+		TAILQ_INSERT_TAIL(&iommu_maplruq, map, dm_maplruq);
+		map->dm_onq = 1;
+	}
+}
+
+static __inline void
+iommu_map_remq(struct iommu_state *is, bus_dmamap_t map)
+{
+
+	IS_LOCK_ASSERT(is);
+	if (map->dm_onq)
+		TAILQ_REMOVE(&iommu_maplruq, map, dm_maplruq);
+	map->dm_onq = 0;
+}
+
+/*
+ * initialise the UltraSPARC IOMMU (SBus or PCI):
+ *	- allocate and setup the iotsb.
+ *	- enable the IOMMU
+ *	- initialise the streaming buffers (if they exist)
+ *	- create a private DVMA map.
+ */
+void
+iommu_init(char *name, struct iommu_state *is, int tsbsize, u_int32_t iovabase,
+    int resvpg)
+{
+	struct iommu_state *first;
+	vm_size_t size;
+	vm_offset_t offs;
+	u_int64_t end;
+	int i;
+
+	/*
+	 * Setup the iommu.
+	 *
+	 * The sun4u iommu is part of the SBUS or PCI controller so we
+	 * will deal with it here..
+	 *
+	 * The IOMMU address space always ends at 0xffffe000, but the starting
+	 * address depends on the size of the map.  The map size is 1024 * 2 ^
+	 * is->is_tsbsize entries, where each entry is 8 bytes.  The start of
+	 * the map can be calculated by (0xffffe000 << (8 + is->is_tsbsize)).
+	 */
+	is->is_cr = (tsbsize << IOMMUCR_TSBSZ_SHIFT) | IOMMUCR_EN;
+	is->is_tsbsize = tsbsize;
+	is->is_dvmabase = iovabase;
+	if (iovabase == -1)
+		is->is_dvmabase = IOTSB_VSTART(is->is_tsbsize);
+
+	size = IOTSB_BASESZ << is->is_tsbsize;
+	printf("%s: DVMA map: %#lx to %#lx\n", name,
+	    is->is_dvmabase, is->is_dvmabase +
+	    (size << (IO_PAGE_SHIFT - IOTTE_SHIFT)) - 1);
+
+	if (STAILQ_EMPTY(&iommu_insts)) {
+		/*
+		 * First IOMMU to be registered; set up resource mamangement
+		 * and allocate TSB memory.
+		 */
+		mtx_init(&iommu_mtx, "iommu", NULL, MTX_DEF);
+		end = is->is_dvmabase + (size << (IO_PAGE_SHIFT - IOTTE_SHIFT));
+		iommu_dvma_rman.rm_type = RMAN_ARRAY;
+		iommu_dvma_rman.rm_descr = "DVMA Memory";
+		if (rman_init(&iommu_dvma_rman) != 0 ||
+		    rman_manage_region(&iommu_dvma_rman,
+		    (is->is_dvmabase >> IO_PAGE_SHIFT) + resvpg,
+		    (end >> IO_PAGE_SHIFT) - 1) != 0)
+			panic("iommu_init: can't initialize dvma rman");
+		/*
+		 * Allocate memory for I/O page tables.  They need to be
+		 * physically contiguous.
+		 */
+		iommu_tsb = contigmalloc(size, M_DEVBUF, M_NOWAIT, 0, ~0UL,
+		    PAGE_SIZE, 0);
+		if (iommu_tsb == 0)
+			panic("iommu_init: contigmalloc failed");
+		iommu_ptsb = pmap_kextract((vm_offset_t)iommu_tsb);
+		bzero(iommu_tsb, size);
+	} else {
+		/*
+		 * Not the first IOMMU; just check that the parameters match
+		 * those of the first one.
+		 */
+		first = STAILQ_FIRST(&iommu_insts);
+		if (is->is_tsbsize != first->is_tsbsize ||
+		    is->is_dvmabase != first->is_dvmabase) {
+			panic("iommu_init: secondary IOMMU state does not "
+			    "match primary");
+		}
+	}
+	STAILQ_INSERT_TAIL(&iommu_insts, is, is_link);
+
+	/*
+	 * Initialize streaming buffer, if it is there.
+	 */
+	if (IOMMU_HAS_SB(is)) {
+		/*
+		 * Find two 64-byte blocks in is_flush that are aligned on
+		 * a 64-byte boundary for flushing.
+		 */
+		offs = roundup2((vm_offset_t)is->is_flush,
+		    STRBUF_FLUSHSYNC_NBYTES);
+		for (i = 0; i < 2; i++, offs += STRBUF_FLUSHSYNC_NBYTES) {
+			is->is_flushva[i] = (int64_t *)offs;
+			is->is_flushpa[i] = pmap_kextract(offs);
+		}
+	}
+
+	/*
+	 * Now actually start up the IOMMU.
+	 */
+	iommu_reset(is);
+}
+
+/*
+ * Streaming buffers don't exist on the UltraSPARC IIi; we should have
+ * detected that already and disabled them.  If not, we will notice that
+ * they aren't there when the STRBUF_EN bit does not remain.
+ */
+void
+iommu_reset(struct iommu_state *is)
+{
+	int i;
+
+	IOMMU_WRITE8(is, is_iommu, IMR_TSB, iommu_ptsb);
+	/* Enable IOMMU in diagnostic mode */
+	IOMMU_WRITE8(is, is_iommu, IMR_CTL, is->is_cr | IOMMUCR_DE);
+
+	for (i = 0; i < 2; i++) {
+		if (is->is_sb[i] != 0) {
+			/* Enable diagnostics mode? */
+			IOMMU_WRITE8(is, is_sb[i], ISR_CTL, STRBUF_EN);
+
+			/* No streaming buffers? Disable them */
+			if (IOMMU_READ8(is, is_sb[i], ISR_CTL) == 0)
+				is->is_sb[i] = 0;
+		}
+	}
+}
+
+/*
+ * Enter a mapping into the TSB. No locking required, since each TSB slot is
+ * uniquely assigned to a single map.
+ */
+static void
+iommu_enter(struct iommu_state *is, vm_offset_t va, vm_paddr_t pa,
+    int stream, int flags)
+{
+	int64_t tte;
+
+	KASSERT(va >= is->is_dvmabase,
+	    ("iommu_enter: va %#lx not in DVMA space", va));
+	KASSERT(pa < IOMMU_MAXADDR,
+	    ("iommu_enter: XXX: physical address too large (%#lx)", pa));
+
+	tte = MAKEIOTTE(pa, !(flags & BUS_DMA_NOWRITE),
+	    !(flags & BUS_DMA_NOCACHE), stream);
+
+	IOMMU_SET_TTE(is, va, tte);
+	iommu_tlb_flush(is, va);
+#ifdef IOMMU_DIAG
+	IS_LOCK(is);
+	iommu_diag(is, va);
+	IS_UNLOCK(is);
+#endif
+}
+
+/*
+ * Remove mappings created by iommu_enter. Flush the streaming buffer, but do
+ * not synchronize it. Returns whether a streaming buffer flush was performed.
+ */
+static int
+iommu_remove(struct iommu_state *is, vm_offset_t va, vm_size_t len)
+{
+	int streamed = 0;
+
+#ifdef IOMMU_DIAG
+	iommu_diag(is, va);
+#endif
+
+	KASSERT(va >= is->is_dvmabase,
+	    ("iommu_remove: va 0x%lx not in DVMA space", (u_long)va));
+	KASSERT(va + len >= va,
+	    ("iommu_remove: va 0x%lx + len 0x%lx wraps", (long)va, (long)len));
+
+	va = trunc_io_page(va);
+	while (len > 0) {
+		if ((IOMMU_GET_TTE(is, va) & IOTTE_STREAM) != 0) {
+			streamed = 1;
+			iommu_strbuf_flush(is, va);
+		}
+		len -= ulmin(len, IO_PAGE_SIZE);
+		IOMMU_SET_TTE(is, va, 0);
+		iommu_tlb_flush(is, va);
+		va += IO_PAGE_SIZE;
+	}
+	return (streamed);
+}
+
+/* Decode an IOMMU fault for host bridge error handlers. */
+void
+iommu_decode_fault(struct iommu_state *is, vm_offset_t phys)
+{
+	bus_addr_t va;
+	long idx;
+
+	idx = phys - iommu_ptsb;
+	if (phys < iommu_ptsb ||
+	    idx > (PAGE_SIZE << is->is_tsbsize))
+		return;
+	va = is->is_dvmabase +
+	    (((bus_addr_t)idx >> IOTTE_SHIFT) << IO_PAGE_SHIFT);
+	printf("IOMMU fault virtual address %#lx\n", (u_long)va);
+}
+
+/*
+ * A barrier operation which makes sure that all previous streaming buffer
+ * flushes complete before it returns.
+ */
+static int
+iommu_strbuf_flush_sync(struct iommu_state *is)
+{
+	struct timeval cur, end;
+	int i;
+
+	IS_LOCK_ASSERT(is);
+	if (!IOMMU_HAS_SB(is))
+		return (0);
+
+	/*
+	 * Streaming buffer flushes:
+	 *
+	 *   1 Tell strbuf to flush by storing va to strbuf_pgflush.  If
+	 *     we're not on a cache line boundary (64-bits):
+	 *   2 Store 0 in flag
+	 *   3 Store pointer to flag in flushsync
+	 *   4 wait till flushsync becomes 0x1
+	 *
+	 * If it takes more than .5 sec, something
+	 * went wrong.
+	 */
+	*is->is_flushva[0] = 1;
+	*is->is_flushva[1] = 1;
+	membar(StoreStore);
+	for (i = 0; i < 2; i++) {
+		if (is->is_sb[i] != 0) {
+			*is->is_flushva[i] = 0;
+			IOMMU_WRITE8(is, is_sb[i], ISR_FLUSHSYNC,
+			    is->is_flushpa[i]);
+		}
+	}
+
+	microuptime(&cur);
+	end.tv_sec = 0;
+	/*
+	 * 0.5s is the recommended timeout from the U2S manual. The actual
+	 * time required should be smaller by at least a factor of 1000.
+	 * We have no choice but to busy-wait.
+	 */
+	end.tv_usec = 500000;
+	timevaladd(&end, &cur);
+
+	while ((!*is->is_flushva[0] || !*is->is_flushva[1]) &&
+	    timevalcmp(&cur, &end, <=))
+		microuptime(&cur);
+
+	if (!*is->is_flushva[0] || !*is->is_flushva[1]) {
+		panic("iommu_strbuf_flush_done: flush timeout %ld, %ld at %#lx",
+		    *is->is_flushva[0], *is->is_flushva[1], is->is_flushpa[0]);
+	}
+
+	return (1);
+}
+
+/* Determine whether we may enable streaming on a mapping. */
+static __inline int
+iommu_use_streaming(struct iommu_state *is, bus_dmamap_t map, bus_size_t size)
+{
+
+	/*
+	 * This cannot be enabled yet, as many driver are still missing
+	 * bus_dmamap_sync() calls. As soon as there is a BUS_DMA_STREAMING
+	 * flag, this should be reenabled conditionally on it.
+	 */
+#ifdef notyet
+	return (size >= IOMMU_STREAM_THRESH && IOMMU_HAS_SB(is) &&
+	    (map->dm_flags & DMF_COHERENT) == 0);
+#else
+	return (0);
+#endif
+}
+
+/*
+ * Allocate DVMA virtual memory for a map. The map may not be on a queue, so
+ * that it can be freely modified.
+ */
+static int
+iommu_dvma_valloc(bus_dma_tag_t t, struct iommu_state *is, bus_dmamap_t map,
+    bus_size_t size)
+{
+	struct resource *res;
+	struct bus_dmamap_res *bdr;
+	bus_size_t align, sgsize;
+
+	KASSERT(!map->dm_onq, ("iommu_dvma_valloc: map on queue!"));
+	if ((bdr = malloc(sizeof(*bdr), M_IOMMU, M_NOWAIT)) == NULL)
+		return (EAGAIN);
+	/*
+	 * If a boundary is specified, a map cannot be larger than it; however
+	 * we do not clip currently, as that does not play well with the lazy
+	 * allocation code.
+	 * Alignment to a page boundary is always enforced.
+	 */
+	align = (t->dt_alignment + IO_PAGE_MASK) >> IO_PAGE_SHIFT;
+	sgsize = round_io_page(size) >> IO_PAGE_SHIFT;
+	if (t->dt_boundary > 0 && t->dt_boundary < IO_PAGE_SIZE)
+		panic("iommu_dvmamap_load: illegal boundary specified");
+	res = rman_reserve_resource_bound(&iommu_dvma_rman, 0L,
+	    t->dt_lowaddr >> IO_PAGE_SHIFT, sgsize,
+	    t->dt_boundary >> IO_PAGE_SHIFT,
+	    RF_ACTIVE | rman_make_alignment_flags(align), NULL);
+	if (res == NULL) {
+		free(bdr, M_IOMMU);
+		return (ENOMEM);
+	}
+
+	bdr->dr_res = res;
+	bdr->dr_used = 0;
+	SLIST_INSERT_HEAD(&map->dm_reslist, bdr, dr_link);
+	return (0);
+}
+
+/* Unload the map and mark all resources as unused, but do not free them. */
+static void
+iommu_dvmamap_vunload(struct iommu_state *is, bus_dmamap_t map)
+{
+	struct bus_dmamap_res *r;
+	int streamed = 0;
+
+	IS_LOCK_ASSERT(is);	/* for iommu_strbuf_sync() below. */
+	SLIST_FOREACH(r, &map->dm_reslist, dr_link) {
+		streamed |= iommu_remove(is, BDR_START(r), r->dr_used);
+		r->dr_used = 0;
+	}
+	if (streamed)
+		iommu_strbuf_sync(is);
+}
+
+/* Free a DVMA virtual memory resource. */
+static __inline void
+iommu_dvma_vfree_res(bus_dmamap_t map, struct bus_dmamap_res *r)
+{
+
+	KASSERT(r->dr_used == 0, ("iommu_dvma_vfree_res: resource busy!"));
+	if (r->dr_res != NULL && rman_release_resource(r->dr_res) != 0)
+		printf("warning: DVMA space lost\n");
+	SLIST_REMOVE(&map->dm_reslist, r, bus_dmamap_res, dr_link);
+	free(r, M_IOMMU);
+}
+
+/* Free all DVMA virtual memory for a map. */
+static void
+iommu_dvma_vfree(struct iommu_state *is, bus_dmamap_t map)
+{
+
+	IS_LOCK(is);
+	iommu_map_remq(is, map);
+	iommu_dvmamap_vunload(is, map);
+	IS_UNLOCK(is);
+	while (!SLIST_EMPTY(&map->dm_reslist))
+		iommu_dvma_vfree_res(map, SLIST_FIRST(&map->dm_reslist));
+}
+
+/* Prune a map, freeing all unused DVMA resources. */
+static bus_size_t
+iommu_dvma_vprune(struct iommu_state *is, bus_dmamap_t map)
+{
+	struct bus_dmamap_res *r, *n;
+	bus_size_t freed = 0;
+
+	IS_LOCK_ASSERT(is);
+	for (r = SLIST_FIRST(&map->dm_reslist); r != NULL; r = n) {
+		n = SLIST_NEXT(r, dr_link);
+		if (r->dr_used == 0) {
+			freed += BDR_SIZE(r);
+			iommu_dvma_vfree_res(map, r);
+		}
+	}
+	if (SLIST_EMPTY(&map->dm_reslist))
+		iommu_map_remq(is, map);
+	return (freed);
+}
+
+/*
+ * Try to find a suitably-sized (and if requested, -aligned) slab of DVMA
+ * memory with IO page offset voffs.
+ */
+static bus_addr_t
+iommu_dvma_vfindseg(bus_dmamap_t map, vm_offset_t voffs, bus_size_t size,
+    bus_addr_t amask)
+{
+	struct bus_dmamap_res *r;
+	bus_addr_t dvmaddr, dvmend;
+
+	KASSERT(!map->dm_onq, ("iommu_dvma_vfindseg: map on queue!"));
+	SLIST_FOREACH(r, &map->dm_reslist, dr_link) {
+		dvmaddr = round_io_page(BDR_START(r) + r->dr_used);
+		/* Alignment can only work with voffs == 0. */
+		dvmaddr = (dvmaddr + amask) & ~amask;
+		dvmaddr += voffs;
+		dvmend = dvmaddr + size;
+		if (dvmend <= BDR_END(r)) {
+			r->dr_used = dvmend - BDR_START(r);
+			return (dvmaddr);
+		}
+	}
+	return (0);
+}
+
+/*
+ * Try to find or allocate a slab of DVMA space; see above.
+ */
+static int
+iommu_dvma_vallocseg(bus_dma_tag_t dt, struct iommu_state *is, bus_dmamap_t map,
+    vm_offset_t voffs, bus_size_t size, bus_addr_t amask, bus_addr_t *addr)
+{
+	bus_dmamap_t tm, last;
+	bus_addr_t dvmaddr, freed;
+	int error, complete = 0;
+
+	dvmaddr = iommu_dvma_vfindseg(map, voffs, size, amask);
+
+	/* Need to allocate. */
+	if (dvmaddr == 0) {
+		while ((error = iommu_dvma_valloc(dt, is, map,
+			voffs + size)) == ENOMEM && !complete) {
+			/*
+			 * Free the allocated DVMA of a few maps until
+			 * the required size is reached. This is an
+			 * approximation to not have to call the allocation
+			 * function too often; most likely one free run
+			 * will not suffice if not one map was large enough
+			 * itself due to fragmentation.
+			 */
+			IS_LOCK(is);
+			freed = 0;
+			last = TAILQ_LAST(&iommu_maplruq, iommu_maplruq_head);
+			do {
+				tm = TAILQ_FIRST(&iommu_maplruq);
+				complete = tm == last;
+				if (tm == NULL)
+					break;
+				freed += iommu_dvma_vprune(is, tm);
+				/* Move to the end. */
+				iommu_map_insq(is, tm);
+			} while (freed < size && !complete);
+			IS_UNLOCK(is);
+		}
+		if (error != 0)
+			return (error);
+		dvmaddr = iommu_dvma_vfindseg(map, voffs, size, amask);
+		KASSERT(dvmaddr != 0,
+		    ("iommu_dvma_vallocseg: allocation failed unexpectedly!"));
+	}
+	*addr = dvmaddr;
+	return (0);
+}
+
+static int
+iommu_dvmamem_alloc(bus_dma_tag_t dt, void **vaddr, int flags,
+    bus_dmamap_t *mapp)
+{
+	struct iommu_state *is = dt->dt_cookie;
+	int error, mflags;
+
+	/*
+	 * XXX: This will break for 32 bit transfers on machines with more than
+	 * 16G (1 << 34 bytes) of memory.
+	 */
+	if ((error = sparc64_dma_alloc_map(dt, mapp)) != 0)
+		return (error);
+
+	if ((flags & BUS_DMA_NOWAIT) != 0)
+		mflags = M_NOWAIT;
+	else
+		mflags = M_WAITOK;
+	if ((flags & BUS_DMA_ZERO) != 0)
+		mflags |= M_ZERO;
+
+	if ((*vaddr = malloc(dt->dt_maxsize, M_IOMMU, mflags)) == NULL) {
+		error = ENOMEM;
+		sparc64_dma_free_map(dt, *mapp);
+		return (error);
+	}
+	if ((flags & BUS_DMA_COHERENT) != 0)
+		(*mapp)->dm_flags |= DMF_COHERENT;
+	/*
+	 * Try to preallocate DVMA space. If this fails, it is retried at load
+	 * time.
+	 */
+	iommu_dvma_valloc(dt, is, *mapp, IOMMU_SIZE_ROUNDUP(dt->dt_maxsize));
+	IS_LOCK(is);
+	iommu_map_insq(is, *mapp);
+	IS_UNLOCK(is);
+	return (0);
+}
+
+static void
+iommu_dvmamem_free(bus_dma_tag_t dt, void *vaddr, bus_dmamap_t map)
+{
+	struct iommu_state *is = dt->dt_cookie;
+
+	iommu_dvma_vfree(is, map);
+	sparc64_dma_free_map(dt, map);
+	free(vaddr, M_IOMMU);
+}
+
+static int
+iommu_dvmamap_create(bus_dma_tag_t dt, int flags, bus_dmamap_t *mapp)
+{
+	struct iommu_state *is = dt->dt_cookie;
+	bus_size_t totsz, presz, currsz;
+	int error, i, maxpre;
+
+	if ((error = sparc64_dma_alloc_map(dt, mapp)) != 0)
+		return (error);
+	if ((flags & BUS_DMA_COHERENT) != 0)
+		(*mapp)->dm_flags |= DMF_COHERENT;
+	/*
+	 * Preallocate DVMA space; if this fails now, it is retried at load
+	 * time. Through bus_dmamap_load_mbuf() and bus_dmamap_load_uio(), it
+	 * is possible to have multiple discontiguous segments in a single map,
+	 * which is handled by allocating additional resources, instead of
+	 * increasing the size, to avoid fragmentation.
+	 * Clamp preallocation to IOMMU_MAX_PRE. In some situations we can
+	 * handle more; that case is handled by reallocating at map load time.
+	 */
+	totsz = ulmin(IOMMU_SIZE_ROUNDUP(dt->dt_maxsize), IOMMU_MAX_PRE); 
+	error = iommu_dvma_valloc(dt, is, *mapp, totsz);
+	if (error != 0)
+		return (0);
+	/*
+	 * Try to be smart about preallocating some additional segments if
+	 * needed.
+	 */
+	maxpre = imin(dt->dt_nsegments, IOMMU_MAX_PRE_SEG);
+	presz = dt->dt_maxsize / maxpre;
+	KASSERT(presz != 0, ("iommu_dvmamap_create: bogus preallocation size "
+	    ", nsegments = %d, maxpre = %d, maxsize = %lu", dt->dt_nsegments,
+	    maxpre, dt->dt_maxsize));
+	for (i = 1; i < maxpre && totsz < IOMMU_MAX_PRE; i++) {
+		currsz = round_io_page(ulmin(presz, IOMMU_MAX_PRE - totsz));
+		error = iommu_dvma_valloc(dt, is, *mapp, currsz);
+		if (error != 0)
+			break;
+		totsz += currsz;
+	}
+	IS_LOCK(is);
+	iommu_map_insq(is, *mapp);
+	IS_UNLOCK(is);
+	return (0);
+}
+
+static int
+iommu_dvmamap_destroy(bus_dma_tag_t dt, bus_dmamap_t map)
+{
+	struct iommu_state *is = dt->dt_cookie;
+
+	iommu_dvma_vfree(is, map);
+	sparc64_dma_free_map(dt, map);
+	return (0);
+}
+
+/*
+ * IOMMU DVMA operations, common to SBUS and PCI.
+ */
+static int
+iommu_dvmamap_load_buffer(bus_dma_tag_t dt, struct iommu_state *is,
+    bus_dmamap_t map, void *buf, bus_size_t buflen, struct thread *td,
+    int flags, bus_dma_segment_t *segs, int *segp, int align)
+{
+	bus_addr_t amask, dvmaddr;
+	bus_size_t sgsize, esize;
+	vm_offset_t vaddr, voffs;
+	vm_paddr_t curaddr;
+	int error, sgcnt, firstpg, stream;
+	pmap_t pmap = NULL;
+
+	KASSERT(buflen != 0, ("iommu_dvmamap_load_buffer: buflen == 0!"));
+	if (buflen > dt->dt_maxsize)
+		return (EINVAL);
+
+	if (td != NULL)
+		pmap = vmspace_pmap(td->td_proc->p_vmspace);
+
+	vaddr = (vm_offset_t)buf;
+	voffs = vaddr & IO_PAGE_MASK;
+	amask = align ? dt->dt_alignment - 1 : 0;
+
+	/* Try to find a slab that is large enough. */
+	error = iommu_dvma_vallocseg(dt, is, map, voffs, buflen, amask,
+	    &dvmaddr);
+	if (error != 0)
+		return (error);
+
+	sgcnt = *segp;
+	firstpg = 1;
+	stream = iommu_use_streaming(is, map, buflen);
+	for (; buflen > 0; ) {
+		/*
+		 * Get the physical address for this page.
+		 */
+		if (pmap != NULL)
+			curaddr = pmap_extract(pmap, vaddr);
+		else
+			curaddr = pmap_kextract(vaddr);
+
+		/*
+		 * Compute the segment size, and adjust counts.
+		 */
+		sgsize = IO_PAGE_SIZE - ((u_long)vaddr & IO_PAGE_MASK);
+		if (buflen < sgsize)
+			sgsize = buflen;
+
+		buflen -= sgsize;
+		vaddr += sgsize;
+
+		iommu_enter(is, trunc_io_page(dvmaddr), trunc_io_page(curaddr),
+		    stream, flags);
+
+		/*
+		 * Chop the chunk up into segments of at most maxsegsz, but try
+		 * to fill each segment as well as possible.
+		 */
+		if (!firstpg) {
+			esize = ulmin(sgsize,
+			    dt->dt_maxsegsz - segs[sgcnt].ds_len);
+			segs[sgcnt].ds_len += esize;
+			sgsize -= esize;
+			dvmaddr += esize;
+		}
+		while (sgsize > 0) {
+			sgcnt++;
+			if (sgcnt >= dt->dt_nsegments)
+				return (EFBIG);
+			/*
+			 * No extra alignment here - the common practice in the
+			 * busdma code seems to be that only the first segment
+			 * needs to satisfy the alignment constraints (and that
+			 * only for bus_dmamem_alloc()ed maps). It is assumed
+			 * that such tags have maxsegsize >= maxsize.
+			 */
+			esize = ulmin(sgsize, dt->dt_maxsegsz);
+			segs[sgcnt].ds_addr = dvmaddr;
+			segs[sgcnt].ds_len = esize;
+			sgsize -= esize;
+			dvmaddr += esize;
+		}
+
+		firstpg = 0;
+	}
+	*segp = sgcnt;
+	return (0);
+}
+
+static int
+iommu_dvmamap_load(bus_dma_tag_t dt, bus_dmamap_t map, void *buf,
+    bus_size_t buflen, bus_dmamap_callback_t *cb, void *cba,
+    int flags)
+{
+	struct iommu_state *is = dt->dt_cookie;
+	int error, seg = -1;
+
+	if ((map->dm_flags & DMF_LOADED) != 0) {
+#ifdef DIAGNOSTIC
+		printf("iommu_dvmamap_load: map still in use\n");
+#endif
+		bus_dmamap_unload(dt, map);
+	}
+
+	/*
+	 * Make sure that the map is not on a queue so that the resource list
+	 * may be safely accessed and modified without needing the lock to
+	 * cover the whole operation.
+	 */
+	IS_LOCK(is);
+	iommu_map_remq(is, map);
+	IS_UNLOCK(is);
+
+	error = iommu_dvmamap_load_buffer(dt, is, map, buf, buflen, NULL,
+	    flags, dt->dt_segments, &seg, 1);
+
+	IS_LOCK(is);
+	iommu_map_insq(is, map);
+	if (error != 0) {
+		iommu_dvmamap_vunload(is, map);
+		IS_UNLOCK(is);
+		(*cb)(cba, dt->dt_segments, 0, error);
+	} else {
+		IS_UNLOCK(is);
+		map->dm_flags |= DMF_LOADED;
+		(*cb)(cba, dt->dt_segments, seg + 1, 0);
+	}
+
+	return (error);
+}
+
+static int
+iommu_dvmamap_load_mbuf(bus_dma_tag_t dt, bus_dmamap_t map, struct mbuf *m0,
+    bus_dmamap_callback2_t *cb, void *cba, int flags)
+{
+	struct iommu_state *is = dt->dt_cookie;
+	struct mbuf *m;
+	int error = 0, first = 1, nsegs = -1;
+
+	M_ASSERTPKTHDR(m0);
+
+	if ((map->dm_flags & DMF_LOADED) != 0) {
+#ifdef DIAGNOSTIC
+		printf("iommu_dvmamap_load_mbuf: map still in use\n");
+#endif
+		bus_dmamap_unload(dt, map);
+	}
+
+	IS_LOCK(is);
+	iommu_map_remq(is, map);
+	IS_UNLOCK(is);
+
+	if (m0->m_pkthdr.len <= dt->dt_maxsize) {
+		for (m = m0; m != NULL && error == 0; m = m->m_next) {
+			if (m->m_len == 0)
+				continue;
+			error = iommu_dvmamap_load_buffer(dt, is, map,
+			    m->m_data, m->m_len, NULL, flags, dt->dt_segments,
+			    &nsegs, first);
+			first = 0;
+		}
+	} else
+		error = EINVAL;
+
+	IS_LOCK(is);
+	iommu_map_insq(is, map);
+	if (error != 0) {
+		iommu_dvmamap_vunload(is, map);
+		IS_UNLOCK(is);
+		/* force "no valid mappings" in callback */
+		(*cb)(cba, dt->dt_segments, 0, 0, error);
+	} else {
+		IS_UNLOCK(is);
+		map->dm_flags |= DMF_LOADED;
+		(*cb)(cba, dt->dt_segments, nsegs + 1, m0->m_pkthdr.len, 0);
+	}
+	return (error);
+}
+
+static int
+iommu_dvmamap_load_mbuf_sg(bus_dma_tag_t dt, bus_dmamap_t map, struct mbuf *m0,
+    bus_dma_segment_t *segs, int *nsegs, int flags)
+{
+	struct iommu_state *is = dt->dt_cookie;
+	struct mbuf *m;
+	int error = 0, first = 1;
+
+	M_ASSERTPKTHDR(m0);
+
+	*nsegs = -1;
+	if ((map->dm_flags & DMF_LOADED) != 0) {
+#ifdef DIAGNOSTIC
+		printf("iommu_dvmamap_load_mbuf: map still in use\n");
+#endif
+		bus_dmamap_unload(dt, map);
+	}
+
+	IS_LOCK(is);
+	iommu_map_remq(is, map);
+	IS_UNLOCK(is);
+
+	if (m0->m_pkthdr.len <= dt->dt_maxsize) {
+		for (m = m0; m != NULL && error == 0; m = m->m_next) {
+			if (m->m_len == 0)
+				continue;
+			error = iommu_dvmamap_load_buffer(dt, is, map,
+			    m->m_data, m->m_len, NULL, flags, segs,
+			    nsegs, first);
+			first = 0;
+		}
+	} else
+		error = EINVAL;
+
+	IS_LOCK(is);
+	iommu_map_insq(is, map);
+	if (error != 0) {
+		iommu_dvmamap_vunload(is, map);
+	} else {
+		map->dm_flags |= DMF_LOADED;
+		++*nsegs;
+	}
+	IS_UNLOCK(is);
+	return (error);
+}
+
+static int
+iommu_dvmamap_load_uio(bus_dma_tag_t dt, bus_dmamap_t map, struct uio *uio,
+    bus_dmamap_callback2_t *cb,  void *cba, int flags)
+{
+	struct iommu_state *is = dt->dt_cookie;
+	struct iovec *iov;
+	struct thread *td = NULL;
+	bus_size_t minlen, resid;
+	int nsegs = -1, error = 0, first = 1, i;
+
+	if ((map->dm_flags & DMF_LOADED) != 0) {
+#ifdef DIAGNOSTIC
+		printf("iommu_dvmamap_load_uio: map still in use\n");
+#endif
+		bus_dmamap_unload(dt, map);
+	}
+
+	IS_LOCK(is);
+	iommu_map_remq(is, map);
+	IS_UNLOCK(is);
+
+	resid = uio->uio_resid;
+	iov = uio->uio_iov;
+
+	if (uio->uio_segflg == UIO_USERSPACE) {
+		td = uio->uio_td;
+		KASSERT(td != NULL,
+		    ("%s: USERSPACE but no proc", __func__));
+	}
+
+	for (i = 0; i < uio->uio_iovcnt && resid != 0 && error == 0; i++) {
+		/*
+		 * Now at the first iovec to load.  Load each iovec
+		 * until we have exhausted the residual count.
+		 */
+		minlen = resid < iov[i].iov_len ? resid : iov[i].iov_len;
+		if (minlen == 0)
+			continue;
+
+		error = iommu_dvmamap_load_buffer(dt, is, map,
+		    iov[i].iov_base, minlen, td, flags, dt->dt_segments, 
+		    &nsegs, first);
+		first = 0;
+
+		resid -= minlen;
+	}
+
+	IS_LOCK(is);
+	iommu_map_insq(is, map);
+	if (error) {
+		iommu_dvmamap_vunload(is, map);
+		IS_UNLOCK(is);
+		/* force "no valid mappings" in callback */
+		(*cb)(cba, dt->dt_segments, 0, 0, error);
+	} else {
+		IS_UNLOCK(is);
+		map->dm_flags |= DMF_LOADED;
+		(*cb)(cba, dt->dt_segments, nsegs + 1, uio->uio_resid, 0);
+	}
+	return (error);
+}
+
+static void
+iommu_dvmamap_unload(bus_dma_tag_t dt, bus_dmamap_t map)
+{
+	struct iommu_state *is = dt->dt_cookie;
+
+	if ((map->dm_flags & DMF_LOADED) == 0)
+		return;
+	IS_LOCK(is);
+	iommu_dvmamap_vunload(is, map);
+	iommu_map_insq(is, map);
+	IS_UNLOCK(is);
+	map->dm_flags &= ~DMF_LOADED;
+}
+
+static void
+iommu_dvmamap_sync(bus_dma_tag_t dt, bus_dmamap_t map, bus_dmasync_op_t op)
+{
+	struct iommu_state *is = dt->dt_cookie;
+	struct bus_dmamap_res *r;
+	vm_offset_t va;
+	vm_size_t len;
+	int streamed = 0;
+
+	if ((map->dm_flags & DMF_LOADED) == 0)
+		return;
+	/* XXX This is probably bogus. */
+	if ((op & BUS_DMASYNC_PREREAD) != 0)
+		membar(Sync);
+	if (IOMMU_HAS_SB(is) &&
+	    ((op & BUS_DMASYNC_POSTREAD) != 0 ||
+	     (op & BUS_DMASYNC_PREWRITE) != 0)) {
+		IS_LOCK(is);
+		SLIST_FOREACH(r, &map->dm_reslist, dr_link) {
+			va = (vm_offset_t)BDR_START(r);
+			len = r->dr_used;
+			/* if we have a streaming buffer, flush it here first */
+			while (len > 0) {
+				if ((IOMMU_GET_TTE(is, va) & IOTTE_STREAM) != 0) {
+					streamed = 1;
+					iommu_strbuf_flush(is, va);
+				}
+				len -= ulmin(len, IO_PAGE_SIZE);
+				va += IO_PAGE_SIZE;
+			}
+		}
+		if (streamed)
+			iommu_strbuf_sync(is);
+		IS_UNLOCK(is);
+	}
+	if ((op & BUS_DMASYNC_PREWRITE) != 0)
+		membar(Sync);
+}
+
+#ifdef IOMMU_DIAG
+
+/*
+ * Perform an IOMMU diagnostic access and print the tag belonging to va.
+ */
+static void
+iommu_diag(struct iommu_state *is, vm_offset_t va)
+{
+	int i;
+	u_int64_t tag, data;
+
+	IS_LOCK_ASSERT(is);
+	IOMMU_WRITE8(is, is_dva, 0, trunc_io_page(va));
+	membar(StoreStore | StoreLoad);
+	printf("iommu_diag: tte entry %#lx", IOMMU_GET_TTE(is, va));
+	if (is->is_dtcmp != 0) {
+		printf(", tag compare register is %#lx\n",
+		    IOMMU_READ8(is, is_dtcmp, 0));
+	} else
+		printf("\n");
+	for (i = 0; i < 16; i++) {
+		tag = IOMMU_READ8(is, is_dtag, i * 8);
+		data = IOMMU_READ8(is, is_ddram, i * 8);
+		printf("iommu_diag: tag %d: %#lx, vpn %#lx, err %lx; "
+		    "data %#lx, pa %#lx, v %d, c %d\n", i,
+		    tag, (tag & IOMMU_DTAG_VPNMASK) << IOMMU_DTAG_VPNSHIFT,
+		    (tag & IOMMU_DTAG_ERRMASK) >> IOMMU_DTAG_ERRSHIFT, data,
+		    (data & IOMMU_DDATA_PGMASK) << IOMMU_DDATA_PGSHIFT,
+		    (data & IOMMU_DDATA_V) != 0, (data & IOMMU_DDATA_C) != 0);
+	}
+}
+
+#endif /* IOMMU_DIAG */
+
+struct bus_dma_methods iommu_dma_methods = {
+	iommu_dvmamap_create,
+	iommu_dvmamap_destroy,
+	iommu_dvmamap_load,
+	iommu_dvmamap_load_mbuf,
+	iommu_dvmamap_load_mbuf_sg,
+	iommu_dvmamap_load_uio,
+	iommu_dvmamap_unload,
+	iommu_dvmamap_sync,
+	iommu_dvmamem_alloc,
+	iommu_dvmamem_free,
+};
diff --git a/sys/sun4v/sun4v/locore.S b/sys/sun4v/sun4v/locore.S
new file mode 100644
index 000000000000..dade43292df7
--- /dev/null
+++ b/sys/sun4v/sun4v/locore.S
@@ -0,0 +1,111 @@
+/*-
+ * Copyright (c) 2001 Jake Burkholder.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+
+	
+#include <machine/asm.h>
+__FBSDID("$FreeBSD$"); 
+
+#include <machine/asi.h>
+#include <machine/asmacros.h>
+#include <machine/pstate.h>
+
+#include "assym.s"
+
+	.register %g2,#ignore
+
+	.globl	kernbase
+	.set	kernbase,KERNBASE
+	
+	.globl	nwin_minus_one
+	.set	nwin_minus_one,7
+	
+/*
+ * void _start(caddr_t metadata, u_long o1, u_long o2, u_long o3,
+ *	       u_long ofw_vec)
+ */
+ENTRY(btext)
+ENTRY(_start)
+	/*
+	 * Initialize misc state to known values.  Interrupts disabled, normal
+	 * globals, windows flushed (cr = 0, cs = nwindows - 1), no clean
+	 * windows, pil 0, and floating point disabled.
+	 */
+	wrpr	%g0, PSTATE_NORMAL, %pstate
+	flushw
+	wrpr	%g0, 0, %cleanwin
+	wrpr	%g0, 0, %pil
+	wr	%g0, 0, %fprs
+
+	/*
+	 * Get onto our per-cpu panic stack, which precedes the struct pcpu in
+	 * the per-cpu page.
+	 */
+	SET(pcpu0 + (PCPU_PAGES * PAGE_SIZE) - PC_SIZEOF, %l1, %l0)
+	sub	%l0, SPOFF + CCFSZ, %sp
+
+	/*
+	 * Do initial bootstrap to setup pmap and thread0.
+	 */
+	call	sparc64_init
+	 nop
+
+	GET_PCB(%g6)
+	/*
+	 * Get onto thread0's kstack.
+	 */
+	ldx	[%g6 + PCB_KSTACK], %sp
+
+	/*
+	 * And away we go.  This doesn't return.
+	 */
+	call	mi_startup
+	 nop
+	sir
+	! NOTREACHED
+END(_start)
+
+/*
+ * void cpu_setregs(struct pcpu *pc)
+ */
+ENTRY(cpu_setregs)
+
+	/*
+	 * Disable interrupts, normal globals.
+	 */
+	mov	%o0, PCPU_REG
+
+	/*
+	 * store PCPU_REG into its scratchpad register
+	 * this allows us to retrieve it in the trap handlers - 
+	 * where we receive a new set of globals
+	 */
+	mov	SCRATCH_REG_PCPU, %g2 
+	stxa	PCPU_REG, [%g2]ASI_SCRATCHPAD 
+
+	retl
+	 nop
+END(cpu_setregs)
diff --git a/sys/sun4v/sun4v/machdep.c b/sys/sun4v/sun4v/machdep.c
new file mode 100644
index 000000000000..1d8208f7320d
--- /dev/null
+++ b/sys/sun4v/sun4v/machdep.c
@@ -0,0 +1,945 @@
+/*-
+ * Copyright (c) 2001 Jake Burkholder.
+ * Copyright (c) 1992 Terrence R. Lambert.
+ * Copyright (c) 1982, 1987, 1990 The Regents of the University of California.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to Berkeley by
+ * William Jolitz.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 4. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	from: @(#)machdep.c	7.4 (Berkeley) 6/3/91
+ * 	from: FreeBSD: src/sys/i386/i386/machdep.c,v 1.477 2001/08/27
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include "opt_compat.h"
+#include "opt_ddb.h"
+#include "opt_kstack_pages.h"
+#include "opt_msgbuf.h"
+
+#include <sys/param.h>
+#include <sys/malloc.h>
+#include <sys/proc.h>
+#include <sys/systm.h>
+#include <sys/bio.h>
+#include <sys/buf.h>
+#include <sys/bus.h>
+#include <sys/cpu.h>
+#include <sys/cons.h>
+#include <sys/eventhandler.h>
+#include <sys/exec.h>
+#include <sys/imgact.h>
+#include <sys/interrupt.h>
+#include <sys/kdb.h>
+#include <sys/kernel.h>
+#include <sys/ktr.h>
+#include <sys/linker.h>
+#include <sys/lock.h>
+#include <sys/msgbuf.h>
+#include <sys/mutex.h>
+#include <sys/pcpu.h>
+#include <sys/ptrace.h>
+#include <sys/reboot.h>
+#include <sys/signalvar.h>
+#include <sys/smp.h>
+#include <sys/sysent.h>
+#include <sys/sysproto.h>
+#include <sys/timetc.h>
+#include <sys/ucontext.h>
+
+#include <dev/ofw/openfirm.h>
+
+#include <vm/vm.h>
+#include <vm/vm_extern.h>
+#include <vm/vm_kern.h>
+#include <vm/vm_page.h>
+#include <vm/vm_map.h>
+#include <vm/vm_object.h>
+#include <vm/vm_pager.h>
+#include <vm/vm_param.h>
+
+#include <ddb/ddb.h>
+
+#include <machine/bus.h>
+#include <machine/cache.h>
+#include <machine/clock.h>
+#include <machine/cpu.h>
+#include <machine/fp.h>
+#include <machine/fsr.h>
+#include <machine/intr_machdep.h>
+#include <machine/md_var.h>
+#include <machine/metadata.h>
+#include <machine/ofw_machdep.h>
+#include <machine/ofw_mem.h>
+#include <machine/pcb.h>
+#include <machine/pmap.h>
+#include <machine/pstate.h>
+#include <machine/reg.h>
+#include <machine/sigframe.h>
+#include <machine/smp.h>
+#include <machine/tick.h>
+#include <machine/tlb.h>
+#include <machine/tstate.h>
+#include <machine/asm.h>
+#include <machine/ver.h>
+#include <machine/hypervisor_api.h>
+
+/* XXX move this to a header */
+extern void mdesc_init(void);
+
+typedef int ofw_vec_t(void *);
+
+#ifdef DDB
+extern vm_offset_t ksym_start, ksym_end;
+#endif
+
+struct tlb_entry *kernel_tlbs;
+int kernel_tlb_slots;
+
+int cold = 1;
+long Maxmem;
+long realmem;
+
+char pcpu0[PCPU_PAGES * PAGE_SIZE];
+struct trapframe frame0;
+
+vm_offset_t kstack0;
+vm_paddr_t kstack0_phys;
+
+struct kva_md_info kmi;
+
+u_long ofw_vec;
+u_long ofw_tba;
+
+/*
+ * Note: timer quality for CPU's is set low to try and prevent them from
+ * being chosen as the primary timecounter.  The CPU counters are not
+ * synchronized among the CPU's so in MP machines this causes problems
+ * when calculating the time.  With this value the CPU's should only be
+ * chosen as the primary timecounter as a last resort.
+ */
+
+#define	UP_TICK_QUALITY	1000
+#ifdef SUN4V
+#define	MP_TICK_QUALITY	1000
+#else
+#define	MP_TICK_QUALITY	-100
+#endif
+
+
+
+
+
+static struct timecounter tick_tc;
+
+char sparc64_model[32];
+
+cpu_block_copy_t *cpu_block_copy;
+cpu_block_zero_t *cpu_block_zero;
+
+static timecounter_get_t tick_get_timecount;
+void sparc64_init(caddr_t mdp, u_long o1, u_long o2, u_long o3,
+		  ofw_vec_t *vec);
+void sparc64_shutdown_final(void *dummy, int howto);
+
+static void cpu_startup(void *);
+SYSINIT(cpu, SI_SUB_CPU, SI_ORDER_FIRST, cpu_startup, NULL);
+
+CTASSERT((1 << INT_SHIFT) == sizeof(int));
+CTASSERT((1 << PTR_SHIFT) == sizeof(char *));
+
+CTASSERT(sizeof(struct reg) == 256);
+CTASSERT(sizeof(struct fpreg) == 272);
+CTASSERT(sizeof(struct __mcontext) == 512);
+
+CTASSERT((sizeof(struct pcb) & (64 - 1)) == 0);
+CTASSERT((offsetof(struct pcb, pcb_kfp) & (64 - 1)) == 0);
+CTASSERT((offsetof(struct pcb, pcb_ufp) & (64 - 1)) == 0);
+CTASSERT(sizeof(struct pcb) <= ((KSTACK_PAGES * PAGE_SIZE) / 8));
+
+CTASSERT(sizeof(struct pcpu) <= ((PCPU_PAGES * PAGE_SIZE) / 2));
+CTASSERT((sizeof(struct pcpu) & ((1<<6)-1)) == 0);
+
+static void
+cpu_startup(void *arg)
+{
+	vm_paddr_t physsz;
+	int i;
+
+	tick_tc.tc_get_timecount = tick_get_timecount;
+	tick_tc.tc_poll_pps = NULL;
+	tick_tc.tc_counter_mask = ~0u;
+	tick_tc.tc_frequency = tick_freq;
+	tick_tc.tc_name = "tick";
+	tick_tc.tc_quality = UP_TICK_QUALITY;
+#ifdef SMP
+	/*
+	 * We do not know if each CPU's tick counter is synchronized.
+	 */
+	if (cpu_mp_probe())
+		tick_tc.tc_quality = MP_TICK_QUALITY;
+#endif
+
+	tc_init(&tick_tc);
+
+	physsz = 0;
+	for (i = 0; i < sparc64_nmemreg; i++)
+		physsz += sparc64_memreg[i].mr_size;
+	printf("real memory  = %lu (%lu MB)\n", physsz,
+	    physsz / (1024 * 1024));
+	realmem = (long)physsz;
+
+	vm_ksubmap_init(&kmi);
+
+	bufinit();
+	vm_pager_bufferinit();
+
+	EVENTHANDLER_REGISTER(shutdown_final, sparc64_shutdown_final, NULL,
+	    SHUTDOWN_PRI_LAST);
+
+	printf("avail memory = %lu (%lu MB)\n", cnt.v_free_count * PAGE_SIZE,
+	    cnt.v_free_count / ((1024 * 1024) / PAGE_SIZE));
+
+	if (bootverbose)
+		printf("machine: %s\n", sparc64_model);
+
+#ifdef notyet
+	cpu_identify(rdpr(ver), tick_freq, PCPU_GET(cpuid));
+#endif 
+}
+
+void
+cpu_pcpu_init(struct pcpu *pcpu, int cpuid, size_t size)
+{
+	struct intr_request *ir;
+	int i;
+
+	pcpu->pc_irtail = &pcpu->pc_irhead;
+	for (i = 0; i < IR_FREE; i++) {
+		ir = &pcpu->pc_irpool[i];
+		ir->ir_next = pcpu->pc_irfree;
+		pcpu->pc_irfree = ir;
+	}
+}
+
+void
+spinlock_enter(void)
+{
+	struct thread *td;
+	register_t pil;
+
+	td = curthread;
+	if (td->td_md.md_spinlock_count == 0) {
+		pil = intr_disable();
+		td->td_md.md_saved_pil = pil;
+	}
+	td->td_md.md_spinlock_count++;
+	critical_enter();
+}
+
+void
+spinlock_exit(void)
+{
+	struct thread *td;
+
+	td = curthread;
+	critical_exit();
+	td->td_md.md_spinlock_count--;
+	if (td->td_md.md_spinlock_count == 0) {
+		intr_restore(td->td_md.md_saved_pil);
+	}
+
+}
+
+unsigned
+tick_get_timecount(struct timecounter *tc)
+{
+	return ((unsigned)rd(tick));
+}
+
+void
+sparc64_init(caddr_t mdp, u_long o1, u_long o2, u_long o3, ofw_vec_t *vec)
+{
+	phandle_t child;
+	phandle_t root;
+	struct pcpu *pc;
+	vm_offset_t end;
+	caddr_t kmdp;
+	u_int clock;
+	char *env;
+	char type[8];
+
+	end = 0;
+	kmdp = NULL;
+
+	/*
+	 * Initialize Open Firmware (needed for console).
+	 */
+	OF_init(vec);
+	/*
+	 * Parse metadata if present and fetch parameters.  Must be before the
+	 * console is inited so cninit gets the right value of boothowto.
+	 */
+	if (mdp != NULL) {
+		preload_metadata = mdp;
+		kmdp = preload_search_by_type("elf kernel");
+		if (kmdp != NULL) {
+			boothowto = MD_FETCH(kmdp, MODINFOMD_HOWTO, int);
+			kern_envp = MD_FETCH(kmdp, MODINFOMD_ENVP, char *);
+			end = MD_FETCH(kmdp, MODINFOMD_KERNEND, vm_offset_t);
+			kernel_tlb_slots = MD_FETCH(kmdp, MODINFOMD_DTLB_SLOTS,
+			    int);
+			kernel_tlbs = (void *)preload_search_info(kmdp,
+			    MODINFO_METADATA | MODINFOMD_DTLB);
+		}
+	}
+
+	init_param1();
+
+	root = OF_peer(0);
+	for (child = OF_child(root); child != 0; child = OF_peer(child)) {
+		OF_getprop(child, "device_type", type, sizeof(type));
+		if (strcmp(type, "cpu") == 0)
+			break;
+	}
+
+	OF_getprop(child, "clock-frequency", &clock, sizeof(clock));
+
+	/*
+	 * Initialize the console before printing anything.
+	 */
+	cninit();
+	tick_init(clock);
+
+	printf("cpu0: UltraSparc T1 Processor (%d.%02d MHz CPU)\n",
+	    (clock + 4999) / 1000000, ((clock + 4999) / 10000) % 100);
+
+	/*
+	 * Panic is there is no metadata.  Most likely the kernel was booted
+	 * directly, instead of through loader(8).
+	 */
+	if (mdp == NULL || kmdp == NULL) {
+		printf("sparc64_init: no loader metadata.\n"
+		       "This probably means you are not using loader(8).\n");
+		panic("sparc64_init");
+	}
+
+	/*
+	 * Sanity check the kernel end, which is important.
+	 */
+	if (end == 0) {
+		printf("sparc64_init: warning, kernel end not specified.\n"
+		       "Attempting to continue anyway.\n");
+		end = (vm_offset_t)_end;
+	}
+
+	cpu_block_copy = bcopy;
+	cpu_block_zero = bzero;
+
+#ifdef SMP
+	mp_tramp = mp_tramp_alloc();
+#endif
+
+	env = getenv("kernelname");
+	if (env != NULL) {
+		strlcpy(kernelname, env, sizeof(kernelname));
+		freeenv(env);
+	}
+
+	/*
+	 * Initialize proc0 stuff (p_contested needs to be done early).
+	 */
+
+	proc_linkup(&proc0, &thread0);
+	proc0.p_md.md_sigtramp = NULL;
+	proc0.p_md.md_utrap = NULL;
+	frame0.tf_tstate = TSTATE_IE | TSTATE_PEF | TSTATE_PRIV;
+	thread0.td_frame = &frame0;
+	if ((u_long)thread0.td_frame & 0x3f) {
+		panic("unaligned frame0");
+	}
+	/*
+	 * Prime our per-cpu data page for use.  Note, we are using it for our
+	 * stack, so don't pass the real size (PAGE_SIZE) to pcpu_init or
+	 * it'll zero it out from under us.
+	 */
+	pc = (struct pcpu *)(pcpu0 + (PCPU_PAGES * PAGE_SIZE)) - 1;
+	pcpu_init(pc, 0, sizeof(struct pcpu));
+	pc->pc_curthread = &thread0;
+#ifdef notyet
+	/* SUN4V_FIXME  what is pc_mid? */
+	pc->pc_mid = UPA_CR_GET_MID(ldxa(0, ASI_UPA_CONFIG_REG));
+#endif
+	pc->pc_addr = (vm_offset_t)pcpu0;
+
+	/*
+	 * Initialize global registers.
+	 * needed for curthread to work
+	 */
+	cpu_setregs(pc);
+
+	/*
+	 * Initialize virtual memory and calculate physmem.
+	 */
+	pmap_bootstrap(end);
+
+	thread0.td_kstack = kstack0;
+	thread0.td_md.md_saved_pil = 0;
+	thread0.td_pcb = (struct pcb *)
+		(thread0.td_kstack + KSTACK_PAGES * PAGE_SIZE) - 1;
+	thread0.td_pcb->pcb_kstack = (uint64_t)(((char *)thread0.td_pcb) - (CCFSZ + SPOFF));
+	thread0.td_pcb = (struct pcb *)TLB_PHYS_TO_DIRECT(vtophys((vm_offset_t)thread0.td_pcb));
+	pc->pc_curpcb = thread0.td_pcb;
+
+	if (((thread0.td_pcb->pcb_kstack + SPOFF) & 0x3f) != 0) {
+		printf("unaligned stack pcb_kstack & 0x3f == 0x%lx\n", 
+		       ((thread0.td_pcb->pcb_kstack + SPOFF) & 0x3f));
+	}
+
+	/*
+	 * Update PCPU_REG to point to direct address
+	 * to support easy phys <-> virt translation in trap handler
+	 */
+	pc = (struct pcpu *)TLB_PHYS_TO_DIRECT(vtophys(pc));
+	cpu_setregs(pc);
+	
+	/*
+	 * Initialize tunables.
+	 */
+	init_param2(physmem);
+
+	/*
+	 * setup trap table and fault status area
+	 */
+	trap_init();
+	
+	/*
+	 * Initialize the message buffer (after setting trap table).
+	 */
+
+	msgbufinit(msgbufp, MSGBUF_SIZE);
+
+	mutex_init();
+	
+	mdesc_init();
+
+	OF_getprop(root, "name", sparc64_model, sizeof(sparc64_model) - 1);
+
+	kdb_init();
+
+#ifdef KDB
+	if (boothowto & RB_KDB)
+		kdb_enter("Boot flags requested debugger");
+#endif
+}
+
+void
+set_openfirm_callback(ofw_vec_t *vec)
+{
+	ofw_tba = rdpr(tba);
+	ofw_vec = (u_long)vec;
+}
+
+void
+sendsig(sig_t catcher, ksiginfo_t *ksi, sigset_t *mask)
+{
+	struct trapframe *tf;
+	struct sigframe *sfp;
+	struct sigacts *psp;
+	struct sigframe sf;
+	struct thread *td;
+	struct frame *fp;
+	struct proc *p;
+	int oonstack;
+	u_long sp;
+	int sig;
+	int code;
+
+	oonstack = 0;
+	td = curthread;
+	p = td->td_proc;
+	PROC_LOCK_ASSERT(p, MA_OWNED);
+	sig = ksi->ksi_signo;
+	code = ksi->ksi_code;
+	psp = p->p_sigacts;
+	mtx_assert(&psp->ps_mtx, MA_OWNED);
+	tf = td->td_frame;
+	sp = tf->tf_sp + SPOFF;
+	oonstack = sigonstack(sp);
+
+	CTR4(KTR_SIG, "sendsig: td=%p (%s) catcher=%p sig=%d", td, p->p_comm,
+	    catcher, sig);
+
+	/* Make sure we have a signal trampoline to return to. */
+	if (p->p_md.md_sigtramp == NULL) {
+		/*
+		 * No signal tramoline... kill the process.
+		 */
+		CTR0(KTR_SIG, "sendsig: no sigtramp");
+		printf("sendsig: %s is too old, rebuild it\n", p->p_comm);
+		sigexit(td, sig);
+		/* NOTREACHED */
+	}
+
+	/* Save user context. */
+	bzero(&sf, sizeof(sf));
+	get_mcontext(td, &sf.sf_uc.uc_mcontext, 0);
+	sf.sf_uc.uc_sigmask = *mask;
+	sf.sf_uc.uc_stack = td->td_sigstk;
+	sf.sf_uc.uc_stack.ss_flags = (td->td_pflags & TDP_ALTSTACK)
+	    ? ((oonstack) ? SS_ONSTACK : 0) : SS_DISABLE;
+
+	/* Allocate and validate space for the signal handler context. */
+	if ((td->td_pflags & TDP_ALTSTACK) != 0 && !oonstack &&
+	    SIGISMEMBER(psp->ps_sigonstack, sig)) {
+		sfp = (struct sigframe *)(td->td_sigstk.ss_sp +
+		    td->td_sigstk.ss_size - sizeof(struct sigframe));
+	} else
+		sfp = (struct sigframe *)sp - 1;
+	mtx_unlock(&psp->ps_mtx);
+	PROC_UNLOCK(p);
+
+	fp = (struct frame *)sfp - 1;
+
+	/* Translate the signal if appropriate. */
+	if (p->p_sysent->sv_sigtbl && sig <= p->p_sysent->sv_sigsize)
+		sig = p->p_sysent->sv_sigtbl[_SIG_IDX(sig)];
+
+	/* Build the argument list for the signal handler. */
+	tf->tf_out[0] = sig;
+	tf->tf_out[2] = (register_t)&sfp->sf_uc;
+	tf->tf_out[4] = (register_t)catcher;
+
+	/* Fill siginfo structure. */
+	sf.sf_si = ksi->ksi_info;
+	sf.sf_si.si_addr = (void *)tf->tf_tpc;
+	if (SIGISMEMBER(psp->ps_siginfo, sig)) {
+		/* Signal handler installed with SA_SIGINFO. */
+		tf->tf_out[1] = (register_t)&sfp->sf_si;
+
+		/* Fill in POSIX parts. */
+		sf.sf_si = ksi->ksi_info;
+		sf.sf_si.si_signo = sig; /* maybe a translated signal */
+	} else {
+		/* Old FreeBSD-style arguments. */
+		tf->tf_out[1] = ksi->ksi_code;
+		tf->tf_out[3] = (register_t)ksi->ksi_addr;
+	}
+
+	/* Copy the sigframe out to the user's stack. */
+	if (rwindow_save(td) != 0 || copyout(&sf, sfp, sizeof(*sfp)) != 0 ||
+	    suword(&fp->fr_in[6], tf->tf_out[6]) != 0) {
+		/*
+		 * Something is wrong with the stack pointer.
+		 * ...Kill the process.
+		 */
+		CTR2(KTR_SIG, "sendsig: sigexit td=%p sfp=%p", td, sfp);
+		PROC_LOCK(p);
+		sigexit(td, SIGILL);
+		/* NOTREACHED */
+	}
+
+	tf->tf_tpc = (u_long)p->p_md.md_sigtramp;
+	tf->tf_tnpc = tf->tf_tpc + 4;
+	tf->tf_sp = (u_long)fp - SPOFF;
+
+	CTR3(KTR_SIG, "sendsig: return td=%p pc=%#lx sp=%#lx", td, tf->tf_tpc,
+	    tf->tf_sp);
+
+	PROC_LOCK(p);
+	mtx_lock(&psp->ps_mtx);
+}
+
+#ifndef	_SYS_SYSPROTO_H_
+struct sigreturn_args {
+	ucontext_t *ucp;
+};
+#endif
+
+/*
+ * MPSAFE
+ */
+int
+sigreturn(struct thread *td, struct sigreturn_args *uap)
+{
+	struct proc *p;
+	mcontext_t *mc;
+	ucontext_t uc;
+	int error;
+
+	p = td->td_proc;
+	if (rwindow_save(td)) {
+		PROC_LOCK(p);
+		sigexit(td, SIGILL);
+	}
+
+	CTR2(KTR_SIG, "sigreturn: td=%p ucp=%p", td, uap->sigcntxp);
+	if (copyin(uap->sigcntxp, &uc, sizeof(uc)) != 0) {
+		CTR1(KTR_SIG, "sigreturn: efault td=%p", td);
+		return (EFAULT);
+	}
+
+	mc = &uc.uc_mcontext;
+	error = set_mcontext(td, mc);
+	if (error != 0)
+		return (error);
+
+	PROC_LOCK(p);
+	td->td_sigmask = uc.uc_sigmask;
+	SIG_CANTMASK(td->td_sigmask);
+	signotify(td);
+	PROC_UNLOCK(p);
+
+	CTR4(KTR_SIG, "sigreturn: return td=%p pc=%#lx sp=%#lx tstate=%#lx",
+	    td, mc->mc_tpc, mc->mc_sp, mc->mc_tstate);
+	return (EJUSTRETURN);
+}
+
+#ifdef COMPAT_FREEBSD4
+int
+freebsd4_sigreturn(struct thread *td, struct freebsd4_sigreturn_args *uap)
+{
+
+	return sigreturn(td, (struct sigreturn_args *)uap);
+}
+#endif
+
+/*
+ * Construct a PCB from a trapframe. This is called from kdb_trap() where
+ * we want to start a backtrace from the function that caused us to enter
+ * the debugger. We have the context in the trapframe, but base the trace
+ * on the PCB. The PCB doesn't have to be perfect, as long as it contains
+ * enough for a backtrace.
+ */
+void
+makectx(struct trapframe *tf, struct pcb *pcb)
+{
+
+	pcb->pcb_pc = tf->tf_tpc;
+	pcb->pcb_sp = tf->tf_sp;
+}
+
+int
+get_mcontext(struct thread *td, mcontext_t *mc, int flags)
+{
+	struct trapframe *tf;
+	struct pcb *pcb;
+
+	tf = td->td_frame;
+	pcb = td->td_pcb;
+	bcopy(tf, mc, sizeof(*tf));
+	if (flags & GET_MC_CLEAR_RET) {
+		mc->mc_out[0] = 0;
+		mc->mc_out[1] = 0;
+	}
+	mc->mc_flags = _MC_VERSION;
+	critical_enter();
+	if ((tf->tf_fprs & FPRS_FEF) != 0) {
+		savefpctx(pcb->pcb_ufp);
+		pcb->pcb_flags |= PCB_FEF;
+		tf->tf_fprs &= ~FPRS_FEF;
+	}
+	if ((pcb->pcb_flags & PCB_FEF) != 0) {
+		bcopy(pcb->pcb_ufp, mc->mc_fp, sizeof(mc->mc_fp));
+		mc->mc_fprs |= FPRS_FEF;
+	}
+	critical_exit();
+	return (0);
+}
+
+int
+set_mcontext(struct thread *td, const mcontext_t *mc)
+{
+	struct trapframe *tf;
+	struct pcb *pcb;
+	uint64_t wstate;
+
+	if (!TSTATE_SECURE(mc->mc_tstate) ||
+	    (mc->mc_flags & ((1L << _MC_VERSION_BITS) - 1)) != _MC_VERSION)
+		return (EINVAL);
+	tf = td->td_frame;
+	pcb = td->td_pcb;
+	/* Make sure the windows are spilled first. */
+	flushw();
+	wstate = tf->tf_wstate;
+	bcopy(mc, tf, sizeof(*tf));
+	tf->tf_wstate = wstate;
+	if ((mc->mc_fprs & FPRS_FEF) != 0) {
+		tf->tf_fprs = 0;
+		bcopy(mc->mc_fp, pcb->pcb_ufp, sizeof(pcb->pcb_ufp));
+		pcb->pcb_flags |= PCB_FEF;
+	}
+	return (0);
+}
+
+/*
+ * Exit the kernel and execute a firmware call that will not return, as
+ * specified by the arguments.
+ */
+void
+cpu_shutdown(void *args)
+{
+
+#ifdef SMP
+	cpu_mp_shutdown();
+#endif
+	OF_exit();
+}
+
+/* Get current clock frequency for the given cpu id. */
+int
+cpu_est_clockrate(int cpu_id, uint64_t *rate)
+{
+
+	return (ENXIO);
+}
+
+/*
+ * Duplicate OF_exit() with a different firmware call function that restores
+ * the trap table, otherwise a RED state exception is triggered in at least
+ * some firmware versions.
+ */
+void
+cpu_halt(void)
+{
+	static struct {
+		cell_t name;
+		cell_t nargs;
+		cell_t nreturns;
+	} args = {
+		(cell_t)"exit",
+		0,
+		0
+	};
+
+	cpu_shutdown(&args);
+}
+
+void
+sparc64_shutdown_final(void *dummy, int howto)
+{
+	static struct {
+		cell_t name;
+		cell_t nargs;
+		cell_t nreturns;
+	} args = {
+		(cell_t)"SUNW,power-off",
+		0,
+		0
+	};
+
+	/* Turn the power off? */
+	if ((howto & RB_POWEROFF) != 0)
+		cpu_shutdown(&args);
+	/* In case of halt, return to the firmware */
+	if ((howto & RB_HALT) != 0)
+		cpu_halt();
+}
+
+void
+cpu_idle(void)
+{
+
+	if (rdpr(pil) != 0) 
+		panic("pil in cpu_idle not 0 - %ld", rdpr(pil));
+	if (rdpr(pstate) != 0x16)
+		panic("interrupts disabled in cpu_idle 0x%lx", rdpr(pstate));
+		/* XXX heinous hack begin*/
+	
+	cpu_yield();
+}
+
+int
+ptrace_set_pc(struct thread *td, u_long addr)
+{
+
+	td->td_frame->tf_tpc = addr;
+	td->td_frame->tf_tnpc = addr + 4;
+	return (0);
+}
+
+int
+ptrace_single_step(struct thread *td)
+{
+	/* TODO; */
+	return (0);
+}
+
+int
+ptrace_clear_single_step(struct thread *td)
+{
+	/* TODO; */
+	return (0);
+}
+
+void
+exec_setregs(struct thread *td, u_long entry, u_long stack, u_long ps_strings)
+{
+	struct trapframe *tf;
+	struct pcb *pcb;
+	struct proc *p;
+	uint64_t kstack;
+	u_long sp;
+
+	/* XXX no cpu_exec */
+
+	p = td->td_proc;
+	p->p_md.md_sigtramp = NULL;
+	if (p->p_md.md_utrap != NULL) {
+		utrap_free(p->p_md.md_utrap);
+		p->p_md.md_utrap = NULL;
+	}
+	pcb = td->td_pcb;
+	kstack = pcb->pcb_kstack;
+	tf = td->td_frame;
+	sp = rounddown(stack, 16);
+	bzero(pcb, sizeof(*pcb));
+	bzero(tf, sizeof(*tf));
+	pcb->pcb_kstack = kstack;
+
+	tf->tf_out[0] = stack;
+	tf->tf_out[3] = p->p_sysent->sv_psstrings;
+	tf->tf_out[6] = sp - SPOFF - sizeof(struct frame);
+
+	tf->tf_tnpc = entry + 4;
+	tf->tf_tpc = entry;
+	tf->tf_tstate = TSTATE_IE | TSTATE_PEF | TSTATE_MM_TSO;
+
+	td->td_retval[0] = tf->tf_out[0];
+	td->td_retval[1] = tf->tf_out[1];
+}
+
+int
+fill_regs(struct thread *td, struct reg *regs)
+{
+
+	bcopy(td->td_frame, regs, sizeof(*regs));
+	return (0);
+}
+
+int
+set_regs(struct thread *td, struct reg *regs)
+{
+	struct trapframe *tf;
+
+	if (!TSTATE_SECURE(regs->r_tstate))
+		return (EINVAL);
+	tf = td->td_frame;
+	regs->r_wstate = tf->tf_wstate;
+	bcopy(regs, tf, sizeof(*regs));
+	return (0);
+}
+
+int
+fill_dbregs(struct thread *td, struct dbreg *dbregs)
+{
+
+	return (ENOSYS);
+}
+
+int
+set_dbregs(struct thread *td, struct dbreg *dbregs)
+{
+
+	return (ENOSYS);
+}
+
+int
+fill_fpregs(struct thread *td, struct fpreg *fpregs)
+{
+	struct trapframe *tf;
+	struct pcb *pcb;
+
+	pcb = td->td_pcb;
+	tf = td->td_frame;
+	tf->tf_fprs = ~FPRS_FEF;
+	bcopy(pcb->pcb_ufp, fpregs->fr_regs, sizeof(fpregs->fr_regs));
+	fpregs->fr_fsr = tf->tf_fsr;
+	fpregs->fr_gsr = tf->tf_gsr;
+	return (0);
+}
+
+int
+set_fpregs(struct thread *td, struct fpreg *fpregs)
+{
+	struct trapframe *tf;
+	struct pcb *pcb;
+
+	pcb = td->td_pcb;
+	tf = td->td_frame;
+	tf->tf_fprs &= ~FPRS_FEF;
+	bcopy(fpregs->fr_regs, pcb->pcb_ufp, sizeof(pcb->pcb_ufp));
+	tf->tf_fsr = fpregs->fr_fsr;
+	tf->tf_gsr = fpregs->fr_gsr;
+	return (0);
+}
+
+struct md_utrap *
+utrap_alloc(void)
+{
+	struct md_utrap *ut;
+
+	ut = malloc(sizeof(struct md_utrap), M_SUBPROC, M_WAITOK | M_ZERO);
+	ut->ut_refcnt = 1;
+	return (ut);
+}
+
+void
+utrap_free(struct md_utrap *ut)
+{
+	int refcnt;
+
+	if (ut == NULL)
+		return;
+	mtx_pool_lock(mtxpool_sleep, ut);
+	ut->ut_refcnt--;
+	refcnt = ut->ut_refcnt;
+	mtx_pool_unlock(mtxpool_sleep, ut);
+	if (refcnt == 0)
+		free(ut, M_SUBPROC);
+}
+
+struct md_utrap *
+utrap_hold(struct md_utrap *ut)
+{
+
+	if (ut == NULL)
+		return (NULL);
+	mtx_pool_lock(mtxpool_sleep, ut);
+	ut->ut_refcnt++;
+	mtx_pool_unlock(mtxpool_sleep, ut);
+	return (ut);
+}
+
+void
+cpu_yield(void)
+{
+	if (rdpr(pil) < PIL_TICK)
+		hv_cpu_yield();
+}
diff --git a/sys/sun4v/sun4v/mem.c b/sys/sun4v/sun4v/mem.c
new file mode 100644
index 000000000000..6a7b9da959c3
--- /dev/null
+++ b/sys/sun4v/sun4v/mem.c
@@ -0,0 +1,183 @@
+/*-
+ * Copyright (c) 1988 University of Utah.
+ * Copyright (c) 1982, 1986, 1990 The Regents of the University of California.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to Berkeley by
+ * the Systems Programming Group of the University of Utah Computer
+ * Science Department, and code derived from software contributed to
+ * Berkeley by William Jolitz.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 4. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	from: Utah $Hdr: mem.c 1.13 89/10/08$
+ *	from: @(#)mem.c	7.2 (Berkeley) 5/9/91
+ *	from: FreeBSD: src/sys/i386/i386/mem.c,v 1.94 2001/09/26
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+/*
+ * Memory special file
+ *
+ * NOTE: other architectures support mmap()'ing the mem and kmem devices; this
+ * might cause illegal aliases to be created for the locked kernel page(s), so
+ * it is not implemented.
+ */
+
+#include <sys/param.h>
+#include <sys/conf.h>
+#include <sys/fcntl.h>
+#include <sys/kernel.h>
+#include <sys/lock.h>
+#include <sys/malloc.h>
+#include <sys/memrange.h>
+#include <sys/module.h>
+#include <sys/mutex.h>
+#include <sys/proc.h>
+#include <sys/signalvar.h>
+#include <sys/systm.h>
+#include <sys/uio.h>
+
+#include <vm/vm.h>
+#include <vm/vm_param.h>
+#include <vm/vm_page.h>
+#include <vm/vm_kern.h>
+#include <vm/pmap.h>
+#include <vm/vm_extern.h>
+
+#include <machine/cache.h>
+#include <machine/md_var.h>
+#include <machine/pmap.h>
+#include <machine/tlb.h>
+
+#include <machine/memdev.h>
+
+struct mem_range_softc mem_range_softc;
+
+/* ARGSUSED */
+int
+memrw(struct cdev *dev, struct uio *uio, int flags)
+{
+	struct iovec *iov;
+	vm_offset_t eva;
+	vm_offset_t off;
+	vm_offset_t ova;
+	vm_offset_t va;
+	vm_prot_t prot;
+	vm_paddr_t pa;
+	vm_size_t cnt;
+	vm_page_t m;
+	int error;
+	int i;
+
+	cnt = 0;
+	error = 0;
+	ova = 0;
+
+	GIANT_REQUIRED;
+
+	while (uio->uio_resid > 0 && error == 0) {
+		iov = uio->uio_iov;
+		if (iov->iov_len == 0) {
+			uio->uio_iov++;
+			uio->uio_iovcnt--;
+			if (uio->uio_iovcnt < 0)
+				panic("memrw");
+			continue;
+		}
+		if (minor(dev) == CDEV_MINOR_MEM) {
+			pa = uio->uio_offset & ~PAGE_MASK;
+			if (!is_physical_memory(pa)) {
+				error = EFAULT;
+				break;
+			}
+
+			off = uio->uio_offset & PAGE_MASK;
+			cnt = PAGE_SIZE - ((vm_offset_t)iov->iov_base &
+			    PAGE_MASK);
+			cnt = ulmin(cnt, PAGE_SIZE - off);
+			cnt = ulmin(cnt, iov->iov_len);
+
+			m = NULL;
+			for (i = 0; phys_avail[i] != 0; i += 2) {
+				if (pa >= phys_avail[i] &&
+				    pa < phys_avail[i + 1]) {
+					m = PHYS_TO_VM_PAGE(pa);
+					break;
+				}
+			}
+
+			if (m != NULL) {
+				if (ova == 0) {
+					ova = kmem_alloc_wait(kernel_map,
+					    PAGE_SIZE);
+				}
+				va = ova;
+				pmap_qenter(va, &m, 1);
+				error = uiomove((void *)(va + off), cnt,
+				    uio);
+				pmap_qremove(va, 1);
+			} else {
+				va = TLB_PHYS_TO_DIRECT(pa);
+				error = uiomove((void *)(va + off), cnt,
+				    uio);
+			}
+			break;
+		}
+		else if (minor(dev) == CDEV_MINOR_KMEM) {
+			va = trunc_page(uio->uio_offset);
+			eva = round_page(uio->uio_offset + iov->iov_len);
+
+			/*
+			 * Make sure that all of the pages are currently
+			 * resident so we don't create any zero fill pages.
+			 */
+			for (; va < eva; va += PAGE_SIZE)
+				if (pmap_kextract(va) == 0)
+					return (EFAULT);
+
+			prot = (uio->uio_rw == UIO_READ) ? VM_PROT_READ :
+			    VM_PROT_WRITE;
+			va = uio->uio_offset;
+			if (va < VM_MIN_DIRECT_ADDRESS &&
+			    kernacc((void *)va, iov->iov_len, prot) == FALSE)
+				return (EFAULT);
+
+			error = uiomove((void *)va, iov->iov_len, uio);
+			break;
+		}
+		/* else panic! */
+	}
+	if (ova != 0)
+		kmem_free_wakeup(kernel_map, ova, PAGE_SIZE);
+	return (error);
+}
+
+void
+dev_mem_md_init(void)
+{
+}
diff --git a/sys/sun4v/sun4v/mp_exception.S b/sys/sun4v/sun4v/mp_exception.S
new file mode 100644
index 000000000000..699d766fa7c3
--- /dev/null
+++ b/sys/sun4v/sun4v/mp_exception.S
@@ -0,0 +1,280 @@
+/*-
+ * Copyright (c) 2002 Jake Burkholder.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <machine/asm.h>
+__FBSDID("$FreeBSD$");
+
+#include <machine/asi.h>
+#include <machine/ktr.h>
+#include <machine/asmacros.h>
+#include <machine/pstate.h>
+
+#include "assym.s"
+
+	.register	%g2, #ignore
+	.register	%g3, #ignore
+
+#define	IPI_DONE(r1, r2, r3, r4) \
+	lduw	[PCPU(CPUMASK)], r4 ;  \
+	ATOMIC_CLEAR_INT(r1, r2, r3, r4)
+
+/*
+ * Invalidate a physical page in the data cache.  For UltraSPARC I and II.
+ */
+#if 0
+ENTRY(tl_ipi_spitfire_dcache_page_inval)
+#if KTR_COMPILE & KTR_SMP
+	CATR(KTR_SMP, "ipi_dcache_page_inval: pa=%#lx"
+	    , %g1, %g2, %g3, 7, 8, 9)
+	ldx	[%g5 + ICA_PA], %g2
+	stx	%g2, [%g1 + KTR_PARM1]
+9:
+#endif
+
+	ldx	[%g5 + ICA_PA], %g6
+	srlx	%g6, PAGE_SHIFT - DC_TAG_SHIFT, %g6
+
+	SET(cache, %g3, %g2)
+	lduw	[%g2 + DC_SIZE], %g3
+	lduw	[%g2 + DC_LINESIZE], %g4
+	sub	%g3, %g4, %g2
+
+1:	ldxa	[%g2] ASI_DCACHE_TAG, %g1
+	srlx	%g1, DC_VALID_SHIFT, %g3
+	andcc	%g3, DC_VALID_MASK, %g0
+	bz,pt	%xcc, 2f
+	 set	DC_TAG_MASK, %g3
+	sllx	%g3, DC_TAG_SHIFT, %g3
+	and	%g1, %g3, %g1
+	cmp	%g1, %g6
+	bne,a,pt %xcc, 2f
+	 nop
+	stxa	%g1, [%g2] ASI_DCACHE_TAG
+	membar	#Sync
+
+2:	brgz,pt	%g2, 1b
+	 sub	%g2, %g4, %g2
+
+	IPI_DONE(%g5, %g1, %g2, %g3)
+	retry
+END(tl_ipi_spitfire_dcache_page_inval)
+
+/*
+ * Invalidate a physical page in the instruction cache.  For UltraSPARC I and
+ * II.
+ */
+ENTRY(tl_ipi_spitfire_icache_page_inval)
+#if KTR_COMPILE & KTR_SMP
+	CATR(KTR_SMP, "ipi_icache_page_inval: pa=%#lx"
+	    , %g1, %g2, %g3, 7, 8, 9)
+	ldx	[%g5 + ICA_PA], %g2
+	stx	%g2, [%g1 + KTR_PARM1]
+9:
+#endif
+
+	ldx	[%g5 + ICA_PA], %g6
+	srlx	%g6, PAGE_SHIFT - IC_TAG_SHIFT, %g6
+
+	SET(cache, %g3, %g2)
+	lduw	[%g2 + IC_SIZE], %g3
+	lduw	[%g2 + IC_LINESIZE], %g4
+	sub	%g3, %g4, %g2
+
+1:	ldda	[%g2] ASI_ICACHE_TAG, %g0 /*, %g1 */
+	srlx	%g1, IC_VALID_SHIFT, %g3
+	andcc	%g3, IC_VALID_MASK, %g0
+	bz,pt	%xcc, 2f
+	 set	IC_TAG_MASK, %g3
+	sllx	%g3, IC_TAG_SHIFT, %g3
+	and	%g1, %g3, %g1
+	cmp	%g1, %g6
+	bne,a,pt %xcc, 2f
+	 nop
+	stxa	%g1, [%g2] ASI_ICACHE_TAG
+	membar	#Sync
+
+2:	brgz,pt	%g2, 1b
+	 sub	%g2, %g4, %g2
+
+	IPI_DONE(%g5, %g1, %g2, %g3)
+	retry
+END(tl_ipi_spitfire_icache_page_inval)
+
+/*
+ * Invalidate a physical page in the data cache.  For UltraSPARC III.
+ */
+ENTRY(tl_ipi_cheetah_dcache_page_inval)
+#if KTR_COMPILE & KTR_SMP
+	CATR(KTR_SMP, "ipi_dcache_page_inval: pa=%#lx"
+	    , %g1, %g2, %g3, 7, 8, 9)
+	ldx	[%g5 + ICA_PA], %g2
+	stx	%g2, [%g1 + KTR_PARM1]
+9:
+#endif
+
+	ldx	[%g5 + ICA_PA], %g1
+
+	set	PAGE_SIZE, %g2
+	add	%g1, %g2, %g3
+
+	SET(cache, %g4, %g2)
+	lduw	[%g2 + DC_LINESIZE], %g2
+
+1:	stxa	%g0, [%g1] ASI_DCACHE_INVALIDATE
+	membar	#Sync
+
+	add	%g1, %g2, %g1
+	cmp	%g1, %g3
+	blt,a,pt %xcc, 1b
+	 nop
+
+	IPI_DONE(%g5, %g1, %g2, %g3)
+	retry
+END(tl_ipi_cheetah_dcache_page_inval)
+#endif
+	
+/*
+ * Trigger a softint at the desired level.
+ */
+ENTRY(tl_ipi_level)
+#if KTR_COMPILE & KTR_SMP
+	CATR(KTR_SMP, "tl_ipi_level: cpuid=%d mid=%d d1=%#lx d2=%#lx"
+	    , %g1, %g2, %g3, 7, 8, 9)
+	lduw	[PCPU(CPUID)], %g2
+	stx	%g2, [%g1 + KTR_PARM1]
+	lduw	[PCPU(MID)], %g2
+	stx	%g2, [%g1 + KTR_PARM2]
+	stx	%g4, [%g1 + KTR_PARM3]
+	stx	%g5, [%g1 + KTR_PARM4]
+9:
+#endif
+
+	mov	1, %g2
+	sllx	%g2, %g1, %g2
+	wr	%g2, 0, %set_softint
+	retry
+END(tl_ipi_level)
+
+/*
+ * Demap a page from the dtlb and/or itlb.
+ */
+#if 0
+ENTRY(tl_ipi_tlb_page_demap)
+#if KTR_COMPILE & KTR_SMP
+	CATR(KTR_SMP, "ipi_tlb_page_demap: pm=%p va=%#lx"
+	    , %g1, %g2, %g3, 7, 8, 9)
+	ldx	[%g5 + ITA_PMAP], %g2
+	stx	%g2, [%g1 + KTR_PARM1]
+	ldx	[%g5 + ITA_VA], %g2
+	stx	%g2, [%g1 + KTR_PARM2]
+9:
+#endif
+
+	ldx	[%g5 + ITA_PMAP], %g1
+
+	SET(kernel_pmap_store, %g3, %g2)
+	mov	TLB_DEMAP_NUCLEUS | TLB_DEMAP_PAGE, %g3
+
+	cmp	%g1, %g2
+	movne	%xcc, TLB_DEMAP_PRIMARY | TLB_DEMAP_PAGE, %g3
+
+	ldx	[%g5 + ITA_VA], %g2
+	or	%g2, %g3, %g2
+
+	stxa	%g0, [%g2] ASI_DMMU_DEMAP
+	stxa	%g0, [%g2] ASI_IMMU_DEMAP
+	membar	#Sync
+
+	IPI_DONE(%g5, %g1, %g2, %g3)
+	retry
+END(tl_ipi_tlb_page_demap)
+
+/*
+ * Demap a range of pages from the dtlb and itlb.
+ */
+ENTRY(tl_ipi_tlb_range_demap)
+#if KTR_COMPILE & KTR_SMP
+	CATR(KTR_SMP, "ipi_tlb_range_demap: pm=%p start=%#lx end=%#lx"
+	    , %g1, %g2, %g3, 7, 8, 9)
+	ldx	[%g5 + ITA_PMAP], %g2
+	stx	%g2, [%g1 + KTR_PARM1]
+	ldx	[%g5 + ITA_START], %g2
+	stx	%g2, [%g1 + KTR_PARM2]
+	ldx	[%g5 + ITA_END], %g2
+	stx	%g2, [%g1 + KTR_PARM3]
+9:
+#endif
+
+	ldx	[%g5 + ITA_PMAP], %g1
+
+	SET(kernel_pmap_store, %g3, %g2)
+	mov	TLB_DEMAP_NUCLEUS | TLB_DEMAP_PAGE, %g3
+
+	cmp	%g1, %g2
+	movne	%xcc, TLB_DEMAP_PRIMARY | TLB_DEMAP_PAGE, %g3
+
+	ldx	[%g5 + ITA_START], %g1
+	ldx	[%g5 + ITA_END], %g2
+
+	set	PAGE_SIZE, %g6
+
+1:	or	%g1, %g3, %g4
+	stxa	%g0, [%g4] ASI_DMMU_DEMAP
+	stxa	%g0, [%g4] ASI_IMMU_DEMAP
+	membar	#Sync
+
+	add	%g1, %g6, %g1
+	cmp	%g1, %g2
+	blt,a,pt %xcc, 1b
+	 nop
+
+	IPI_DONE(%g5, %g1, %g2, %g3)
+	retry
+END(tl_ipi_tlb_range_demap)
+
+/*
+ * Demap the primary context from the dtlb and itlb.
+ */
+ENTRY(tl_ipi_tlb_context_demap)
+#if KTR_COMPILE & KTR_SMP
+	CATR(KTR_SMP, "ipi_tlb_page_demap: pm=%p va=%#lx"
+	    , %g1, %g2, %g3, 7, 8, 9)
+	ldx	[%g5 + ITA_PMAP], %g2
+	stx	%g2, [%g1 + KTR_PARM1]
+	ldx	[%g5 + ITA_VA], %g2
+	stx	%g2, [%g1 + KTR_PARM2]
+9:
+#endif
+
+	mov	TLB_DEMAP_PRIMARY | TLB_DEMAP_CONTEXT, %g1
+	stxa	%g0, [%g1] ASI_DMMU_DEMAP
+	stxa	%g0, [%g1] ASI_IMMU_DEMAP
+	membar	#Sync
+
+	IPI_DONE(%g5, %g1, %g2, %g3)
+	retry
+END(tl_ipi_tlb_context_demap)
+#endif
diff --git a/sys/sun4v/sun4v/mp_locore.S b/sys/sun4v/sun4v/mp_locore.S
new file mode 100644
index 000000000000..7316cc0d0c0b
--- /dev/null
+++ b/sys/sun4v/sun4v/mp_locore.S
@@ -0,0 +1,171 @@
+/*-
+ * Copyright (c) 2002 Jake Burkholder.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <machine/asm.h>
+__FBSDID("$FreeBSD$");
+
+#include <machine/asi.h>
+#include <machine/asmacros.h>
+#include <machine/ktr.h>
+#include <machine/pstate.h>
+#include <machine/hypervisorvar.h>
+
+#include "assym.s"
+
+	.register	%g2, #ignore
+	.register	%g3, #ignore
+
+	.text
+	_ALIGN_TEXT
+	
+1:	rd	%pc, %l0			! startpc
+	mov	%o0, %g1			! save arg to to mp_tramp_func
+	ldx	[%l0 + (6f-1b)], %l1		! read mp_tramp_tte_slots
+	add	%l0, (7f-1b), %l2		! %l2 points to start of slot area
+	clr	%l3
+2:	cmp	%l3, %l1			! number of slots entered == total?
+	be	%xcc, 3f
+	  nop
+	ldx	[%l2], %o0			! VA
+	mov	0, %o1				! ctx0
+	ldx	[%l2 + 8], %o2			! TTE
+	mov	MAP_ITLB|MAP_DTLB, %o3		
+	mov	MAP_PERM_ADDR, %o5
+	ta	FAST_TRAP
+#ifdef DEBUG
+	brz	%o0, 9f
+	  nop
+	ta	0x77
+	ta	0x71	
+9:
+#endif		
+	add	%l2, 16, %l2			! point %l2 at next slot
+	inc	%l3
+	ba	%xcc, 2b
+	  nop
+3:		
+	ldx	[%l0 + (4f-1b)], %o1		! read mp_tramp_tsb_ra
+	mov	2, %o0
+	mov	MMU_TSB_CTX0, %o5		! set ctx0 TSBs
+	ta	FAST_TRAP
+#ifdef DEBUG
+	brz	%o0, 9f
+	  nop
+	ta	0x77
+	ta	0x71	
+9:
+#endif		
+	ldx	[%l0 + (5f-1b)], %l1		! fetch mp_tramp_func
+	jmpl	%l1, %g0			! and off we go
+	  mov	%g1, %o0
+	_ALIGN_DATA
+4:	.xword	0x0
+5:	.xword	0x0
+6:	.xword	0x0
+7:	
+
+DATA(mp_tramp_code)
+	.xword	1b
+DATA(mp_tramp_code_len)
+	.xword	7b-1b
+DATA(mp_tramp_tsb_desc_ra)
+	.xword	4b-1b
+DATA(mp_tramp_func)
+	.xword	5b-1b
+DATA(mp_tramp_tte_slots)
+	.xword	6b-1b
+
+
+#define PUTCHAR(x) \
+	mov	x, %o0		; \
+	mov	CONS_WRITE, %o5	; \
+	ta	FAST_TRAP
+/*
+ * void mp_startup(void)
+ */
+ENTRY(mp_startup)
+	wrpr	%g0, PSTATE_NORMAL, %pstate
+	wrpr	%g0, 0, %cleanwin
+	wrpr	%g0, 0, %pil
+	wr	%g0, 0, %fprs
+
+	SET(cpu_start_args, %l1, %l0)
+#if KTR_COMPILE & KTR_SMP
+	CATR(KTR_SMP, "mp_start: cpu %d entered kernel"
+	    , %g1, %g2, %g3, 7, 8, 9)
+	stx	%o0, [%g1 + KTR_PARM1]
+9:
+#endif
+
+	/*
+	 * Inform the boot processor we have inited.
+	 */
+	mov	CPU_INIT, %l1
+	membar	#LoadStore
+	stw	%l1, [%l0 + CSA_STATE]
+
+#if KTR_COMPILE & KTR_SMP
+	CATR(KTR_SMP, "_mp_start: cpu %d got start signal"
+	    , %g1, %g2, %g3, 7, 8, 9)
+	stx	%o0, [%g1 + KTR_PARM1]
+9:
+#endif
+2:	
+	 ld	[%l0 + CSA_CPUID], %l1
+	cmp	%l1, %o0
+	bne	%xcc, 2b
+	  nop
+	
+	/*
+	 * Get onto our per-cpu panic stack, which precedes the struct pcpu
+	 * in the per-cpu page.
+	 */
+	ldx	[%l0 + CSA_PCPU], %l1
+	set	PCPU_PAGES * PAGE_SIZE - PC_SIZEOF, %l2
+	add	%l1, %l2, %l1
+	sub	%l1, SPOFF + CCFSZ, %sp
+	wrpr	%g0, PIL_TICK, %pil
+#if KTR_COMPILE & KTR_SMP
+	CATR(KTR_SMP,
+	    "_mp_start: bootstrap cpuid=%d mid=%d pcpu=%#lx data=%#lx sp=%#lx"
+	    , %g1, %g2, %g3, 7, 8, 9)
+	lduw	[%l1 + PC_CPUID], %g2
+	stx	%g2, [%g1 + KTR_PARM1]
+	lduw	[%l1 + PC_MID], %g2
+	stx	%g2, [%g1 + KTR_PARM2]
+	stx	%l1, [%g1 + KTR_PARM3]
+	stx	%sp, [%g1 + KTR_PARM5]
+9:
+#endif
+
+	/*
+	 * And away we go.  This doesn't return.
+	 */
+	call	cpu_mp_bootstrap
+	 mov	%l1, %o0
+	sir
+	! NOTREACHED
+END(mp_startup)
diff --git a/sys/sun4v/sun4v/mp_machdep.c b/sys/sun4v/sun4v/mp_machdep.c
new file mode 100644
index 000000000000..8ca6b7fd5139
--- /dev/null
+++ b/sys/sun4v/sun4v/mp_machdep.c
@@ -0,0 +1,545 @@
+/*-
+ * Copyright (c) 1997 Berkeley Software Design, Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Berkeley Software Design Inc's name may not be used to endorse or
+ *    promote products derived from this software without specific prior
+ *    written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * from BSDI: locore.s,v 1.36.2.15 1999/08/23 22:34:41 cp Exp
+ */
+/*-
+ * Copyright (c) 2002 Jake Burkholder.
+ * Copyright (c) 2006 Kip Macy <kmacy@FreeBSD.org>.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include "opt_trap_trace.h"
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/lock.h>
+#include <sys/kdb.h>
+#include <sys/kernel.h>
+#include <sys/ktr.h>
+#include <sys/mutex.h>
+#include <sys/pcpu.h>
+#include <sys/proc.h>
+#include <sys/smp.h>
+
+#include <vm/vm.h>
+#include <vm/vm_param.h>
+#include <vm/pmap.h>
+#include <vm/vm_kern.h>
+#include <vm/vm_extern.h>
+#include <vm/vm_map.h>
+
+#include <dev/ofw/openfirm.h>
+
+#include <machine/asi.h>
+#include <machine/atomic.h>
+#include <machine/bus.h>
+#include <machine/md_var.h>
+#include <machine/metadata.h>
+#include <machine/ofw_machdep.h>
+#include <machine/pcb.h>
+#include <machine/smp.h>
+#include <machine/tick.h>
+#include <machine/pstate.h>
+#include <machine/tlb.h>
+#include <machine/tte.h>
+#include <machine/tte_hash.h>
+#include <machine/tsb.h>
+#include <machine/trap.h>
+#include <machine/hypervisor_api.h>
+#include <machine/asm.h>
+
+/*
+ * Argument area used to pass data to non-boot processors as they start up.
+ * This must be statically initialized with a known invalid cpuid,
+ * 
+ */
+struct	cpu_start_args cpu_start_args = { 0, -1, 0, -1 };
+struct	ipi_cache_args ipi_cache_args;
+struct	ipi_tlb_args ipi_tlb_args;
+struct	pcb stoppcbs[MAXCPU];
+
+struct	mtx ipi_mtx;
+
+vm_offset_t mp_tramp;
+
+u_int	mp_boot_mid;
+
+static volatile u_int	shutdown_cpus;
+
+void cpu_mp_unleash(void *);
+SYSINIT(cpu_mp_unleash, SI_SUB_SMP, SI_ORDER_FIRST, cpu_mp_unleash, NULL);
+
+#ifdef TRAP_TRACING
+#ifndef TRAP_TRACE_ENTRIES
+#define TRAP_TRACE_ENTRIES	64
+#endif
+extern trap_trace_entry_t trap_trace_entry[MAXCPU][TRAP_TRACE_ENTRIES];
+
+static void
+mp_trap_trace_init(void)
+{
+	uint64_t ret, ret1;
+
+	printf("curcpu %d trap_trace_entry %p TRAP_TRACE_ENTRIES %d\n", curcpu, &trap_trace_entry[curcpu][0], TRAP_TRACE_ENTRIES);
+
+	/* Configure the trap trace buffer for the current CPU. */
+	if ((ret = hv_ttrace_buf_conf((uint64_t) vtophys(&trap_trace_entry[curcpu][0]),
+	    (uint64_t) TRAP_TRACE_ENTRIES, &ret1)) != 0)
+		printf("%s: hv_ttrace_buf_conf error %lu\n", __FUNCTION__, ret);
+
+	/* Enable trap tracing for the current CPU. */
+	else if ((ret = hv_ttrace_enable((uint64_t) -1, &ret1)) != 0)
+		printf("%s: hv_ttrace_enable error %lu\n", __FUNCTION__, ret);
+}
+
+void trap_trace_report(int);
+
+static int	trace_trap_lock;
+
+void
+trap_trace_report(int cpuid)
+{
+	int i, j;
+
+	while (!atomic_cmpset_acq_int(&trace_trap_lock, 0, 1))
+		DELAY(10000);
+
+	for (i = 0; i < MAXCPU; i++) {
+		if (cpuid != -1 && cpuid != i)
+			continue;
+
+		for (j = 0; j < TRAP_TRACE_ENTRIES; j++) {
+			trap_trace_entry_t *p = &trap_trace_entry[i][j];
+
+			printf("0x%08jx [%02d][%04d] tpc 0x%jx type 0x%x hpstat 0x%x tl %u gl %u tt 0x%hx tag 0x%hx tstate 0x%jx f1 0x%jx f2 0x%jx f3 0x%jx f4 0x%jx\n",
+			    p->tte_tick, i, j, p->tte_tpc,p->tte_type,p->tte_hpstat,
+			    p->tte_tl,p->tte_gl,p->tte_tt,p->tte_tag,p->tte_tstate,
+			    p->tte_f1,p->tte_f2,p->tte_f3,p->tte_f4);
+		}
+	}
+
+	atomic_store_rel_int(&trace_trap_lock, 0);
+}
+#endif
+
+vm_offset_t
+mp_tramp_alloc(void)
+{
+	char *v;
+	int i;
+
+	v = OF_claim(NULL, PAGE_SIZE, PAGE_SIZE);
+	if (v == NULL)
+		panic("mp_tramp_alloc");
+	bcopy(mp_tramp_code, v, mp_tramp_code_len);
+
+	*(u_long *)(v + mp_tramp_func) = (u_long)mp_startup;
+
+	for (i = 0; i < PAGE_SIZE; i += sizeof(long)*4 /* XXX L1 cacheline size */)
+		flush(v + i);
+	return (vm_offset_t)v;
+}
+
+void
+mp_set_tsb_desc_ra(vm_paddr_t tsb_desc_ra)
+{
+	*(u_long *)(mp_tramp + mp_tramp_tsb_desc_ra) = tsb_desc_ra;
+}
+
+void
+mp_add_nucleus_mapping(vm_offset_t va, tte_t tte_data)
+{
+	static int slot;
+	uint64_t *entry;
+
+	entry = (uint64_t *)(mp_tramp + mp_tramp_code_len + slot*sizeof(*entry)*2);
+	*(entry) = va;
+	*(entry + 1) = tte_data;
+	*(uint64_t *)(mp_tramp + mp_tramp_tte_slots) = slot + 1;
+	slot++;
+}
+
+/*
+ * Probe for other cpus.
+ */
+void
+cpu_mp_setmaxid(void)
+{
+	phandle_t child;
+	phandle_t root;
+	char buf[128];
+	int cpus;
+
+	all_cpus = 1 << PCPU_GET(cpuid);
+	mp_ncpus = 1;
+
+	cpus = 0;
+	root = OF_peer(0);
+	for (child = OF_child(root); child != 0; child = OF_peer(child)) {
+		if (OF_getprop(child, "device_type", buf, sizeof(buf)) > 0 &&
+		    strcmp(buf, "cpu") == 0)
+			cpus++;
+	}
+	mp_maxid = cpus - 1;
+
+}
+
+int
+cpu_mp_probe(void)
+{
+	return (mp_maxid > 0);
+}
+
+static int
+start_ap_bycpuid(int cpuid, void *func, u_long arg)
+{
+	static struct {
+		cell_t	name;
+		cell_t	nargs;
+		cell_t	nreturns;
+		cell_t	cpuid;
+		cell_t	func;
+		cell_t	arg;
+		cell_t	result;
+	} args = {
+		(cell_t)"SUNW,start-cpu-by-cpuid",
+		3,
+		1,
+		0,
+		0,
+		0,
+		0
+	};
+
+	args.cpuid = cpuid;
+	args.func = (cell_t)func;
+	args.arg = (cell_t)arg;
+	openfirmware(&args);
+	return (int)args.result;
+	
+}
+
+/*
+ * Fire up any non-boot processors.
+ */
+void
+cpu_mp_start(void)
+{
+	volatile struct cpu_start_args *csa;
+	struct pcpu *pc;
+	phandle_t child;
+	phandle_t root;
+	vm_offset_t va;
+	char buf[128];
+	u_int clock;
+	int cpuid, bp_skipped;
+	u_long s;
+
+	root = OF_peer(0);
+	csa = &cpu_start_args;
+	cpuid = bp_skipped = 0;
+	for (child = OF_child(root); child != 0; child = OF_peer(child)) {
+		if (OF_getprop(child, "device_type", buf, sizeof(buf)) <= 0 ||
+		    strcmp(buf, "cpu") != 0)
+			continue;
+		/* skip boot processor */
+		if (!bp_skipped) {
+			bp_skipped = 1;
+			continue;
+		}
+		cpuid++;
+
+		if (OF_getprop(child, "clock-frequency", &clock,
+		    sizeof(clock)) <= 0)
+			panic("cpu_mp_start: can't get clock");
+
+		csa->csa_state = 0;
+		start_ap_bycpuid(cpuid, (void *)mp_tramp, (uint64_t)cpuid);
+		s = intr_disable();
+		while (csa->csa_state != CPU_INIT)
+			;
+		intr_restore(s);
+		mp_ncpus = cpuid + 1;
+#if 0
+		cpu_identify(0, clock, cpuid);
+#endif
+		va = kmem_alloc(kernel_map, PCPU_PAGES * PAGE_SIZE);
+		pc = (struct pcpu *)(va + (PCPU_PAGES * PAGE_SIZE)) - 1;
+		pcpu_init(pc, cpuid, sizeof(*pc));
+		pc->pc_addr = va;
+
+		all_cpus |= 1 << cpuid;
+
+		if (mp_ncpus == MAXCPU)
+			break;
+	}
+	printf("%d cpus: UltraSparc T1 Processor (%d.%02d MHz CPU)\n", mp_ncpus,
+	    (clock + 4999) / 1000000, ((clock + 4999) / 10000) % 100);
+
+	PCPU_SET(other_cpus, all_cpus & ~(1 << PCPU_GET(cpuid)));
+	smp_active = 1;
+}
+
+void
+cpu_mp_announce(void)
+{
+}
+
+void
+cpu_mp_unleash(void *v)
+{
+	volatile struct cpu_start_args *csa;
+	struct pcpu *pc;
+	u_long s;
+
+	csa = &cpu_start_args;
+	csa->csa_count = mp_ncpus;
+	printf("mp_ncpus=%d\n", mp_ncpus);
+	SLIST_FOREACH(pc, &cpuhead, pc_allcpu) {
+		if (pc->pc_cpuid == PCPU_GET(cpuid)) 
+			continue;
+
+		KASSERT(pc->pc_idlethread != NULL,
+		    ("cpu_mp_unleash: idlethread is NULL"));
+		pc->pc_curthread = pc->pc_idlethread;	
+		pc->pc_curpcb = pc->pc_curthread->td_pcb;
+		pc->pc_curpmap = kernel_pmap;
+		csa->csa_state = 0;
+		csa->csa_pcpu = TLB_PHYS_TO_DIRECT(vtophys(pc->pc_addr));
+		DELAY(300);
+		/* allow AP to run */
+		csa->csa_cpuid = pc->pc_cpuid;
+		membar(Sync);
+		s = intr_disable();
+		while (csa->csa_state != CPU_BOOTSTRAP)
+			;
+		intr_restore(s);
+	}
+
+	membar(StoreLoad);
+	csa->csa_count = 0; 
+	smp_started = 1;
+}
+
+void
+cpu_mp_bootstrap(struct pcpu *pc)
+{
+	volatile struct cpu_start_args *csa;
+
+	csa = &cpu_start_args;
+	cpu_setregs(pc);
+	tsb_set_scratchpad_kernel(&kernel_pmap->pm_tsb);
+	tte_hash_set_scratchpad_kernel(kernel_pmap->pm_hash);
+	trap_init();
+	cpu_intrq_init();
+	tick_start();
+
+#ifdef TRAP_TRACING
+	mp_trap_trace_init();
+#endif
+
+	/* 
+	 * enable interrupts now that we have our trap table set
+	 */
+	intr_restore_all(PSTATE_KERNEL);
+
+	smp_cpus++;
+	KASSERT(curthread != NULL, ("cpu_mp_bootstrap: curthread"));
+	PCPU_SET(other_cpus, all_cpus & ~(1 << PCPU_GET(cpuid)));
+	printf("AP: #%d\n", PCPU_GET(cpuid));
+	csa->csa_count--;
+	membar(StoreLoad);
+	csa->csa_state = CPU_BOOTSTRAP;
+
+	while (csa->csa_count != 0)
+		;
+	/* ok, now grab sched_lock and enter the scheduler */
+	mtx_lock_spin(&sched_lock);
+	spinlock_exit();
+	PCPU_SET(switchtime, cpu_ticks());
+	PCPU_SET(switchticks, ticks);
+
+	cpu_throw(NULL, choosethread());	/* doesn't return */
+}
+
+void
+cpu_mp_shutdown(void)
+{
+	int i;
+
+	critical_enter();
+	shutdown_cpus = PCPU_GET(other_cpus);
+	if (stopped_cpus != PCPU_GET(other_cpus))	/* XXX */
+		stop_cpus(stopped_cpus ^ PCPU_GET(other_cpus));
+	i = 0;
+	while (shutdown_cpus != 0) {
+		if (i++ > 100000) {
+			printf("timeout shutting down CPUs.\n");
+			break;
+		}
+	}
+	/* XXX: delay a bit to allow the CPUs to actually enter the PROM. */
+	DELAY(100000);
+	critical_exit();
+}
+
+void
+cpu_ipi_ast(struct trapframe *tf)
+{
+}
+
+void
+cpu_ipi_stop(struct trapframe *tf)
+{
+
+	CTR1(KTR_SMP, "cpu_ipi_stop: stopped %d", PCPU_GET(cpuid));
+	savectx(&stoppcbs[PCPU_GET(cpuid)]);
+	atomic_set_acq_int(&stopped_cpus, PCPU_GET(cpumask));
+	while ((started_cpus & PCPU_GET(cpumask)) == 0) {
+		if ((shutdown_cpus & PCPU_GET(cpumask)) != 0) {
+			atomic_clear_int(&shutdown_cpus, PCPU_GET(cpumask));
+			OF_exit();
+		}
+	}
+	atomic_clear_rel_int(&started_cpus, PCPU_GET(cpumask));
+	atomic_clear_rel_int(&stopped_cpus, PCPU_GET(cpumask));
+	CTR1(KTR_SMP, "cpu_ipi_stop: restarted %d", PCPU_GET(cpuid));
+}
+
+void
+cpu_ipi_selected(int cpu_count, uint16_t *cpulist, u_long d0, u_long d1, u_long d2, uint64_t *ackmask)
+{
+
+	int i, retries;
+
+	init_mondo(d0, d1, d2, (uint64_t)pmap_kextract((vm_offset_t)ackmask));
+
+	retries = 0;
+
+retry:
+	if (cpu_count) {
+		int error, new_cpu_count;
+		vm_paddr_t cpulist_ra;
+
+		cpulist_ra = TLB_DIRECT_TO_PHYS((vm_offset_t)cpulist);
+		if ((error = hv_cpu_mondo_send(cpu_count, cpulist_ra)) == H_EWOULDBLOCK) {
+			new_cpu_count = 0;
+			for (i = 0; i < cpu_count; i++) {
+				if (cpulist[i] != 0xffff)
+					cpulist[new_cpu_count++] = cpulist[i];
+			}      
+			cpu_count = new_cpu_count;
+			retries++;
+			if (cpu_count == 0) {
+				printf("no more cpus to send to but mondo_send returned EWOULDBLOCK\n");
+				return;
+			}
+			if ((retries & 0x1) == 0x1)
+				DELAY(10);
+
+			if (retries < 50000)
+				goto retry;
+			else {
+				printf("used up retries - cpus remaining: %d  - cpus: ",
+				       cpu_count);
+				for (i = 0; i < cpu_count; i++)
+					printf("#%d ", cpulist[i]);
+				printf("\n");
+			}
+		}
+		if (error == H_ENOCPU) {
+			printf("bad cpuid: ");
+			for (i = 0; i < cpu_count; i++)
+				printf("#%d ", cpulist[i]);
+			printf("\n");
+		}		
+		if (error)
+			panic("can't handle error %d from cpu_mondo_send\n", error);
+	}
+}
+
+
+void
+ipi_selected(u_int icpus, u_int ipi)
+{
+	int i, cpu_count;
+	uint16_t *cpulist;
+	cpumask_t cpus;
+	uint64_t ackmask;
+
+	/* 
+	 * 
+	 * 3) forward_wakeup appears to abuse ASTs
+	 * 4) handling 4-way threading vs 2-way threading should happen here
+	 *    and not in forward wakeup
+	 */
+	
+	cpulist = PCPU_GET(cpulist);
+	cpus = (icpus & ~PCPU_GET(cpumask));
+	
+	for (cpu_count = 0, i = 0; i < 32 && cpus; cpus = cpus >> 1, i++) {
+		if (!(cpus & 0x1))
+			continue;
+		
+		cpulist[cpu_count] = (uint16_t)i;
+		cpu_count++;
+	}
+
+	cpu_ipi_selected(cpu_count, cpulist, (u_long)tl_ipi_level, ipi, 0, &ackmask);
+	
+}
+
+void
+ipi_all_but_self(u_int ipi)
+{
+	ipi_selected(PCPU_GET(other_cpus), ipi);
+}
diff --git a/sys/sun4v/sun4v/nexus.c b/sys/sun4v/sun4v/nexus.c
new file mode 100644
index 000000000000..6c3a393efc89
--- /dev/null
+++ b/sys/sun4v/sun4v/nexus.c
@@ -0,0 +1,625 @@
+/*-
+ * Copyright 1998 Massachusetts Institute of Technology
+ * Copyright 2001 by Thomas Moestl <tmm@FreeBSD.org>.
+ * Copyright 2006 by Marius Strobl <marius@FreeBSD.org>.
+ * All rights reserved.
+ *
+ * Permission to use, copy, modify, and distribute this software and
+ * its documentation for any purpose and without fee is hereby
+ * granted, provided that both the above copyright notice and this
+ * permission notice appear in all copies, that both the above
+ * copyright notice and this permission notice appear in all
+ * supporting documentation, and that the name of M.I.T. not be used
+ * in advertising or publicity pertaining to distribution of the
+ * software without specific, written prior permission.  M.I.T. makes
+ * no representations about the suitability of this software for any
+ * purpose.  It is provided "as is" without express or implied
+ * warranty.
+ *
+ * THIS SOFTWARE IS PROVIDED BY M.I.T. ``AS IS''.  M.I.T. DISCLAIMS
+ * ALL EXPRESS OR IMPLIED WARRANTIES WITH REGARD TO THIS SOFTWARE,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. IN NO EVENT
+ * SHALL M.I.T. BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * 	from: FreeBSD: src/sys/i386/i386/nexus.c,v 1.43 2001/02/09
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/bus.h>
+#include <sys/kernel.h>
+#include <sys/malloc.h>
+#include <sys/module.h>
+
+#include <dev/ofw/ofw_bus.h>
+#include <dev/ofw/ofw_bus_subr.h>
+#include <dev/ofw/openfirm.h>
+
+#include <machine/bus.h>
+#include <machine/bus_common.h>
+#include <machine/intr_machdep.h>
+#include <machine/nexusvar.h>
+#include <machine/ofw_nexus.h>
+#include <machine/resource.h>
+#include <machine/ver.h>
+
+#include <sys/rman.h>
+
+/*
+ * The nexus (which is a pseudo-bus actually) iterates over the nodes that
+ * hang from the Open Firmware root node and adds them as devices to this bus
+ * (except some special nodes which are excluded) so that drivers can be
+ * attached to them.
+ *
+ * Additionally, interrupt setup/teardown and some resource management are
+ * done at this level.
+ *
+ * Maybe this code should get into dev/ofw to some extent, as some of it should
+ * work for all Open Firmware based machines...
+ */
+
+struct nexus_devinfo {
+	struct ofw_bus_devinfo	ndi_obdinfo;
+	struct resource_list	ndi_rl;
+	struct rman		ndi_intr_rman;
+
+	devhandle_t		ndi_devhandle;
+};
+
+struct nexus_softc {
+	struct rman	sc_mem_rman;
+};
+
+static device_probe_t nexus_probe;
+static device_attach_t nexus_attach;
+static bus_print_child_t nexus_print_child;
+static bus_add_child_t nexus_add_child;
+static bus_probe_nomatch_t nexus_probe_nomatch;
+static bus_read_ivar_t nexus_read_ivar;
+static bus_setup_intr_t nexus_setup_intr;
+static bus_teardown_intr_t nexus_teardown_intr;
+static bus_alloc_resource_t nexus_alloc_resource;
+static bus_get_resource_list_t nexus_get_resource_list;
+static bus_activate_resource_t nexus_activate_resource;
+static bus_deactivate_resource_t nexus_deactivate_resource;
+static bus_release_resource_t nexus_release_resource;
+static ofw_bus_get_devinfo_t nexus_get_devinfo;
+
+static int nexus_inlist(const char *, const char **);
+static struct nexus_devinfo * nexus_setup_dinfo(device_t, phandle_t);
+static void nexus_destroy_dinfo(struct nexus_devinfo *);
+static int nexus_print_res(struct nexus_devinfo *);
+
+static device_method_t nexus_methods[] = {
+	/* Device interface */
+	DEVMETHOD(device_probe,		nexus_probe),
+	DEVMETHOD(device_attach,	nexus_attach),
+	DEVMETHOD(device_detach,	bus_generic_detach),
+	DEVMETHOD(device_shutdown,	bus_generic_shutdown),
+	DEVMETHOD(device_suspend,	bus_generic_suspend),
+	DEVMETHOD(device_resume,	bus_generic_resume),
+
+	/* Bus interface */
+	DEVMETHOD(bus_print_child,	nexus_print_child),
+	DEVMETHOD(bus_probe_nomatch,	nexus_probe_nomatch),
+	DEVMETHOD(bus_read_ivar,	nexus_read_ivar),
+	DEVMETHOD(bus_write_ivar,	bus_generic_write_ivar),
+	DEVMETHOD(bus_add_child,	nexus_add_child),
+	DEVMETHOD(bus_alloc_resource,	nexus_alloc_resource),
+	DEVMETHOD(bus_activate_resource,	nexus_activate_resource),
+	DEVMETHOD(bus_deactivate_resource,	nexus_deactivate_resource),
+	DEVMETHOD(bus_release_resource,	nexus_release_resource),
+	DEVMETHOD(bus_setup_intr,	nexus_setup_intr),
+	DEVMETHOD(bus_teardown_intr,	nexus_teardown_intr),
+	DEVMETHOD(bus_get_resource,	bus_generic_rl_get_resource),
+	DEVMETHOD(bus_get_resource_list, nexus_get_resource_list),
+
+	/* ofw_bus interface */
+	DEVMETHOD(ofw_bus_get_devinfo,	nexus_get_devinfo),
+	DEVMETHOD(ofw_bus_get_compat,	ofw_bus_gen_get_compat),
+	DEVMETHOD(ofw_bus_get_model,	ofw_bus_gen_get_model),
+	DEVMETHOD(ofw_bus_get_name,	ofw_bus_gen_get_name),
+	DEVMETHOD(ofw_bus_get_node,	ofw_bus_gen_get_node),
+	DEVMETHOD(ofw_bus_get_type,	ofw_bus_gen_get_type),
+
+	{ 0, 0 }
+};
+
+static devclass_t nexus_devclass;
+
+DEFINE_CLASS_0(nexus, nexus_driver, nexus_methods, sizeof(struct nexus_softc));
+DRIVER_MODULE(nexus, root, nexus_driver, nexus_devclass, 0, 0);
+
+static const char *nexus_excl_name[] = {
+	"aliases",
+	"associations",
+	"chosen",
+	"counter-timer",	/* No separate device; handled by psycho/sbus */
+	"memory",
+	"openprom",
+	"options",
+	"packages",
+	"rsc",
+	"virtual-memory",
+	NULL
+};
+
+static const char *nexus_excl_type[] = {
+	"cpu",
+	NULL
+};
+
+extern struct bus_space_tag nexus_bustag;
+
+#define	SUN4V_REG_SPEC2CFG_HDL(x)	((x >> 32) & ~(0xfull << 28))
+
+static int
+nexus_inlist(const char *name, const char **list)
+{
+	int i;
+
+	if (name == NULL)
+		return (0);
+	for (i = 0; list[i] != NULL; i++)
+		if (strcmp(name, list[i]) == 0)
+			return (1);
+	return (0);
+}
+
+#define	NEXUS_EXCLUDED(name, type)					\
+	(nexus_inlist((name), nexus_excl_name) ||			\
+	((type) != NULL && nexus_inlist((type), nexus_excl_type)))
+
+static int
+nexus_probe(device_t dev)
+{
+
+	/* Nexus does always match. */
+	device_set_desc(dev, "Open Firmware Nexus device");
+	return (0);
+}
+
+static int
+nexus_attach(device_t dev)
+{
+	struct nexus_devinfo *ndi;
+	struct nexus_softc *sc;
+	device_t cdev;
+	phandle_t node;
+
+	node = OF_peer(0);
+	if (node == -1)
+		panic("%s: OF_peer failed.", __func__);
+
+	sc = device_get_softc(dev);
+	sc->sc_mem_rman.rm_type = RMAN_ARRAY;
+	sc->sc_mem_rman.rm_descr = "Device Memory";
+	if (rman_init(&sc->sc_mem_rman) != 0 ||
+	    rman_manage_region(&sc->sc_mem_rman, 0ULL, ~0ULL) != 0)
+		panic("%s: failed to set up rmans.", __func__);
+
+	/*
+	 * Allow devices to identify.
+	 */
+	bus_generic_probe(dev);
+
+	/*
+	 * Now walk the OFW tree and attach top-level devices.
+	 */
+	for (node = OF_child(node); node > 0; node = OF_peer(node)) {
+		if ((ndi = nexus_setup_dinfo(dev, node)) == NULL)
+			continue;
+		cdev = device_add_child(dev, NULL, -1);
+		if (cdev == NULL) {
+			device_printf(dev, "<%s>: device_add_child failed\n",
+			    ndi->ndi_obdinfo.obd_name);
+			nexus_destroy_dinfo(ndi);
+			continue;
+		}
+		device_set_ivars(cdev, ndi);
+	}
+	return (bus_generic_attach(dev));
+}
+
+static device_t
+nexus_add_child(device_t dev, int order, const char *name, int unit)
+{
+	device_t cdev;
+	struct nexus_devinfo *ndi;
+
+	cdev = device_add_child_ordered(dev, order, name, unit);
+	if (cdev == NULL)
+		return (NULL);
+
+	ndi = malloc(sizeof(*ndi), M_DEVBUF, M_WAITOK | M_ZERO);
+	ndi->ndi_obdinfo.obd_node = -1;
+	ndi->ndi_obdinfo.obd_name = strdup(name, M_OFWPROP);
+	resource_list_init(&ndi->ndi_rl);
+	ndi->ndi_intr_rman.rm_type = RMAN_ARRAY;
+	ndi->ndi_intr_rman.rm_descr = "Interrupts";
+	if (rman_init(&ndi->ndi_intr_rman) != 0 ||
+	    rman_manage_region(&ndi->ndi_intr_rman, 0, IV_MAX - 1) != 0)
+		panic("%s: failed to set up rmans.", __func__);
+
+	device_set_ivars(cdev, ndi);
+
+	return (cdev);
+}
+
+static int
+nexus_print_child(device_t dev, device_t child)
+{
+	int rv;
+
+	rv = bus_print_child_header(dev, child);
+	rv += nexus_print_res(device_get_ivars(child));
+	rv += bus_print_child_footer(dev, child);
+	return (rv);
+}
+
+static void
+nexus_probe_nomatch(device_t dev, device_t child)
+{
+	const char *type;
+
+	device_printf(dev, "<%s>", ofw_bus_get_name(child));
+	nexus_print_res(device_get_ivars(child));
+	type = ofw_bus_get_type(child);
+	printf(" type %s (no driver attached)\n",
+	    type != NULL ? type : "unknown");
+}
+
+static int
+nexus_read_ivar(device_t dev, device_t child, int which, uintptr_t *result)
+{
+	struct nexus_devinfo *ndi;
+
+	ndi = device_get_ivars(child);
+
+	switch (which) {
+	case NEXUS_IVAR_DEVHANDLE:
+		*(uint64_t *)result = ndi->ndi_devhandle;
+		break;
+
+	default:
+		return (ENOENT);
+	}
+
+	return (0);
+}
+
+#ifdef SUN4V
+
+static int
+nexus_setup_intr(device_t dev, device_t child, struct resource *res, int flags,
+    driver_intr_t *intr, void *arg, void **cookiep)
+{
+	struct nexus_devinfo *ndi;
+	device_t ichild;
+	uint64_t ihdl;
+	uint64_t ino;
+	int error, cpuid;
+
+	/* we need the devinfo from the immediate child */
+	ichild = child;
+	while (device_get_parent(ichild) != dev)
+		ichild = device_get_parent(ichild);
+
+	ndi = device_get_ivars(ichild);
+
+	if (res == NULL)
+		panic("%s: NULL interrupt resource!", __func__);
+#ifdef DEBUG
+	printf("dev=%s child=%s\n", ofw_bus_get_name(dev), ofw_bus_get_name(child));
+#endif
+	ino = rman_get_start(res);
+#ifdef DEBUG
+	printf("child=%s reg=0x%lx ino=0x%lx\n", ofw_bus_get_name(child),
+	       ndi->ndi_devhandle, ino);
+#endif
+
+	if (hvio_intr_devino_to_sysino(ndi->ndi_devhandle, (uint32_t)ino,
+	    &ihdl) != H_EOK) {
+		error = ENXIO;
+		goto fail;
+	}
+
+	if ((rman_get_flags(res) & RF_SHAREABLE) == 0)
+		flags |= INTR_EXCL;
+
+	/* We depend here on rman_activate_resource() being idempotent. */
+	if ((error = rman_activate_resource(res)))
+		goto fail;
+
+	error = inthand_add(device_get_nameunit(child), ihdl,
+			   intr, arg, flags, cookiep);
+
+	cpuid = 0;
+	if (hvio_intr_settarget(ihdl, cpuid) != H_EOK) {
+		error = ENXIO;
+		goto fail;
+	}
+	if (hvio_intr_setstate(ihdl, HV_INTR_IDLE_STATE) != H_EOK) {
+		error = ENXIO;
+		goto fail;
+	}
+	if (hvio_intr_setvalid(ihdl, HV_INTR_VALID) != H_EOK) {
+		error = ENXIO;
+		goto fail;
+	}
+
+
+fail:
+
+	return (error);
+}
+
+static int
+nexus_teardown_intr(device_t dev, device_t child, struct resource *r, void *ih)
+{
+
+	inthand_remove(rman_get_start(r), ih);
+	return (0);
+}
+
+#else
+
+static int
+nexus_setup_intr(device_t dev, device_t child, struct resource *res, int flags,
+    driver_intr_t *intr, void *arg, void **cookiep)
+{
+	int error;
+
+	if (res == NULL)
+		panic("%s: NULL interrupt resource!", __func__);
+
+	if ((rman_get_flags(res) & RF_SHAREABLE) == 0)
+		flags |= INTR_EXCL;
+
+	/* We depend here on rman_activate_resource() being idempotent. */
+	error = rman_activate_resource(res);
+	if (error)
+		return (error);
+
+	error = inthand_add(device_get_nameunit(child), rman_get_start(res),
+	    intr, arg, flags, cookiep);
+
+	return (error);
+}
+
+static int
+nexus_teardown_intr(device_t dev, device_t child, struct resource *r, void *ih)
+{
+
+	inthand_remove(rman_get_start(r), ih);
+	return (0);
+}
+
+#endif
+
+static struct resource *
+nexus_alloc_resource(device_t bus, device_t child, int type, int *rid,
+    u_long start, u_long end, u_long count, u_int flags)
+{
+	struct nexus_softc *sc;
+	struct rman *rm;
+	struct resource *rv;
+	struct resource_list_entry *rle;
+	struct nexus_devinfo *ndi;
+	device_t subord;
+	int isdefault, needactivate, passthrough;
+
+	isdefault = (start == 0UL && end == ~0UL);
+	needactivate = flags & RF_ACTIVE;
+	passthrough = (device_get_parent(child) != bus);
+	sc = device_get_softc(bus);
+	rle = NULL;
+
+	if (!passthrough) {
+		rle = resource_list_find(BUS_GET_RESOURCE_LIST(bus, child),
+		    type, *rid);
+		if (rle == NULL)
+			return (NULL);
+		if (rle->res != NULL)
+			panic("%s: resource entry is busy", __func__);
+		if (isdefault) {
+			start = rle->start;
+			count = ulmax(count, rle->count);
+			end = ulmax(rle->end, start + count - 1);
+		}
+	}
+
+	switch (type) {
+	case SYS_RES_IRQ:
+		/* find the immediate subordinate */
+		subord = child;
+		while (device_get_parent(subord) != bus)
+			subord = device_get_parent(subord);
+		ndi = device_get_ivars(subord);
+		rm = &ndi->ndi_intr_rman;
+		break;
+	case SYS_RES_MEMORY:
+		rm = &sc->sc_mem_rman;
+		break;
+	default:
+		return (NULL);
+	}
+
+	flags &= ~RF_ACTIVE;
+	rv = rman_reserve_resource(rm, start, end, count, flags, child);
+	if (type == SYS_RES_IRQ && rv == NULL) {
+		printf("%s: start: %ld, end: %ld, flags: %#x\n", __func__,
+		    start, end, flags);
+	}
+	if (rv == NULL)
+		return (NULL);
+	if (type == SYS_RES_MEMORY) {
+		rman_set_bustag(rv, &nexus_bustag);
+		rman_set_bushandle(rv, rman_get_start(rv));
+	}
+
+	if (needactivate) {
+		if (bus_activate_resource(child, type, *rid, rv) != 0) {
+			rman_release_resource(rv);
+			return (NULL);
+		}
+	}
+
+	if (!passthrough)
+		rle->res = rv;
+
+	return (rv);
+}
+
+static int
+nexus_activate_resource(device_t bus, device_t child, int type, int rid,
+    struct resource *r)
+{
+
+	/* Not much to be done yet... */
+	return (rman_activate_resource(r));
+}
+
+static int
+nexus_deactivate_resource(device_t bus, device_t child, int type, int rid,
+    struct resource *r)
+{
+
+	/* Not much to be done yet... */
+	return (rman_deactivate_resource(r));
+}
+
+static int
+nexus_release_resource(device_t bus, device_t child, int type, int rid,
+		       struct resource *r)
+{
+	int error;
+
+	if (rman_get_flags(r) & RF_ACTIVE) {
+		error = bus_deactivate_resource(child, type, rid, r);
+		if (error)
+			return (error);
+	}
+	return (rman_release_resource(r));
+}
+
+static struct resource_list *
+nexus_get_resource_list(device_t dev, device_t child)
+{
+	struct nexus_devinfo *ndi;
+
+	ndi = device_get_ivars(child);
+	return (&ndi->ndi_rl);
+}
+
+static const struct ofw_bus_devinfo *
+nexus_get_devinfo(device_t dev, device_t child)
+{
+	struct nexus_devinfo *ndi;
+
+	ndi = device_get_ivars(child);
+	return (&ndi->ndi_obdinfo);
+}
+
+static struct nexus_devinfo *
+nexus_setup_dinfo(device_t dev, phandle_t node)
+{
+	struct nexus_devinfo *ndi;
+	struct nexus_regs *reg;
+	bus_addr_t phys;
+	uint32_t naddrcell, nintrcell;
+	uint32_t *intr;
+	int i, rid;
+	int nintr;
+	int nreg;
+
+	ndi = malloc(sizeof(*ndi), M_DEVBUF, M_WAITOK | M_ZERO);
+	if (ofw_bus_gen_setup_devinfo(&ndi->ndi_obdinfo, node) != 0) {
+		free(ndi, M_DEVBUF);
+		return (NULL);
+	}
+	if (NEXUS_EXCLUDED(ndi->ndi_obdinfo.obd_name,
+	    ndi->ndi_obdinfo.obd_type)) {
+		ofw_bus_gen_destroy_devinfo(&ndi->ndi_obdinfo);
+		free(ndi, M_DEVBUF);
+		return (NULL);
+	}
+	resource_list_init(&ndi->ndi_rl);
+	nreg = OF_getprop_alloc(node, "reg", sizeof(*reg), (void **)&reg);
+	if (nreg != 1) {
+		device_printf(dev, "<%s>: incomplete: %d\n",
+		    ndi->ndi_obdinfo.obd_name, nreg);
+		nexus_destroy_dinfo(ndi);
+		return (NULL);
+	}
+	phys = NEXUS_REG_PHYS(&reg[0]);
+	ndi->ndi_devhandle = SUN4V_REG_SPEC2CFG_HDL(phys);
+	free(reg, M_OFWPROP);
+
+	if (OF_getprop(node, "#address-cells", &naddrcell,
+	    sizeof naddrcell) != sizeof naddrcell || OF_getprop(node,
+	    "#interrupt-cells", &nintrcell, sizeof nintrcell) !=
+	    sizeof nintrcell || nintrcell != 1)
+		panic("can't get cell sizes"); /* or invalid intr cell size */
+
+	nintr = OF_getprop_alloc(node, "interrupt-map",  sizeof(*intr),
+	    (void **)&intr);
+	for (i = 0; i < nintr; rid++, i += naddrcell + nintrcell + 2) {
+		/*
+		 * interrupt-map is:
+		 * addr,intr,phandle,parent's intr
+		 */
+		rid = intr[i + naddrcell];
+		/* XXX - technically, we need to pull the parent's
+		 * #interrupt-cells, but we'll assume it's 1 like we enforce
+		 * nintercell to be above.
+		 */
+		resource_list_add(&ndi->ndi_rl, SYS_RES_IRQ, rid,
+		    intr[i + naddrcell + nintrcell + 1],
+		    intr[i + naddrcell + nintrcell + 1], 1);
+	}
+	free(intr, M_OFWPROP);
+
+	ndi->ndi_intr_rman.rm_type = RMAN_ARRAY;
+	ndi->ndi_intr_rman.rm_descr = "Interrupts";
+	if (rman_init(&ndi->ndi_intr_rman) != 0 ||
+	    rman_manage_region(&ndi->ndi_intr_rman, 0, IV_MAX - 1) != 0)
+		panic("%s: failed to set up rmans.", __func__);
+
+	return (ndi);
+}
+
+static void
+nexus_destroy_dinfo(struct nexus_devinfo *ndi)
+{
+
+	resource_list_free(&ndi->ndi_rl);
+	ofw_bus_gen_destroy_devinfo(&ndi->ndi_obdinfo);
+	free(ndi, M_DEVBUF);
+}
+
+static int
+nexus_print_res(struct nexus_devinfo *ndi)
+{
+	int rv;
+
+	rv = 0;
+	rv += resource_list_print_type(&ndi->ndi_rl, "mem", SYS_RES_MEMORY,
+	    "%#lx");
+	rv += resource_list_print_type(&ndi->ndi_rl, "irq", SYS_RES_IRQ,
+	    "%ld");
+	return (rv);
+}
+
+
diff --git a/sys/sun4v/sun4v/ofw_bus.c b/sys/sun4v/sun4v/ofw_bus.c
new file mode 100644
index 000000000000..f7458dcc62fd
--- /dev/null
+++ b/sys/sun4v/sun4v/ofw_bus.c
@@ -0,0 +1,198 @@
+/*-
+ * Copyright (C) 1996 Wolfgang Solfrank.
+ * Copyright (C) 1996 TooLs GmbH.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ *    must display the following acknowledgement:
+ *	This product includes software developed by TooLs GmbH.
+ * 4. The name of TooLs GmbH may not be used to endorse or promote products
+ *    derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+ * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+/*-
+ * Copyright (c) 2001 - 2003 by Thomas Moestl <tmm@FreeBSD.org>.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+ * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ *	from: $NetBSD: ofw_machdep.c,v 1.16 2001/07/20 00:07:14 eeh Exp $
+ *
+ * $FreeBSD$
+ */
+
+/*
+ * Open Firmware bus support code that is (hopefully) independent from the
+ * used hardware.
+ * Maybe this should go into dev/ofw/; there may however be sparc specific
+ * bits left.
+ */
+
+#include <sys/param.h>
+#include <sys/malloc.h>
+#include <sys/systm.h>
+
+#include <dev/ofw/openfirm.h>
+
+#include <machine/ofw_bus.h>
+
+static int
+ofw_bus_searchprop(phandle_t node, char *propname, void *buf, int buflen)
+{
+	int rv;
+
+	for (; node != 0; node = OF_parent(node)) {
+		if ((rv = OF_getprop(node, propname, buf, buflen)) != -1)
+			return (rv);
+	}
+	return (-1);
+}
+
+void
+ofw_bus_setup_iinfo(phandle_t node, struct ofw_bus_iinfo *ii, int intrsz)
+{
+	pcell_t addrc;
+	int msksz;
+
+	if (OF_getprop(node, "#address-cells", &addrc, sizeof(addrc)) == -1)
+		addrc = 2;
+	ii->opi_addrc = addrc * sizeof(pcell_t);
+
+	ii->opi_imapsz = OF_getprop_alloc(node, "interrupt-map", 1,
+	    (void **)&ii->opi_imap);
+	if (ii->opi_imapsz > 0) {
+		msksz = OF_getprop_alloc(node, "interrupt-map-mask", 1,
+		    (void **)&ii->opi_imapmsk);
+		/*
+		 * Failure to get the mask is ignored; a full mask is used then.
+		 * Barf on bad mask sizes, however.
+		 */
+		if (msksz != -1 && msksz != ii->opi_addrc + intrsz) {
+			panic("ofw_bus_setup_iinfo: bad interrupt-map-mask "
+			    "property!");
+		}
+	}
+
+}
+
+int
+ofw_bus_lookup_imap(phandle_t node, struct ofw_bus_iinfo *ii, void *reg,
+    int regsz, void *pintr, int pintrsz, void *mintr, int mintrsz,
+    void *maskbuf)
+{
+	int rv;
+
+	if (ii->opi_imapsz <= 0)
+		return (0);
+	KASSERT(regsz >= ii->opi_addrc,
+	    ("ofw_bus_lookup_imap: register size too small: %d < %d",
+		regsz, ii->opi_addrc));
+	rv = OF_getprop(node, "reg", reg, regsz);
+	if (rv < regsz)
+		panic("ofw_bus_lookup_imap: could not get reg property");
+	return (ofw_bus_search_intrmap(pintr, pintrsz, reg, ii->opi_addrc,
+	    ii->opi_imap, ii->opi_imapsz, ii->opi_imapmsk, maskbuf, mintr,
+	    mintrsz));
+}
+
+/*
+ * Map an interrupt using the firmware reg, interrupt-map and
+ * interrupt-map-mask properties.
+ * The interrupt property to be mapped must be of size intrsz, and pointed to
+ * by intr. The regs property of the node for which the mapping is done must
+ * be passed as regs. This property is an array of register specifications;
+ * the size of the address part of such a specification must be passed as
+ * physsz. Only the first element of the property is used.
+ * imap and imapsz hold the interrupt mask and it's size.
+ * imapmsk is a pointer to the interrupt-map-mask property, which must have
+ * a size of physsz + intrsz; it may be NULL, in which case a full mask is
+ * assumed.
+ * maskbuf must point to a buffer of length physsz + intrsz.
+ * The interrupt is returned in result, which must point to a buffer of length
+ * rintrsz (which gives the expected size of the mapped interrupt).
+ * Returns 1 if a mapping was found, 0 otherwise.
+ */
+int
+ofw_bus_search_intrmap(void *intr, int intrsz, void *regs, int physsz,
+    void *imap, int imapsz, void *imapmsk, void *maskbuf, void *result,
+    int rintrsz)
+{
+	phandle_t parent;
+	u_int8_t *ref = maskbuf;
+	u_int8_t *uiintr = intr;
+	u_int8_t *uiregs = regs;
+	u_int8_t *uiimapmsk = imapmsk;
+	u_int8_t *mptr;
+	pcell_t pintrsz;
+	int i, rsz, tsz;
+
+	rsz = -1;
+	if (imapmsk != NULL) {
+		for (i = 0; i < physsz; i++)
+			ref[i] = uiregs[i] & uiimapmsk[i];
+		for (i = 0; i < intrsz; i++)
+			ref[physsz + i] = uiintr[i] & uiimapmsk[physsz + i];
+	} else {
+		bcopy(regs, ref, physsz);
+		bcopy(intr, ref + physsz, intrsz);
+	}
+
+	mptr = imap;
+	i = imapsz;
+	tsz = physsz + intrsz + sizeof(phandle_t) + rintrsz;
+	while (i > 0) {
+		KASSERT(i >= tsz, ("ofw_bus_search_intrmap: truncated map"));
+		bcopy(mptr + physsz + intrsz, &parent, sizeof(parent));
+		if (ofw_bus_searchprop(parent, "#interrupt-cells",
+		    &pintrsz, sizeof(pintrsz)) == -1)
+			pintrsz = 1;	/* default */
+		pintrsz *= sizeof(pcell_t);
+		if (pintrsz != rintrsz)
+			panic("ofw_bus_search_intrmap: expected interrupt cell "
+			    "size incorrect: %d != %d", rintrsz, pintrsz);
+		if (bcmp(ref, mptr, physsz + intrsz) == 0) {
+			bcopy(mptr + physsz + intrsz + sizeof(parent),
+			    result, rintrsz);
+			return (1);
+		}
+		mptr += tsz;
+		i -= tsz;
+	}
+	return (0);
+}
diff --git a/sys/sun4v/sun4v/ofw_machdep.c b/sys/sun4v/sun4v/ofw_machdep.c
new file mode 100644
index 000000000000..016b43292245
--- /dev/null
+++ b/sys/sun4v/sun4v/ofw_machdep.c
@@ -0,0 +1,267 @@
+/*-
+ * Copyright (c) 2001 by Thomas Moestl <tmm@FreeBSD.org>.
+ * Copyright (c) 2005 by Marius Strobl <marius@FreeBSD.org>.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+ * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+ 
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+/*
+ * Some Open Firmware helper functions that are likely machine dependent.
+ */
+
+#include <sys/param.h>
+#include <sys/bus.h>
+#include <sys/systm.h>
+
+#include <net/ethernet.h>
+
+#include <dev/ofw/ofw_bus.h>
+#include <dev/ofw/ofw_pci.h>
+#include <dev/ofw/openfirm.h>
+
+#include <machine/bus.h>
+#include <machine/idprom.h>
+#include <machine/ofw_bus.h>
+#include <machine/ofw_machdep.h>
+
+void
+OF_getetheraddr(device_t dev, u_char *addr)
+{
+	char buf[sizeof("true")];
+	phandle_t node;
+	struct idprom idp;
+
+	if ((node = OF_finddevice("/options")) > 0 &&
+	    OF_getprop(node, "local-mac-address?", buf, sizeof(buf)) > 0) {
+		buf[sizeof(buf) - 1] = '\0';
+		if (strcmp(buf, "true") == 0 &&
+		    (node = ofw_bus_get_node(dev)) > 0 &&
+		    OF_getprop(node, "local-mac-address", addr,
+		    ETHER_ADDR_LEN) == ETHER_ADDR_LEN)
+			return;
+	}
+
+	node = OF_peer(0);
+	if (node <= 0 || OF_getprop(node, "idprom", &idp, sizeof(idp)) == -1)
+		panic("Could not determine the machine ethernet address");
+	bcopy(&idp.id_ether, addr, ETHER_ADDR_LEN);
+}
+
+static __inline uint32_t
+phys_hi_mask_space(const char *bus, uint32_t phys_hi)
+{
+	uint32_t space;
+
+	space = phys_hi;
+	if (strcmp(bus, "ebus") == 0 || strcmp(bus, "isa") == 0)
+		space &= 0x1;
+	else if (strcmp(bus, "pci") == 0)
+		space &= OFW_PCI_PHYS_HI_SPACEMASK;
+	/* The phys.hi cells of the other busses only contain space bits. */
+	return (space);
+}
+
+/*
+ * Return the physical address and the bus space to use for a node
+ * referenced by its package handle and the index of the register bank
+ * to decode. Intended to be used to together with sparc64_fake_bustag()
+ * by console drivers in early boot only.
+ * Works by mapping the address of the node's bank given in the address
+ * space of its parent upward in the device tree at each bridge along the
+ * path.
+ * Currently only really deals with max. 64-bit addresses, i.e. addresses
+ * consisting of max. 2 phys cells (phys.hi and phys.lo). If we encounter
+ * a 3 phys cells address (as with PCI addresses) we assume phys.hi can
+ * be ignored except for the space bits (generally contained in phys.hi)
+ * and treat phys.mid as phys.hi.
+ */
+int
+OF_decode_addr(phandle_t node, int bank, int *space, bus_addr_t *addr)
+{
+	char name[32];
+	uint64_t cend, cstart, end, phys, sz, start;
+	pcell_t addrc, szc, paddrc;
+	phandle_t bus, lbus, pbus;
+	uint32_t banks[10 * 5];	/* 10 PCI banks */
+	uint32_t cspace, spc;
+	int i, j, nbank;
+
+	/*
+	 * In general the addresses are contained in the "reg" property
+	 * of a node. The first address in the "reg" property of a PCI
+	 * node however is the address of its configuration registers in
+	 * the configuration space of the host bridge. Additional entries
+	 * denote the memory and I/O addresses. For relocatable addresses
+	 * the "reg" property contains the BAR, for non-relocatable
+	 * addresses it contains the absolute PCI address. The PCI-only
+	 * "assigned-addresses" property however always contains the
+	 * absolute PCI addresses.
+	 * The "assigned-addresses" and "reg" properties are arrays of
+	 * address structures consisting of #address-cells 32-bit phys
+	 * cells and #size-cells 32-bit size cells. If a parent lacks
+	 * the "#address-cells" or "#size-cells" property the default
+	 * for #address-cells to use is 2 and for #size-cells 1.
+	 */
+	bus = OF_parent(node);
+	if (bus == 0)
+		return (ENXIO);
+	if (OF_getprop(bus, "name", name, sizeof(name)) == -1)
+		return (ENXIO);
+	name[sizeof(name) - 1] = '\0';
+	if (OF_getprop(bus, "#address-cells", &addrc, sizeof(addrc)) == -1)
+		addrc = 2;
+	if (OF_getprop(bus, "#size-cells", &szc, sizeof(szc)) == -1)
+		szc = 1;
+	if (addrc < 2 || addrc > 3 || szc < 1 || szc > 2)
+		return (ENXIO);
+	if (strcmp(name, "pci") == 0) {
+		if (addrc > 3)
+			return (ENXIO);
+		nbank = OF_getprop(node, "assigned-addresses", &banks,
+		    sizeof(banks));
+	} else {
+		if (addrc > 2)
+			return (ENXIO);
+		nbank = OF_getprop(node, "reg", &banks, sizeof(banks));
+	}
+	if (nbank == -1)
+		return (ENXIO);
+	nbank /= sizeof(banks[0]) * (addrc + szc);
+	if (bank < 0 || bank > nbank - 1)
+		return (ENXIO);
+	phys = 0;
+	for (i = 0; i < MIN(2, addrc); i++)
+		phys |= (uint64_t)banks[(addrc + szc) * bank + addrc - 2 + i] <<
+		    32 * (MIN(2, addrc) - i - 1);
+	sz = 0;
+	for (i = 0; i < szc; i++)
+		sz |= (uint64_t)banks[(addrc + szc) * bank + addrc + i] <<
+		    32 * (szc - i - 1);
+	start = phys;
+	end = phys + sz - 1;
+	spc = phys_hi_mask_space(name, banks[(addrc + szc) * bank]);
+
+	/*
+	 * Map upward in the device tree at every bridge we encounter
+	 * using their "ranges" properties.
+	 * The "ranges" property of a bridge is an array of a structure
+	 * consisting of that bridge's #address-cells 32-bit child-phys
+	 * cells, its parent bridge #address-cells 32-bit parent-phys
+	 * cells and that bridge's #size-cells 32-bit size cells.
+	 * If a bridge doesn't have a "ranges" property no mapping is
+	 * necessary at that bridge.
+	 */
+	cspace = 0;
+	lbus = bus;
+	while ((pbus = OF_parent(bus)) != 0) {
+		if (OF_getprop(pbus, "#address-cells", &paddrc,
+		    sizeof(paddrc)) == -1)
+			paddrc = 2;
+		if (paddrc < 2 || paddrc > 3)
+			return (ENXIO);
+		nbank = OF_getprop(bus, "ranges", &banks, sizeof(banks));
+		if (nbank == -1) {
+			if (OF_getprop(pbus, "name", name, sizeof(name)) == -1)
+				return (ENXIO);
+			name[sizeof(name) - 1] = '\0';
+			goto skip;
+		}
+		if (lbus != bus) {
+			if (OF_getprop(bus, "#size-cells", &szc,
+			    sizeof(szc)) == -1)
+				szc = 1;
+			if (szc < 1 || szc > 2)
+				return (ENXIO);
+		}
+		nbank /= sizeof(banks[0]) * (addrc + paddrc + szc);
+		for (i = 0; i < nbank; i++) {
+			cspace = phys_hi_mask_space(name,
+			    banks[(addrc + paddrc + szc) * i]);
+			if (cspace != spc)
+				continue;
+			phys = 0;
+			for (j = 0; j < MIN(2, addrc); j++)
+				phys |= (uint64_t)banks[
+				    (addrc + paddrc + szc) * i +
+				    addrc - 2 + j] <<
+				    32 * (MIN(2, addrc) - j - 1);
+			sz = 0;
+			for (j = 0; j < szc; j++)
+				sz |= (uint64_t)banks[
+				    (addrc + paddrc + szc) * i + addrc +
+				    paddrc + j] <<
+				    32 * (szc - j - 1);
+			cstart = phys;
+			cend = phys + sz - 1;
+			if (start < cstart || start > cend)
+				continue;
+			if (end < cstart || end > cend)
+				return (ENXIO);
+			phys = 0;
+			for (j = 0; j < MIN(2, paddrc); j++)
+				phys |= (uint64_t)banks[
+				    (addrc + paddrc + szc) * i + addrc +
+				    paddrc - 2 + j] <<
+				    32 * (MIN(2, paddrc) - j - 1);
+			start += phys - cstart;
+			end += phys - cstart;
+			if (OF_getprop(pbus, "name", name, sizeof(name)) == -1)
+				return (ENXIO);
+			name[sizeof(name) - 1] = '\0';
+			spc = phys_hi_mask_space(name,
+			    banks[(addrc + paddrc + szc) * i + addrc]);
+			break;
+		}
+		if (i == nbank)
+			return (ENXIO);
+ skip:
+		addrc = paddrc;
+		lbus = bus;
+		bus = pbus;
+	}
+
+	/* Done with mapping. Return the bus space as used by FreeBSD. */
+	*addr = start;
+	if (OF_getprop(lbus, "name", name, sizeof(name)) == -1)
+		return (ENXIO);
+	name[sizeof(name) - 1] = '\0';
+	if (strcmp(name, "central") == 0) {
+		*space = UPA_BUS_SPACE;
+		return (0);
+	} else if (strcmp(name, "pci") == 0) {
+		switch (cspace) {
+		case OFW_PCI_PHYS_HI_SPACE_IO:
+			*space = PCI_IO_BUS_SPACE;
+			return (0);
+		case OFW_PCI_PHYS_HI_SPACE_MEM32:
+			*space = PCI_MEMORY_BUS_SPACE;
+			return (0);
+		}
+	} else if (strcmp(name, "sbus") == 0) {
+		*space = SBUS_BUS_SPACE;
+		return (0);
+	}
+	return (ENXIO);
+}
diff --git a/sys/sun4v/sun4v/pmap.c b/sys/sun4v/sun4v/pmap.c
new file mode 100644
index 000000000000..4e7283fc4836
--- /dev/null
+++ b/sys/sun4v/sun4v/pmap.c
@@ -0,0 +1,2002 @@
+/*-
+ * Copyright (c) 2006 Kip Macy
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include "opt_kstack_pages.h"
+#include "opt_msgbuf.h"
+#include "opt_pmap.h"
+#include "opt_trap_trace.h"
+
+#include <sys/param.h>
+#include <sys/kernel.h>
+#include <sys/kdb.h>
+#include <sys/ktr.h>
+#include <sys/lock.h>
+#include <sys/msgbuf.h>
+#include <sys/mutex.h>
+#include <sys/proc.h>
+#include <sys/smp.h>
+#include <sys/sched.h>
+#include <sys/sysctl.h>
+#include <sys/systm.h>
+#include <sys/vmmeter.h>
+
+#include <dev/ofw/openfirm.h>
+
+#include <vm/vm.h> 
+#include <vm/vm_page.h>
+#include <vm/vm_param.h>
+#include <vm/vm_kern.h>
+#include <vm/vm_map.h>
+#include <vm/vm_object.h>
+#include <vm/vm_extern.h>
+#include <vm/vm_pageout.h>
+#include <vm/vm_pager.h>
+#include <vm/uma.h>
+
+#include <machine/cpu.h>
+#include <machine/cache.h>
+#include <machine/frame.h>
+#include <machine/instr.h>
+#include <machine/md_var.h>
+#include <machine/metadata.h>
+#include <machine/ofw_mem.h>
+#include <machine/mmu.h>
+#include <machine/smp.h>
+#include <machine/tlb.h>
+#include <machine/tte.h>
+#include <machine/tte_hash.h>
+#include <machine/pcb.h>
+#include <machine/pstate.h>
+#include <machine/tsb.h>
+
+#include <machine/hypervisor_api.h>
+
+#ifdef TRAP_TRACING
+void trap_trace_report(int);
+#endif
+
+#if 1
+#define	PMAP_DEBUG
+#endif
+#ifndef	PMAP_SHPGPERPROC
+#define	PMAP_SHPGPERPROC	200
+#endif
+
+cache_enable_t *cache_enable;
+cache_flush_t *cache_flush;
+dcache_page_inval_t *dcache_page_inval;
+icache_page_inval_t *icache_page_inval;
+
+/*
+ * Virtual and physical address of message buffer.
+ */
+struct msgbuf *msgbufp;
+vm_paddr_t msgbuf_phys;
+
+/*
+ * Map of physical memory reagions.
+ */
+vm_paddr_t phys_avail[128];
+vm_paddr_t phys_avail_tmp[128];
+static struct ofw_mem_region mra[128];
+static struct ofw_map translations[128];
+static int translations_size;
+
+
+struct ofw_mem_region sparc64_memreg[128];
+int sparc64_nmemreg;
+
+extern vm_paddr_t mmu_fault_status_area;
+
+/*
+ * First and last available kernel virtual addresses.
+ */
+vm_offset_t virtual_avail;
+vm_offset_t virtual_end;
+vm_offset_t kernel_vm_end;
+vm_offset_t vm_max_kernel_address;
+
+#ifndef PMAP_SHPGPERPROC
+#define PMAP_SHPGPERPROC 200
+#endif
+/*
+ * Data for the pv entry allocation mechanism
+ */
+static uma_zone_t pvzone;
+static struct vm_object pvzone_obj;
+static int pv_entry_count = 0, pv_entry_max = 0, pv_entry_high_water = 0;
+int pmap_debug = 0;
+
+static struct mtx pmap_ctx_lock;
+static uint16_t ctx_stack[PMAP_CONTEXT_MAX];
+static int ctx_stack_top; 
+
+static int permanent_mappings = 0;
+static uint64_t nucleus_memory;
+static uint64_t nucleus_mappings[2];
+/*
+ * Kernel pmap.
+ */
+struct pmap kernel_pmap_store;
+
+hv_tsb_info_t kernel_td[MAX_TSB_INFO];
+
+/*
+ * This should be determined at boot time
+ * with tiny TLBS it doesn't make sense to try and selectively
+ * invalidate more than this 
+ */
+#define MAX_INVALIDATES   32
+#define MAX_TSB_CLEARS   128
+
+/*
+ * Allocate physical memory for use in pmap_bootstrap.
+ */
+static vm_paddr_t pmap_bootstrap_alloc(vm_size_t size);
+
+/*
+ * If user pmap is processed with pmap_remove and with pmap_remove and the
+ * resident count drops to 0, there are no more pages to remove, so we
+ * need not continue.
+ */
+#define	PMAP_REMOVE_DONE(pm) \
+	((pm) != kernel_pmap && (pm)->pm_stats.resident_count == 0)
+
+/*
+ * Kernel MMU interface
+ */
+
+#ifdef PMAP_DEBUG
+#define KDPRINTF if (pmap_debug) printf
+#define DPRINTF \
+	if (PCPU_GET(curpmap) && (PCPU_GET(curpmap)->pm_context != 0) && ((PCPU_GET(cpumask) & PCPU_GET(curpmap)->pm_active)) == 0) \
+   	panic("cpumask(0x%x) & active (0x%x) == 0 pid == %d\n",  \
+	      PCPU_GET(cpumask), PCPU_GET(curpmap)->pm_active, curthread->td_proc->p_pid); \
+if (pmap_debug) printf
+
+
+#else
+#define DPRINTF(...)
+#define KDPRINTF(...)
+#endif
+
+
+static void free_pv_entry(pv_entry_t pv);
+static pv_entry_t get_pv_entry(pmap_t locked_pmap);
+
+static void pmap_insert_entry(pmap_t pmap, vm_offset_t va, vm_page_t m);
+static void pmap_remove_entry(struct pmap *pmap, vm_page_t m, vm_offset_t va);
+static void pmap_remove_tte(pmap_t pmap, tte_t tte_data, vm_offset_t va);
+static void pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot);
+void pmap_set_ctx_panic(uint64_t error, vm_paddr_t tsb_ra, pmap_t pmap);
+
+
+/*
+ * Quick sort callout for comparing memory regions.
+ */
+static int mr_cmp(const void *a, const void *b);
+static int om_cmp(const void *a, const void *b);
+static int
+mr_cmp(const void *a, const void *b)
+{
+	const struct ofw_mem_region *mra;
+	const struct ofw_mem_region *mrb;
+
+	mra = a;
+	mrb = b;
+	if (mra->mr_start < mrb->mr_start)
+		return (-1);
+	else if (mra->mr_start > mrb->mr_start)
+		return (1);
+	else
+		return (0);
+}
+static int
+om_cmp(const void *a, const void *b)
+{
+	const struct ofw_map *oma;
+	const struct ofw_map *omb;
+
+	oma = a;
+	omb = b;
+	if (oma->om_start < omb->om_start)
+		return (-1);
+	else if (oma->om_start > omb->om_start)
+		return (1);
+	else
+		return (0);
+}
+
+static __inline void
+free_context(uint16_t ctx)
+{
+	mtx_lock_spin(&pmap_ctx_lock);
+	ctx_stack[ctx_stack_top++] = ctx;
+	mtx_unlock_spin(&pmap_ctx_lock);
+
+	KASSERT(ctx_stack_top < PMAP_CONTEXT_MAX, 
+		("context stack overrun - system error"));
+}
+
+static __inline uint16_t
+get_context(void)
+{
+	uint16_t ctx;
+
+	mtx_lock_spin(&pmap_ctx_lock);
+	ctx = ctx_stack[--ctx_stack_top];
+	mtx_unlock_spin(&pmap_ctx_lock);
+
+	KASSERT(ctx_stack_top > 0,
+		("context stack underrun - need to implement context stealing"));
+
+	return ctx;
+}
+
+static __inline void
+free_pv_entry(pv_entry_t pv)
+{
+	pv_entry_count--;
+	uma_zfree(pvzone, pv);
+}
+
+/*
+ * get a new pv_entry, allocating a block from the system
+ * when needed.
+ */
+static pv_entry_t
+get_pv_entry(pmap_t locked_pmap)
+{
+	static const struct timeval printinterval = { 60, 0 };
+	static struct timeval lastprint;
+	struct vpgqueues *vpq;
+	uint64_t tte_data;
+	pmap_t pmap;
+	pv_entry_t allocated_pv, next_pv, pv;
+	vm_offset_t va;
+	vm_page_t m;
+
+	PMAP_LOCK_ASSERT(locked_pmap, MA_OWNED);
+	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+	allocated_pv = uma_zalloc(pvzone, M_NOWAIT);
+	if (allocated_pv != NULL) {
+		pv_entry_count++;
+		if (pv_entry_count > pv_entry_high_water)
+			pagedaemon_wakeup();
+		else
+			return (allocated_pv);
+	}
+
+	/*
+	 * Reclaim pv entries: At first, destroy mappings to inactive
+	 * pages.  After that, if a pv entry is still needed, destroy
+	 * mappings to active pages.
+	 */
+	if (ratecheck(&lastprint, &printinterval))
+		printf("Approaching the limit on PV entries, "
+		    "increase the vm.pmap.shpgperproc tunable.\n");
+
+	vpq = &vm_page_queues[PQ_INACTIVE];
+retry:
+	sched_pin();
+	TAILQ_FOREACH(m, &vpq->pl, pageq) {
+		if (m->hold_count || m->busy || (m->flags & PG_BUSY))
+			continue;
+		TAILQ_FOREACH_SAFE(pv, &m->md.pv_list, pv_list, next_pv) {
+			va = pv->pv_va;
+			pmap = pv->pv_pmap;
+			/* Avoid deadlock and lock recursion. */
+			if (pmap > locked_pmap)
+				PMAP_LOCK(pmap);
+			else if (pmap != locked_pmap && !PMAP_TRYLOCK(pmap))
+				continue;
+			pmap->pm_stats.resident_count--;
+
+			tte_data = tte_hash_delete(pmap->pm_hash, va);
+
+			KASSERT((tte_data & VTD_WIRED) == 0,
+			    ("get_pv_entry: wired pte %#jx", (uintmax_t)tte_data));
+			if (tte_data & VTD_REF)
+				vm_page_flag_set(m, PG_REFERENCED);
+			if (tte_data & VTD_W) {
+				KASSERT((tte_data & VTD_SW_W),
+				("get_pv_entry: modified page not writable: va: %lx, tte: %lx",
+				    va, tte_data));
+				if (pmap_track_modified(locked_pmap, va))
+					vm_page_dirty(m);
+			}
+
+			pmap_invalidate_page(pmap, va, TRUE);
+			TAILQ_REMOVE(&pmap->pm_pvlist, pv, pv_plist);
+			TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
+			if (TAILQ_EMPTY(&m->md.pv_list))
+				vm_page_flag_clear(m, PG_WRITEABLE);
+			m->md.pv_list_count--;
+
+			if (pmap != locked_pmap)
+				PMAP_UNLOCK(pmap);
+			if (allocated_pv == NULL)
+				allocated_pv = pv;
+			else
+				free_pv_entry(pv);
+		}
+	}
+	sched_unpin();
+	if (allocated_pv == NULL) {
+		if (vpq == &vm_page_queues[PQ_INACTIVE]) {
+			vpq = &vm_page_queues[PQ_ACTIVE];
+			goto retry;
+		}
+		panic("get_pv_entry: increase the vm.pmap.shpgperproc tunable");
+	}
+	return (allocated_pv);
+}
+
+/*
+ * Allocate a physical page of memory directly from the phys_avail map.
+ * Can only be called from pmap_bootstrap before avail start and end are
+ * calculated.
+ */
+static vm_paddr_t
+pmap_bootstrap_alloc(vm_size_t size)
+{
+	vm_paddr_t pa;
+	int i;
+
+	size = round_page(size);
+
+	for (i = 0; phys_avail[i + 1] != 0; i += 2) {
+		if (phys_avail[i + 1] - phys_avail[i] < size)
+			continue;
+		pa = phys_avail[i];
+		phys_avail[i] += size;
+		pmap_scrub_pages(pa, size);
+		return (pa);
+	}
+	panic("pmap_bootstrap_alloc");
+}
+
+/*
+ * Activate a user pmap.  The pmap must be activated before its address space
+ * can be accessed in any way.
+ */
+void
+pmap_activate(struct thread *td)
+{
+	pmap_t pmap, oldpmap;
+	int err;
+	
+	critical_enter();
+	pmap = vmspace_pmap(td->td_proc->p_vmspace);
+	oldpmap = PCPU_GET(curpmap);
+#if defined(SMP)
+	atomic_clear_int(&oldpmap->pm_active, PCPU_GET(cpumask));
+	atomic_set_int(&pmap->pm_tlbactive, PCPU_GET(cpumask));
+	atomic_set_int(&pmap->pm_active, PCPU_GET(cpumask));
+#else
+	oldpmap->pm_active &= ~1;
+	pmap->pm_active |= 1;
+	pmap->pm_tlbactive |= 1;
+#endif
+
+	pmap->pm_hashscratch = tte_hash_set_scratchpad_user(pmap->pm_hash, pmap->pm_context);
+	pmap->pm_tsbscratch = tsb_set_scratchpad_user(&pmap->pm_tsb);
+	PCPU_SET(curpmap, pmap);
+	if (pmap->pm_context != 0)
+		if ((err = hv_set_ctxnon0(1, pmap->pm_tsb_ra)) != H_EOK)
+			panic("failed to set TSB 0x%lx - context == %ld\n", 
+			      pmap->pm_tsb_ra, pmap->pm_context);
+	stxa(MMU_CID_S, ASI_MMU_CONTEXTID, pmap->pm_context);
+	membar(Sync);
+	critical_exit();
+}
+
+vm_offset_t 
+pmap_addr_hint(vm_object_t object, vm_offset_t va, vm_size_t size)
+{
+	return (va);
+}
+
+/*
+ * Bootstrap the system enough to run with virtual memory.
+ */
+void
+pmap_bootstrap(vm_offset_t ekva)
+{
+	struct pmap *pm;
+	vm_offset_t off, va;
+	vm_paddr_t pa, kernel_hash_pa, phys_avail_start, nucleus_memory_start;
+	vm_size_t physsz, virtsz, kernel_hash_size;
+	ihandle_t pmem, vmem;
+	int i, sz, j;
+	uint64_t tsb_8k_size, tsb_4m_size, error, physmem_tunable;
+
+
+
+	if ((vmem = OF_finddevice("/virtual-memory")) == -1)
+		panic("pmap_bootstrap: finddevice /virtual-memory");
+	if ((sz = OF_getproplen(vmem, "translations")) == -1)
+		panic("pmap_bootstrap: getproplen translations");
+	if (sizeof(translations) < sz)
+		panic("pmap_bootstrap: translations too small");
+	bzero(translations, sz);
+	if (OF_getprop(vmem, "translations", translations, sz) == -1)
+		panic("pmap_bootstrap: getprop /virtual-memory/translations");
+	sz /= sizeof(*translations);
+	translations_size = sz;
+	nucleus_memory_start = 0;
+	CTR0(KTR_PMAP, "pmap_bootstrap: translations");
+	qsort(translations, sz, sizeof (*translations), om_cmp);
+	for (i = 0; i < sz; i++) {
+		KDPRINTF("om_size=%ld om_start=%lx om_tte=%lx\n", 
+			translations[i].om_size, translations[i].om_start, 
+			translations[i].om_tte);
+		if (translations[i].om_size == PAGE_SIZE_4M && 
+		    (translations[i].om_start == KERNBASE ||
+		     translations[i].om_start == KERNBASE + PAGE_SIZE_4M)) {
+			KDPRINTF("mapping permanent translation\n");
+			pa = TTE_GET_PA(translations[i].om_tte);
+			error = hv_mmu_map_perm_addr((char *)translations[i].om_start, 
+				KCONTEXT, pa | TTE_KERNEL | VTD_4M, MAP_ITLB | MAP_DTLB);
+			if (error != H_EOK)
+				panic("map_perm_addr returned error=%ld", error);
+			
+			if ((nucleus_memory_start == 0) || (pa < nucleus_memory_start))
+				nucleus_memory_start = pa;
+			nucleus_mappings[permanent_mappings++] = pa;
+			nucleus_memory += PAGE_SIZE_4M;
+#ifdef SMP
+			mp_add_nucleus_mapping(translations[i].om_start, 
+					       pa | TTE_KERNEL | VTD_4M);
+#endif
+		}  
+	}
+
+	/*
+	 * Find out what physical memory is available from the prom and
+	 * initialize the phys_avail array.  This must be done before
+	 * pmap_bootstrap_alloc is called.
+	 */
+	if ((pmem = OF_finddevice("/memory")) == -1)
+		panic("pmap_bootstrap: finddevice /memory");
+	if ((sz = OF_getproplen(pmem, "available")) == -1)
+		panic("pmap_bootstrap: getproplen /memory/available");
+	if (sizeof(vm_paddr_t)*128 < sz) /* FIXME */
+		panic("pmap_bootstrap: phys_avail too small");
+	if (sizeof(mra) < sz)
+		panic("pmap_bootstrap: mra too small");
+	bzero(mra, sz);
+	if (OF_getprop(pmem, "available", mra, sz) == -1)
+		panic("pmap_bootstrap: getprop /memory/available");
+
+	sz /= sizeof(*mra);
+	CTR0(KTR_PMAP, "pmap_bootstrap: physical memory");
+
+	qsort(mra, sz, sizeof (*mra), mr_cmp);
+	physsz = 0;
+
+        if (TUNABLE_ULONG_FETCH("hw.physmem", &physmem_tunable))
+                physmem = atop(physmem_tunable);
+
+	KDPRINTF("desired physmem=0x%lx\n", physmem_tunable);
+	for (i = 0, j = 0; i < sz; i++) {
+		vm_paddr_t start = mra[i].mr_start;
+		uint64_t size = mra[i].mr_size;
+		CTR2(KTR_PMAP, "start=%#lx size=%#lx\n", mra[i].mr_start, mra[i].mr_size);
+		KDPRINTF("start=%#lx size=%#lx\n", mra[i].mr_start, mra[i].mr_size);
+		if (nucleus_memory_start == mra[i].mr_start) {
+			mra[i].mr_start += 2*PAGE_SIZE_4M;
+			mra[i].mr_size -= 2*PAGE_SIZE_4M;
+		}
+		if (nucleus_memory_start == (start + size - 2*PAGE_SIZE_4M)) 
+			mra[i].mr_size -= 2*PAGE_SIZE_4M;
+
+		if ((nucleus_memory_start > start) && (nucleus_memory_start < (start + size))) {
+			uint64_t firstsize = (nucleus_memory_start - start);
+			phys_avail[j] = start;
+			if ((physmem_tunable != 0) && ((physsz + firstsize) > physmem_tunable)) {
+				phys_avail[j+1] = start + (physmem_tunable - physsz);
+				physsz = physmem_tunable;
+				break;
+			}
+			phys_avail[j+1] = nucleus_memory_start;
+			size = size - firstsize - 2*PAGE_SIZE_4M;
+			mra[i].mr_start =  nucleus_memory_start + 2*PAGE_SIZE_4M;
+			mra[i].mr_size = size;
+			physsz += firstsize + 2*PAGE_SIZE_4M;
+			j += 2;
+		}
+		if (mra[i].mr_size < PAGE_SIZE_4M)
+			continue;
+		if ((mra[i].mr_start & PAGE_MASK_4M) && (mra[i].mr_size < 2*PAGE_SIZE_4M))
+			continue;
+		if (mra[i].mr_start & PAGE_MASK_4M) {
+			uint64_t newstart, roundup;
+			newstart = ((mra[i].mr_start + (PAGE_SIZE_4M-1)) & ~PAGE_MASK_4M);
+			roundup = newstart - mra[i].mr_start;
+			mra[i].mr_size -= roundup;
+			mra[i].mr_start = newstart;
+		}
+		mra[i].mr_size &= ~PAGE_MASK_4M;
+		phys_avail[j] = mra[i].mr_start;
+		if (physmem_tunable != 0 && ((physsz + mra[i].mr_size) >= physmem_tunable)) {
+			size = physmem_tunable - physsz;
+			phys_avail[j + 1] = mra[i].mr_start + size;
+			physsz = physmem_tunable;
+			break;
+		}
+		phys_avail[j + 1] = mra[i].mr_start + mra[i].mr_size;
+		physsz += mra[i].mr_size;
+		j += 2;
+	}
+	phys_avail_start = phys_avail[0];
+	physmem = btoc(physsz);
+	
+
+	for (i = 0; phys_avail[i] != 0; i += 2)
+		KDPRINTF("phys_avail[%d]=0x%lx phys_avail[%d]=0x%lx\n",
+			i, phys_avail[i], i+1, phys_avail[i+1]);
+	/*
+	 * Calculate the size of kernel virtual memory, and the size and mask
+	 * for the kernel tsb.
+	 */
+	virtsz = roundup(physsz, PAGE_SIZE_4M << (PAGE_SHIFT - TTE_SHIFT));
+	vm_max_kernel_address = VM_MIN_KERNEL_ADDRESS + virtsz;
+
+	/*
+	 * Set the start and end of kva.  The kernel is loaded at the first
+	 * available 4 meg super page, so round up to the end of the page.
+	 */
+	virtual_avail = roundup2(ekva, PAGE_SIZE_4M);
+	virtual_end = vm_max_kernel_address;
+	kernel_vm_end = vm_max_kernel_address;
+
+	/*
+	 * Allocate and map a 4MB page for the kernel hashtable 
+	 *
+	 */
+#ifndef SIMULATOR
+	kernel_hash_size = PAGE_SIZE_4M*2;
+#else
+	kernel_hash_size = PAGE_SIZE_8K*64;
+#endif
+
+	kernel_hash_pa = pmap_bootstrap_alloc(kernel_hash_size);
+	if (kernel_hash_pa & PAGE_MASK_4M)
+		panic("pmap_bootstrap: hashtable pa unaligned\n");
+	/*
+	 * Set up TSB descriptors for the hypervisor
+	 *
+	 */
+#ifdef notyet
+	tsb_8k_size = virtsz >> (PAGE_SHIFT - TTE_SHIFT);
+#else
+	/* avoid alignment complaints from the hypervisor */
+	tsb_8k_size = PAGE_SIZE_4M;
+#endif
+
+	pa = pmap_bootstrap_alloc(tsb_8k_size);
+	if (pa & PAGE_MASK_4M)
+		panic("pmap_bootstrap: tsb unaligned\n");
+	KDPRINTF("tsb_8k_size is 0x%lx, tsb_8k_pa is 0x%lx\n", tsb_8k_size, pa);
+	kernel_td[TSB8K_INDEX].hvtsb_idxpgsz = TTE8K;
+	kernel_td[TSB8K_INDEX].hvtsb_assoc = 1;
+	kernel_td[TSB8K_INDEX].hvtsb_ntte = (tsb_8k_size >> TTE_SHIFT);
+	kernel_td[TSB8K_INDEX].hvtsb_ctx_index = 0;
+	kernel_td[TSB8K_INDEX].hvtsb_pgszs = TSB8K;
+	kernel_td[TSB8K_INDEX].hvtsb_rsvd = 0;
+	kernel_td[TSB8K_INDEX].hvtsb_pa = pa;
+
+	/*
+	 * Initialize kernel's private TSB from 8K page TSB
+	 *
+	 */
+	kernel_pmap->pm_tsb.hvtsb_idxpgsz = TTE8K;
+	kernel_pmap->pm_tsb.hvtsb_assoc = 1;
+	kernel_pmap->pm_tsb.hvtsb_ntte = (tsb_8k_size >> TTE_SHIFT);
+	kernel_pmap->pm_tsb.hvtsb_ctx_index = 0;
+	kernel_pmap->pm_tsb.hvtsb_pgszs = TSB8K;
+	kernel_pmap->pm_tsb.hvtsb_rsvd = 0;
+	kernel_pmap->pm_tsb.hvtsb_pa = pa;
+	
+	kernel_pmap->pm_tsb_ra = vtophys((vm_offset_t)&kernel_pmap->pm_tsb);
+	tsb_set_scratchpad_kernel(&kernel_pmap->pm_tsb);
+	
+	/*
+	 * Initialize kernel TSB for 4M pages
+	 * currently (not by design) used for permanent mappings
+	 */
+	
+	tsb_4m_size = (virtsz >> (PAGE_SHIFT_4M - TTE_SHIFT)) << 3;
+	pa = pmap_bootstrap_alloc(tsb_4m_size);
+
+	KDPRINTF("tsb_4m_pa is 0x%lx tsb_4m_size is 0x%lx\n", pa, tsb_4m_size);
+	kernel_td[TSB4M_INDEX].hvtsb_idxpgsz = TTE4M;
+	kernel_td[TSB4M_INDEX].hvtsb_assoc = 1;
+	kernel_td[TSB4M_INDEX].hvtsb_ntte = (tsb_4m_size >> TTE_SHIFT);
+	kernel_td[TSB4M_INDEX].hvtsb_ctx_index = 0;
+	kernel_td[TSB4M_INDEX].hvtsb_pgszs = TSB4M; 
+	kernel_td[TSB4M_INDEX].hvtsb_rsvd = 0;
+	kernel_td[TSB4M_INDEX].hvtsb_pa = pa;
+
+	/*
+	 * allocate MMU fault status areas for all CPUS
+	 */
+	mmu_fault_status_area = pmap_bootstrap_alloc(MMFSA_SIZE*MAXCPU);
+
+	/*
+	 * Allocate and map the message buffer.
+	 */
+	msgbuf_phys = pmap_bootstrap_alloc(MSGBUF_SIZE);
+	msgbufp = (struct msgbuf *)TLB_PHYS_TO_DIRECT(msgbuf_phys);
+
+	/*
+	 * Allocate a kernel stack with guard page for thread0 and map it into
+	 * the kernel tsb.  
+	 */
+	pa = pmap_bootstrap_alloc(KSTACK_PAGES*PAGE_SIZE);
+	kstack0_phys = pa;
+	virtual_avail += KSTACK_GUARD_PAGES * PAGE_SIZE;
+	kstack0 = virtual_avail;
+	virtual_avail += KSTACK_PAGES * PAGE_SIZE;
+	for (i = 0; i < KSTACK_PAGES; i++) {
+		pa = kstack0_phys + i * PAGE_SIZE;
+		va = kstack0 + i * PAGE_SIZE;
+		tsb_set_tte_real(&kernel_td[TSB8K_INDEX], va,
+			    pa | TTE_KERNEL | VTD_8K, 0);
+	}
+	/*
+	 * Calculate the last available physical address.
+	 */
+	for (i = 0; phys_avail[i + 2] != 0; i += 2)
+		KDPRINTF("phys_avail[%d]=0x%lx phys_avail[%d]=0x%lx\n",
+			i, phys_avail[i], i+1, phys_avail[i+1]);
+	KDPRINTF("phys_avail[%d]=0x%lx phys_avail[%d]=0x%lx\n",
+			i, phys_avail[i], i+1, phys_avail[i+1]);
+
+	Maxmem = sparc64_btop(phys_avail[i + 1]);
+	
+	/*
+	 * Add the prom mappings to the kernel tsb.
+	 */
+	for (i = 0; i < sz; i++) {
+		CTR3(KTR_PMAP,
+		    "translation: start=%#lx size=%#lx tte=%#lx",
+		    translations[i].om_start, translations[i].om_size,
+		    translations[i].om_tte);
+		KDPRINTF("om_size=%ld om_start=%lx om_tte=%lx\n", 
+		       translations[i].om_size, translations[i].om_start, 
+		       translations[i].om_tte);
+
+		if (translations[i].om_start < VM_MIN_PROM_ADDRESS ||
+		    translations[i].om_start > VM_MAX_PROM_ADDRESS) 
+			continue;
+
+		for (off = 0; off < translations[i].om_size;
+		     off += PAGE_SIZE) {
+			va = translations[i].om_start + off;
+			pa = TTE_GET_PA(translations[i].om_tte) + off;
+			tsb_assert_invalid(&kernel_td[TSB8K_INDEX], va);
+			tsb_set_tte_real(&kernel_td[TSB8K_INDEX], va, pa | 
+				    TTE_KERNEL | VTD_8K, 0);
+		}
+	}
+
+	error = hv_set_ctx0(MAX_TSB_INFO, vtophys((vm_offset_t)&kernel_td));
+	if (error != H_EOK)
+		panic("failed to set ctx0 TSBs error: %ld", error);
+
+#ifdef SMP
+	mp_set_tsb_desc_ra(vtophys((vm_offset_t)&kernel_td));
+#endif
+	/*
+	 * setup direct mappings
+	 * 
+	 */
+	i = 0;
+	pa = phys_avail_start;
+	do {
+		for (; pa < phys_avail[i + 1]; pa += PAGE_SIZE_4M) {
+			tsb_assert_invalid(&kernel_td[TSB4M_INDEX], TLB_PHYS_TO_DIRECT(pa));
+			tsb_set_tte_real(&kernel_td[TSB4M_INDEX], TLB_PHYS_TO_DIRECT(pa), 
+					 pa | TTE_KERNEL | VTD_4M, 0);
+				
+		}
+		i += 2;
+		pa = phys_avail[i];
+	} while (pa != 0);
+
+	for (i = 0; i < 2; i++) {
+                pa = nucleus_mappings[i];
+		tsb_assert_invalid(&kernel_td[TSB4M_INDEX], TLB_PHYS_TO_DIRECT(pa));
+                tsb_set_tte_real(&kernel_td[TSB4M_INDEX], TLB_PHYS_TO_DIRECT(pa),
+				 pa | TTE_KERNEL | VTD_4M, 0);
+	}
+	/*
+	 * Get the available physical memory ranges from /memory/reg. These
+	 * are only used for kernel dumps, but it may not be wise to do prom
+	 * calls in that situation.
+	 */
+	if ((sz = OF_getproplen(pmem, "reg")) == -1)
+		panic("pmap_bootstrap: getproplen /memory/reg");
+	if (sizeof(sparc64_memreg) < sz)
+		panic("pmap_bootstrap: sparc64_memreg too small");
+	if (OF_getprop(pmem, "reg", sparc64_memreg, sz) == -1)
+		panic("pmap_bootstrap: getprop /memory/reg");
+	sparc64_nmemreg = sz / sizeof(*sparc64_memreg);
+
+	pm = kernel_pmap;
+	pm->pm_active = ~0;
+	pm->pm_tlbactive = ~0;
+
+	PMAP_LOCK_INIT(kernel_pmap);
+
+	TAILQ_INIT(&kernel_pmap->pm_pvlist);
+
+	/* 
+	 * This could happen earlier - but I put it here to avoid 
+	 * attempts to do updates until they're legal
+	 */
+	pm->pm_hash = tte_hash_kernel_create(TLB_PHYS_TO_DIRECT(kernel_hash_pa), kernel_hash_size, pmap_bootstrap_alloc(PAGE_SIZE));
+	pm->pm_hashscratch = tte_hash_set_scratchpad_kernel(pm->pm_hash);
+
+	for (i = 0; i < translations_size; i++) {
+		KDPRINTF("om_size=%ld om_start=%lx om_tte=%lx\n", 
+		       translations[i].om_size, translations[i].om_start, 
+		       translations[i].om_tte);
+
+		if (translations[i].om_start < VM_MIN_PROM_ADDRESS ||
+		    translations[i].om_start > VM_MAX_PROM_ADDRESS) {
+			KDPRINTF("skipping\n");
+			continue;
+		}
+		for (off = 0; off < translations[i].om_size; off += PAGE_SIZE) {
+			va = translations[i].om_start + off;
+			pa = TTE_GET_PA(translations[i].om_tte) + off;
+			tte_hash_insert(pm->pm_hash, va, pa | TTE_KERNEL | VTD_8K);
+		}
+		KDPRINTF("set om_size=%ld om_start=%lx om_tte=%lx\n", 
+		       translations[i].om_size, translations[i].om_start, 
+		       translations[i].om_tte);
+	}
+	for (i = 0; i < KSTACK_PAGES; i++) {
+		pa = kstack0_phys + i * PAGE_SIZE;
+		va = kstack0 + i * PAGE_SIZE;
+		tte_hash_insert(pm->pm_hash, va, pa | TTE_KERNEL | VTD_8K);
+	}
+	/*
+	 * Add direct mappings to hash
+	 *
+	 */
+#ifdef notyet
+	/* hash only supports 8k pages */
+	for (pa = PAGE_SIZE_4M; pa < phys_avail[2]; pa += PAGE_SIZE_4M)
+		tte_hash_insert(pm->pm_hash, TLB_PHYS_TO_DIRECT(pa), 
+				pa | TTE_KERNEL | VTD_4M);
+#endif
+        /* XXX relies on the fact that memory ranges only get smaller */
+        for (i = 0; phys_avail[i + 2] != 0; i += 2)
+                if (phys_avail[i + 1] - phys_avail[i] < PAGE_SIZE_4M)
+                        phys_avail[i] = phys_avail[i+1] = 0;
+
+}
+
+
+
+/*
+ *	Routine:	pmap_change_wiring
+ *	Function:	Change the wiring attribute for a map/virtual-address
+ *			pair.
+ *	In/out conditions:
+ *			The mapping must already exist in the pmap.
+ */
+void
+pmap_change_wiring(pmap_t pmap, vm_offset_t va, boolean_t wired)
+{
+	boolean_t iswired;
+	PMAP_LOCK(pmap);
+	iswired = tte_get_virt_bit(pmap, va, VTD_WIRED);
+
+	if (wired && !iswired) {
+		pmap->pm_stats.wired_count++;
+		tte_set_virt_bit(pmap, va, VTD_WIRED);
+	} else if (!wired && iswired) {
+		pmap->pm_stats.wired_count--;
+		tte_clear_virt_bit(pmap, va, VTD_WIRED);
+	}
+	PMAP_UNLOCK(pmap);
+}
+
+void
+pmap_clear_modify(vm_page_t m)
+{
+	KDPRINTF("pmap_clear_modify(0x%lx)\n", VM_PAGE_TO_PHYS(m));
+	tte_clear_phys_bit(m, VTD_W);
+}
+
+void
+pmap_clear_reference(vm_page_t m)
+{
+	KDPRINTF("pmap_clear_reference(0x%lx)\n", VM_PAGE_TO_PHYS(m));
+	tte_clear_phys_bit(m, VTD_REF);
+}
+
+void
+pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr,
+	  vm_size_t len, vm_offset_t src_addr)
+{
+	vm_offset_t addr, end_addr;
+
+	end_addr = src_addr + len;
+	
+	/*
+	 * Don't let optional prefaulting of pages make us go
+	 * way below the low water mark of free pages or way
+	 * above high water mark of used pv entries.
+	 */
+	if (cnt.v_free_count < cnt.v_free_reserved ||
+	    pv_entry_count > pv_entry_high_water)
+		return;
+	
+
+	vm_page_lock_queues();
+	if (dst_pmap < src_pmap) {
+		PMAP_LOCK(dst_pmap);
+		PMAP_LOCK(src_pmap);
+	} else {
+		PMAP_LOCK(src_pmap);
+		PMAP_LOCK(dst_pmap);
+	}
+	sched_pin();
+	for (addr = src_addr; addr < end_addr; addr += PAGE_SIZE) {
+		tte_t tte_data;
+		vm_page_t m;
+
+		tte_data = tte_hash_lookup(src_pmap->pm_hash, addr);
+
+		if ((tte_data & VTD_MANAGED) != 0) {
+			if (tte_hash_lookup(dst_pmap->pm_hash, addr) == 0) {
+				m = PHYS_TO_VM_PAGE(TTE_GET_PA(tte_data));
+
+				tte_hash_insert(dst_pmap->pm_hash, addr, tte_data & ~(VTD_W|VTD_REF));
+				dst_pmap->pm_stats.resident_count++;
+				pmap_insert_entry(dst_pmap, addr, m);
+			} 
+		}		
+	}
+	
+	sched_unpin();
+	vm_page_unlock_queues();
+	PMAP_UNLOCK(src_pmap);
+	PMAP_UNLOCK(dst_pmap);
+
+}
+
+void
+pmap_copy_page(vm_page_t src, vm_page_t dst)
+{
+	vm_paddr_t srcpa, dstpa;
+	srcpa = VM_PAGE_TO_PHYS(src);
+	dstpa = VM_PAGE_TO_PHYS(dst);
+
+	novbcopy((char *)TLB_PHYS_TO_DIRECT(srcpa), (char *)TLB_PHYS_TO_DIRECT(dstpa), PAGE_SIZE);
+
+
+}
+
+static __inline void
+pmap_add_tte(pmap_t pmap, vm_offset_t va, vm_page_t m, tte_t *tte_data, int wired)
+{
+
+	if (wired)
+		pmap->pm_stats.wired_count++;
+	
+	if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0) {
+		pmap_insert_entry(pmap, va, m);
+		*tte_data |= VTD_MANAGED;
+	}
+}
+
+/*
+ * Map the given physical page at the specified virtual address in the
+ * target pmap with the protection requested.  If specified the page
+ * will be wired down.
+ */
+void
+pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,
+	   boolean_t wired)
+{
+	vm_paddr_t pa, opa;
+	uint64_t tte_data, otte_data;
+	vm_page_t om;
+	int invlva;
+
+	if (pmap->pm_context)
+		DPRINTF("pmap_enter(va=%lx, pa=0x%lx, prot=%x)\n", va, 
+			VM_PAGE_TO_PHYS(m), prot);
+
+	om = NULL;
+	
+	vm_page_lock_queues();
+	PMAP_LOCK(pmap);
+	sched_pin();
+
+	tte_data = pa = VM_PAGE_TO_PHYS(m);
+	otte_data = tte_hash_delete(pmap->pm_hash, va);
+	opa = TTE_GET_PA(otte_data);
+
+	/*
+	 * Mapping has not changed, must be protection or wiring change.
+	 */
+
+	if (opa == 0) {
+		pmap->pm_stats.resident_count++;
+		pmap_add_tte(pmap, va, m, &tte_data, wired);
+
+	} else if (pa != opa) {
+		/*
+		 * Mapping has changed, invalidate old range and fall through to
+		 * handle validating new mapping.
+		 */
+		if (otte_data & VTD_WIRED)
+			pmap->pm_stats.wired_count--;
+
+		if (otte_data & VTD_MANAGED) {
+			om = PHYS_TO_VM_PAGE(opa);
+			pmap_remove_entry(pmap, om, va);
+		}
+
+		pmap_add_tte(pmap, va, m, &tte_data, wired);
+
+	} else /* (pa == opa) */ {
+		/*
+		 * Wiring change, just update stats. We don't worry about
+		 * wiring PT pages as they remain resident as long as there
+		 * are valid mappings in them. Hence, if a user page is wired,
+		 * the PT page will be also.
+		 */
+		if (wired && ((otte_data & VTD_WIRED) == 0))
+			pmap->pm_stats.wired_count++;
+		else if (!wired && (otte_data & VTD_WIRED))
+			pmap->pm_stats.wired_count--;
+
+		/*
+		 * We might be turning off write access to the page,
+		 * so we go ahead and sense modify status.
+		 */
+		if (otte_data & VTD_MANAGED) {
+			om = m;
+			tte_data |= VTD_MANAGED;
+		}
+	} 
+
+	/*
+	 * Now validate mapping with desired protection/wiring.
+	 */
+	if ((prot & VM_PROT_WRITE) != 0)
+		tte_data |= VTD_SW_W; 
+	if ((prot & VM_PROT_EXECUTE) != 0)
+		tte_data |= VTD_X;
+	if (wired)
+		tte_data |= VTD_WIRED;
+	if (pmap == kernel_pmap)
+		tte_data |= TTE_KERNEL_MINFLAGS;
+
+
+	tte_hash_insert(pmap->pm_hash, va, tte_data|TTE_MINFLAGS|VTD_REF);
+	tsb_set_tte(&pmap->pm_tsb, va, tte_data|TTE_MINFLAGS|VTD_REF, pmap->pm_context);
+	
+	invlva = FALSE;
+	if ((otte_data & ~(VTD_W|VTD_REF)) != tte_data) {
+		if (otte_data & VTD_V) {
+			if (otte_data & VTD_REF) {
+				if (otte_data & VTD_MANAGED)
+					vm_page_flag_set(om, PG_REFERENCED);
+				if (opa != pa)
+					invlva = TRUE;
+			}
+			if (otte_data & VTD_W) {
+				if ((otte_data & VTD_MANAGED) &&
+				    pmap_track_modified(pmap, va))
+					vm_page_dirty(om);
+#if 0
+				if ((prot & VM_PROT_WRITE) == 0) /* XXX double check */
+#endif
+					invlva = TRUE;
+			}
+		}
+	} 
+	if (invlva)
+		pmap_invalidate_page(pmap, va, TRUE);
+
+	sched_unpin();
+	PMAP_UNLOCK(pmap);
+	vm_page_unlock_queues();
+
+	
+}
+
+/*
+ * Maps a sequence of resident pages belonging to the same object.
+ * The sequence begins with the given page m_start.  This page is
+ * mapped at the given virtual address start.  Each subsequent page is
+ * mapped at a virtual address that is offset from start by the same
+ * amount as the page is offset from m_start within the object.  The
+ * last page in the sequence is the page with the largest offset from
+ * m_start that can be mapped at a virtual address less than the given
+ * virtual address end.  Not every virtual page between start and end
+ * is mapped; only those for which a resident page exists with the
+ * corresponding offset from m_start are mapped.
+ */
+void
+pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end, 
+		  vm_page_t m_start, vm_prot_t prot)
+{
+	vm_page_t m, mpte;
+        vm_pindex_t diff, psize;
+
+        VM_OBJECT_LOCK_ASSERT(m_start->object, MA_OWNED);
+        psize = atop(end - start);
+        mpte = NULL;
+        m = m_start;
+        PMAP_LOCK(pmap);
+        while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) {
+		pmap_enter_quick_locked(pmap, start + ptoa(diff), m,
+					       prot);
+                m = TAILQ_NEXT(m, listq);
+        }
+        PMAP_UNLOCK(pmap);
+}
+
+void
+pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot)
+{
+	PMAP_LOCK(pmap);
+	pmap_enter_quick_locked(pmap, va, m, prot);
+	PMAP_UNLOCK(pmap);
+
+}
+
+static void
+pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot)
+{
+
+	tte_t tte_data;
+
+	if (pmap->pm_context)
+		KDPRINTF("pmap_enter_quick(ctx=0x%lx va=%lx, pa=0x%lx prot=%x)\n", 
+			pmap->pm_context, va, VM_PAGE_TO_PHYS(m), prot);
+
+        PMAP_LOCK_ASSERT(pmap, MA_OWNED);
+	if (tte_hash_lookup(pmap->pm_hash, va))
+		return;
+		
+	tte_data = VM_PAGE_TO_PHYS(m);
+	/*
+	 * Enter on the PV list if part of our managed memory. Note that we
+	 * raise IPL while manipulating pv_table since pmap_enter can be
+	 * called at interrupt time.
+	 */
+	if ((m->flags & (PG_FICTITIOUS|PG_UNMANAGED)) == 0) {
+		pmap_insert_entry(pmap, va, m);
+		tte_data |= VTD_MANAGED;
+	}
+
+	pmap->pm_stats.resident_count++;
+
+	if ((prot & VM_PROT_EXECUTE) != 0)
+		tte_data |= VTD_X;
+
+	if (pmap == kernel_pmap)
+		tte_data |= TTE_KERNEL_MINFLAGS;
+
+	tte_hash_insert(pmap->pm_hash, va, tte_data | TTE_MINFLAGS);
+
+}
+
+/*
+ * Extract the physical page address associated with the given
+ * map/virtual_address pair.
+ */
+vm_paddr_t
+pmap_extract(pmap_t pmap, vm_offset_t va)
+{
+	vm_paddr_t pa;
+	tte_t tte_data;
+
+	tte_data = tte_hash_lookup(pmap->pm_hash, va);
+	pa = TTE_GET_PA(tte_data) | (va & TTE_GET_PAGE_MASK(tte_data));
+
+	return (pa);
+}
+
+/*
+ * Atomically extract and hold the physical page with the given
+ * pmap and virtual address pair if that mapping permits the given
+ * protection.
+ */
+vm_page_t
+pmap_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot)
+{
+	tte_t tte_data;
+	vm_page_t m;
+
+	m = NULL;
+	vm_page_lock_queues();
+	PMAP_LOCK(pmap);
+	sched_pin();
+	tte_data = tte_hash_lookup(pmap->pm_hash, va);
+	if (tte_data != 0 && 
+	    ((tte_data & VTD_SW_W) || (prot & VM_PROT_WRITE) == 0)) {
+		m = PHYS_TO_VM_PAGE(TTE_GET_PA(tte_data));
+		vm_page_hold(m);
+	}
+	sched_unpin();
+	vm_page_unlock_queues();
+	PMAP_UNLOCK(pmap);
+
+	return (m);
+}
+
+void 
+pmap_growkernel(vm_offset_t addr)
+{
+	return;
+}
+
+void 
+pmap_init(void)
+{
+
+	/* allocate pv_entry zones */
+	int shpgperproc = PMAP_SHPGPERPROC;
+
+	for (ctx_stack_top = 1; ctx_stack_top < PMAP_CONTEXT_MAX; ctx_stack_top++) 
+		ctx_stack[ctx_stack_top] = ctx_stack_top;
+
+	mtx_init(&pmap_ctx_lock, "ctx lock", NULL, MTX_SPIN);
+
+	/*
+	 * Initialize the address space (zone) for the pv entries.  Set a
+	 * high water mark so that the system can recover from excessive
+	 * numbers of pv entries.
+	 */
+	pvzone = uma_zcreate("PV ENTRY", sizeof(struct pv_entry), NULL, NULL, 
+	    NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM | UMA_ZONE_NOFREE);
+	TUNABLE_INT_FETCH("vm.pmap.shpgperproc", &shpgperproc);
+	pv_entry_max = shpgperproc * maxproc + cnt.v_page_count;
+	TUNABLE_INT_FETCH("vm.pmap.pv_entries", &pv_entry_max);
+	pv_entry_high_water = 9 * (pv_entry_max / 10);
+	uma_zone_set_obj(pvzone, &pvzone_obj, pv_entry_max);
+
+	tte_hash_init();
+
+}
+
+/*
+ * Create a pv entry for page at pa for
+ * (pmap, va).
+ */
+static void
+pmap_insert_entry(pmap_t pmap, vm_offset_t va, vm_page_t m)
+{
+	pv_entry_t pv;
+
+	KDPRINTF("pmap_insert_entry(va=0x%lx, pa=0x%lx)\n", va, VM_PAGE_TO_PHYS(m));
+	pv = get_pv_entry(pmap);
+	pv->pv_va = va;
+	pv->pv_pmap = pmap;
+
+	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
+	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+	TAILQ_INSERT_TAIL(&pmap->pm_pvlist, pv, pv_plist);
+	TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list);
+	m->md.pv_list_count++;
+}
+
+#ifdef TRAP_TRACING
+static int trap_trace_report_done;
+#endif
+
+#ifdef SMP
+static cpumask_t
+pmap_ipi(pmap_t pmap, char *func, uint64_t arg1, uint64_t arg2)
+{
+
+	int i, cpu_count, retried;
+	u_int cpus;
+	cpumask_t cpumask, active, curactive;
+	cpumask_t active_total, ackmask;
+	uint16_t *cpulist;
+
+	retried = 0;
+
+	if (!smp_started)
+		return (0);
+
+	cpumask = PCPU_GET(cpumask);
+	cpulist = PCPU_GET(cpulist);
+	curactive = 0;
+
+	if (rdpr(pil) != 14)
+		panic("pil %ld != 14", rdpr(pil));
+
+#ifndef CPUMASK_NOT_BEING_ERRONEOUSLY_CHANGED
+	/* by definition cpumask should have curcpu's bit set */
+	if (cpumask != (1 << curcpu)) 
+		panic("cpumask(0x%x) != (1 << curcpu) (0x%x)\n", 
+		      cpumask, (1 << curcpu));
+
+#endif
+#ifdef notyet
+	if ((active_total = (pmap->pm_tlbactive & ~cpumask)) == 0)
+		goto done;
+
+	if (pmap->pm_context != 0)
+		active_total = active = (pmap->pm_tlbactive & ~cpumask);
+	else 
+#endif
+		active_total = active = PCPU_GET(other_cpus);
+
+	if (active == 0)
+		goto done;
+	
+ retry:
+	
+	for (i = curactive = cpu_count = 0, cpus = active; i < mp_ncpus && cpus; i++, cpus = (cpus>>1)) {
+		if ((cpus & 0x1) == 0)
+			continue;
+		
+		curactive |= (1 << i);
+		cpulist[cpu_count] = (uint16_t)i;
+		cpu_count++;
+	}
+
+	ackmask = 0;
+	cpu_ipi_selected(cpu_count, cpulist, (uint64_t)func, (uint64_t)arg1, 
+			 (uint64_t)arg2, (uint64_t *)&ackmask);
+
+	while (ackmask != curactive) {
+		membar(Sync);
+		i++;
+		if (i > 10000000) {
+#ifdef TRAP_TRACING
+			int j;
+#endif
+			uint64_t cpu_state;
+			printf("cpu with cpumask=0x%x appears to not be responding to ipis\n",
+			       curactive & ~ackmask);
+
+#ifdef TRAP_TRACING
+			if (!trap_trace_report_done) {
+				trap_trace_report_done = 1;
+				for (j = 0; j < MAXCPU; j++)
+					if (((1 << j) & curactive & ~ackmask) != 0) {
+						struct pcpu *pc = pcpu_find(j);
+						printf("pcpu pad 0x%jx 0x%jx 0x%jx 0x%jx 0x%jx 0x%jx 0x%jx\n",
+						    pc->pad[0], pc->pad[1], pc->pad[2], pc->pad[3],
+						    pc->pad[4], pc->pad[5], pc->pad[6]);
+						trap_trace_report(j);
+					}
+			}
+#endif
+
+			hv_cpu_state((uint64_t)ffs64(curactive & ~ackmask), &cpu_state);
+			printf("cpu_state of %ld is %ld\n", ffs64(curactive & ~ackmask), cpu_state);
+			if (!retried) {
+				printf("I'm going to send off another ipi just to confirm that it isn't a memory barrier bug\n"
+			       "and then I'm going to panic\n");
+
+				retried = 1;
+				goto retry;
+			}
+
+			panic(" ackmask=0x%x active=0x%x\n", ackmask, curactive);
+		}
+	}
+
+	active_total |= curactive;
+	if ((active = ((pmap->pm_tlbactive & all_cpus) & ~(active_total|cpumask))) != 0) {
+		printf("pmap_ipi: retrying");
+		goto retry;
+	}
+ done:
+	return (active_total);
+}
+#endif
+
+void
+pmap_invalidate_page(pmap_t pmap, vm_offset_t va, int cleartsb)
+{
+
+	if (cleartsb == TRUE)
+		tsb_clear_tte(&pmap->pm_tsb, va);
+
+	DPRINTF("pmap_invalidate_page(va=0x%lx)\n", va);
+	spinlock_enter();
+	invlpg(va, pmap->pm_context);
+#ifdef SMP
+	pmap_ipi(pmap, (void *)tl_invlpg, (uint64_t)va, (uint64_t)pmap->pm_context);
+#endif
+	spinlock_exit();
+}
+
+void
+pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int cleartsb)
+{
+	vm_offset_t tva;
+#ifdef SMP
+	char *func;
+	cpumask_t active;
+#endif
+	if ((eva - sva) == PAGE_SIZE) {
+		pmap_invalidate_page(pmap, sva, cleartsb);
+		return;
+	}
+	
+
+	if (sva >= eva) 
+		panic("invalidating negative or zero range sva=0x%lx eva=0x%lx", sva, eva);
+
+	if (cleartsb == TRUE) 
+		tsb_clear_range(&pmap->pm_tsb, sva, eva);
+
+	spinlock_enter();
+	if ((sva - eva) < PAGE_SIZE*64) {
+		for (tva = sva; tva < eva; tva += PAGE_SIZE_8K)
+			invlpg(tva, pmap->pm_context);
+	} else if (pmap->pm_context) 
+		invlctx(pmap->pm_context);
+	else 
+		invltlb();
+	
+#ifdef SMP
+	if (pmap == kernel_pmap)
+		func = tl_invltlb;
+	else
+		func = tl_invlctx;
+
+	active = pmap_ipi(pmap, (void *)func, pmap->pm_context, 0);
+	active &= ~pmap->pm_active;
+	atomic_clear_int(&pmap->pm_tlbactive, active);
+#endif
+	spinlock_exit();
+}
+
+void
+pmap_invalidate_all(pmap_t pmap)
+{
+
+	if (pmap == kernel_pmap)
+		panic("invalidate_all called on kernel_pmap");
+
+	tsb_clear(&pmap->pm_tsb);
+
+	spinlock_enter();
+	invlctx(pmap->pm_context);
+#ifdef SMP
+	pmap_ipi(pmap, tl_invlctx, pmap->pm_context, 0);
+	pmap->pm_tlbactive = pmap->pm_active;
+#endif
+	spinlock_exit();
+}
+
+boolean_t
+pmap_is_modified(vm_page_t m)
+{
+	return (tte_get_phys_bit(m, VTD_W));
+}
+
+
+boolean_t 
+pmap_is_prefaultable(pmap_t pmap, vm_offset_t va)
+{
+	return (tte_hash_lookup(pmap->pm_hash, va) == 0);
+}
+
+/*
+ * Map a wired page into kernel virtual address space.
+ */
+static void
+pmap_kenter(vm_offset_t va, vm_paddr_t pa)
+{
+	tte_hash_update(kernel_pmap->pm_hash, va, pa | TTE_KERNEL | VTD_8K);
+	tsb_set_tte(&kernel_td[TSB8K_INDEX], va, pa | TTE_KERNEL | VTD_8K, 0);
+}
+
+/*
+ * Extract the physical page address associated with the given kernel virtual
+ * address.
+ */
+
+vm_paddr_t
+pmap_kextract(vm_offset_t va)
+{
+	tte_t tte_data;
+	vm_paddr_t pa;
+
+        pa = 0;
+	if (va > KERNBASE && va < KERNBASE + nucleus_memory) {
+		uint64_t offset;
+		offset = va - KERNBASE; 
+		pa = nucleus_mappings[offset >> 22] | (va & PAGE_MASK_4M);
+	}
+	if ((pa == 0) && (tte_data = tsb_lookup_tte(va, 0)) != 0)
+		pa = TTE_GET_PA(tte_data) | (va & TTE_GET_PAGE_MASK(tte_data));
+
+	if ((pa == 0) && (tte_data = tte_hash_lookup(kernel_pmap->pm_hash, va)) != 0)
+		pa = TTE_GET_PA(tte_data) | (va & TTE_GET_PAGE_MASK(tte_data));
+
+	return pa;
+}
+
+static void
+pmap_kremove(vm_offset_t va)
+{ 
+	tte_hash_delete(kernel_pmap->pm_hash, va);
+}
+
+/*
+ * Map a range of physical addresses into kernel virtual address space.
+ *
+ * The value passed in *virt is a suggested virtual address for the mapping.
+ * Architectures which can support a direct-mapped physical to virtual region
+ * can return the appropriate address within that region, leaving '*virt'
+ * unchanged.
+ */
+vm_offset_t
+pmap_map(vm_offset_t *virt, vm_paddr_t start, vm_paddr_t end, int prot)
+{
+	return TLB_PHYS_TO_DIRECT(start);
+}
+
+int 
+pmap_mincore(pmap_t pmap, vm_offset_t addr)
+{
+	return (0);
+}
+
+void 
+pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, vm_object_t object, 
+		    vm_pindex_t index, vm_size_t size)
+{
+	printf("pmap_object_init_pt\n");
+	return;
+}
+
+/*
+ * Returns true if the pmap's pv is one of the first
+ * 16 pvs linked to from this page.  This count may
+ * be changed upwards or downwards in the future; it
+ * is only necessary that true be returned for a small
+ * subset of pmaps for proper page aging.
+ */
+boolean_t
+pmap_page_exists_quick(pmap_t pmap, vm_page_t m)
+{
+	pv_entry_t pv;
+	int loops = 0;
+
+	if (m->flags & PG_FICTITIOUS)
+		return FALSE;
+
+	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+	TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
+		if (pv->pv_pmap == pmap) {
+			return TRUE;
+		}
+		loops++;
+		if (loops >= 16)
+			break;
+	}	
+
+	return (FALSE);
+}
+
+/*
+ * Initialize a vm_page's machine-dependent fields.
+ */
+void
+pmap_page_init(vm_page_t m)
+{
+
+	TAILQ_INIT(&m->md.pv_list);
+	m->md.pv_list_count = 0;
+}
+/*
+ * Lower the permission for all mappings to a given page.
+ */
+void
+pmap_remove_write(vm_page_t m)
+{
+
+	tte_clear_phys_bit(m, VTD_SW_W);
+}
+/*
+ * Initialize the pmap associated with process 0.
+ */
+void
+pmap_pinit0(pmap_t pmap)
+{
+	PMAP_LOCK_INIT(pmap);
+	pmap->pm_active = pmap->pm_tlbactive = ~0;
+	pmap->pm_context = 0;
+	pmap->pm_tsb_ra = kernel_pmap->pm_tsb_ra;
+	pmap->pm_hash = kernel_pmap->pm_hash;
+	PCPU_SET(curpmap, pmap);
+	TAILQ_INIT(&pmap->pm_pvlist);
+	bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
+}
+
+/*
+ * Initialize a preallocated and zeroed pmap structure, such as one in a
+ * vmspace structure.
+ */
+void
+pmap_pinit(pmap_t pmap)
+{
+
+	pmap->pm_context = get_context();
+
+	pmap->pm_hash = tte_hash_create(pmap->pm_context, &pmap->pm_hashscratch);
+	pmap->pm_tsb_ra = tsb_init(&pmap->pm_tsb, &pmap->pm_tsbscratch);
+	pmap->pm_active = pmap->pm_tlbactive = 0;
+	TAILQ_INIT(&pmap->pm_pvlist);
+	PMAP_LOCK_INIT(pmap);
+	bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
+
+}
+
+/*
+ * Set the physical protection on the specified range of this map as requested.
+ */
+void
+pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot)
+{
+
+	int anychanged;
+	vm_offset_t tva;
+
+	DPRINTF("pmap_protect(0x%lx, 0x%lx, %d)\n", sva, eva, prot);
+	
+	if ((prot & VM_PROT_READ) == VM_PROT_NONE) {
+		pmap_remove(pmap, sva, eva);
+		return;
+	}
+	
+	if (prot & VM_PROT_WRITE)
+		return;
+
+	anychanged = 0;
+
+	vm_page_lock_queues();
+	PMAP_LOCK(pmap);
+	sched_pin();
+
+	for (tva = sva; tva < eva; tva += PAGE_SIZE) {
+		uint64_t otte_data;
+		vm_page_t m;
+
+		if ((otte_data = tte_hash_clear_bits(pmap->pm_hash, tva, 
+						     (VTD_SW_W|VTD_W))) == 0)
+			continue;
+
+		if (!anychanged && (otte_data & VTD_W))
+			anychanged = 1;
+		
+		if (otte_data & VTD_MANAGED) {
+			m = NULL;
+
+			if (otte_data & VTD_REF) {
+				m = PHYS_TO_VM_PAGE(TTE_GET_PA(otte_data));
+				vm_page_flag_set(m, PG_REFERENCED);
+			}
+			if ((otte_data & VTD_W) && pmap_track_modified(pmap, tva)) {
+				m = PHYS_TO_VM_PAGE(TTE_GET_PA(otte_data));
+				vm_page_dirty(m);
+			}
+		} 
+	}
+	if (anychanged)
+		pmap_invalidate_range(pmap, sva, eva, TRUE);
+
+	sched_unpin();
+	vm_page_unlock_queues();
+	PMAP_UNLOCK(pmap);
+}
+
+/*
+ * Map a list of wired pages into kernel virtual address space.  This is
+ * intended for temporary mappings which do not need page modification or
+ * references recorded.  Existing mappings in the region are overwritten.
+ */
+void
+pmap_qenter(vm_offset_t sva, vm_page_t *m, int count)
+{
+	vm_offset_t va;
+
+	va = sva;
+	while (count-- > 0) {
+		pmap_kenter(va, VM_PAGE_TO_PHYS(*m));
+		va += PAGE_SIZE;
+		m++;
+	}
+	pmap_invalidate_range(kernel_pmap, sva, va, FALSE);
+}
+
+/*
+ * Remove page mappings from kernel virtual address space.  Intended for
+ * temporary mappings entered by pmap_qenter.
+ */
+void
+pmap_qremove(vm_offset_t sva, int count)
+{
+	vm_offset_t va;
+
+	va = sva;
+
+#ifndef NFS_NOT_BROKEN
+	if (count == 0)
+		count = 1;
+#endif
+	while (count-- > 0) {
+		pmap_kremove(va);
+		va += PAGE_SIZE;
+	}
+	pmap_invalidate_range(kernel_pmap, sva, va, TRUE);
+}
+
+/*
+ * Release any resources held by the given physical map.
+ * Called when a pmap initialized by pmap_pinit is being released.
+ * Should only be called if the map contains no valid mappings.
+ */
+void
+pmap_release(pmap_t pmap)
+{
+	KASSERT(pmap->pm_stats.resident_count == 0,
+	    ("pmap_release: pmap resident count %ld != 0",
+	    pmap->pm_stats.resident_count));
+
+	tsb_deinit(&pmap->pm_tsb);
+	tte_hash_destroy(pmap->pm_hash);
+	free_context(pmap->pm_context);
+	PMAP_LOCK_DESTROY(pmap);
+}
+
+/*
+ * Remove the given range of addresses from the specified map.
+ */
+void
+pmap_remove(pmap_t pmap, vm_offset_t start, vm_offset_t end)
+{
+	int invlva;
+	vm_offset_t tva;
+	uint64_t tte_data;
+	/*
+	 * Perform an unsynchronized read.  This is, however, safe.
+	 */
+	if (pmap->pm_stats.resident_count == 0)
+		return;
+	
+	DPRINTF("pmap_remove(start=0x%lx, end=0x%lx)\n", 
+		start, end);
+	invlva = 0;
+	vm_page_lock_queues();
+	sched_pin();
+	PMAP_LOCK(pmap);
+	for (tva = start; tva < end; tva += PAGE_SIZE) {
+		if ((tte_data = tte_hash_delete(pmap->pm_hash, tva)) == 0)
+			continue;
+		pmap_remove_tte(pmap, tte_data, tva);
+		if (!invlva && (tte_data & (VTD_REF|VTD_W)))
+			invlva = 1;
+	}
+	vm_page_unlock_queues();
+
+	if (invlva)
+		pmap_invalidate_range(pmap, start, end, TRUE);
+
+	sched_unpin();
+	PMAP_UNLOCK(pmap);
+
+}
+
+/*
+ *	Routine:	pmap_remove_all
+ *	Function:
+ *		Removes this physical page from
+ *		all physical maps in which it resides.
+ *		Reflects back modify bits to the pager.
+ *
+ *	Notes:
+ *		Original versions of this routine were very
+ *		inefficient because they iteratively called
+ *		pmap_remove (slow...)
+ */
+
+void
+pmap_remove_all(vm_page_t m)
+{
+	pv_entry_t pv;
+	uint64_t tte_data;
+	DPRINTF("pmap_remove_all 0x%lx\n", VM_PAGE_TO_PHYS(m));
+
+	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+	sched_pin();
+	while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) {
+		PMAP_LOCK(pv->pv_pmap);
+		pv->pv_pmap->pm_stats.resident_count--;
+
+		tte_data = tte_hash_delete(pv->pv_pmap->pm_hash, pv->pv_va);
+
+		if (tte_data & VTD_WIRED)
+			pv->pv_pmap->pm_stats.wired_count--;
+		if (tte_data & VTD_REF)
+			vm_page_flag_set(m, PG_REFERENCED);
+		
+		/*
+		 * Update the vm_page_t clean and reference bits.
+		 */
+		if (tte_data & VTD_W) {
+			KASSERT((tte_data & VTD_SW_W),
+	("pmap_remove_all: modified page not writable: va: %lx, tte: %lx",
+			    pv->pv_va, tte_data));
+			if (pmap_track_modified(pv->pv_pmap, pv->pv_va))
+				vm_page_dirty(m);
+		}
+	
+		pmap_invalidate_page(pv->pv_pmap, pv->pv_va, TRUE);
+		TAILQ_REMOVE(&pv->pv_pmap->pm_pvlist, pv, pv_plist);
+		TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
+		m->md.pv_list_count--;
+		PMAP_UNLOCK(pv->pv_pmap);
+		free_pv_entry(pv);
+	}
+	vm_page_flag_clear(m, PG_WRITEABLE);
+	sched_unpin();
+}
+
+static void
+pmap_remove_entry(pmap_t pmap, vm_page_t m, vm_offset_t va)
+{
+	pv_entry_t pv;
+	if (pmap != kernel_pmap)
+		DPRINTF("pmap_remove_entry(va=0x%lx, pa=0x%lx)\n", va, VM_PAGE_TO_PHYS(m));
+	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
+	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+	if (m->md.pv_list_count < pmap->pm_stats.resident_count) {
+		TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
+			if (pmap == pv->pv_pmap && va == pv->pv_va) 
+				break;
+		}
+	} else {
+		TAILQ_FOREACH(pv, &pmap->pm_pvlist, pv_plist) {
+			if (va == pv->pv_va) 
+				break;
+		}
+	}
+	KASSERT(pv != NULL, ("pmap_remove_entry: pv not found va=0x%lx pa=0x%lx", va, VM_PAGE_TO_PHYS(m)));
+	TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
+	m->md.pv_list_count--;
+	if (TAILQ_EMPTY(&m->md.pv_list))
+		vm_page_flag_clear(m, PG_WRITEABLE);
+	TAILQ_REMOVE(&pmap->pm_pvlist, pv, pv_plist);
+	free_pv_entry(pv);
+}
+
+
+void
+pmap_remove_pages(pmap_t pmap)
+{
+	
+	vm_page_t m;
+	pv_entry_t pv, npv;
+	tte_t tte_data;
+	
+	DPRINTF("pmap_remove_pages(ctx=0x%lx)\n", pmap->pm_context);
+	vm_page_lock_queues();
+	PMAP_LOCK(pmap);
+	sched_pin();
+	for (pv = TAILQ_FIRST(&pmap->pm_pvlist); pv; pv = npv) {
+		tte_data = tte_hash_delete(pmap->pm_hash, pv->pv_va);
+		
+		if (tte_data == 0) {
+			membar(Sync);
+			DELAY(100);
+			membar(Sync);
+			tte_data = tte_hash_delete(pmap->pm_hash, pv->pv_va);
+			if (tte_data == 0) {
+				printf("TTE IS ZERO @ VA %016lx\n", pv->pv_va);
+				panic("bad tte");
+			}
+		}
+		
+		if (tte_data & VTD_WIRED) 
+			pmap->pm_stats.wired_count--;
+
+		m = PHYS_TO_VM_PAGE(TTE_GET_PA(tte_data));
+
+		pmap->pm_stats.resident_count--;
+		
+		if (tte_data & VTD_W) {
+			vm_page_dirty(m);
+		}
+		
+		npv = TAILQ_NEXT(pv, pv_plist);
+		TAILQ_REMOVE(&pmap->pm_pvlist, pv, pv_plist);
+		
+		m->md.pv_list_count--;
+		TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
+		if (TAILQ_EMPTY(&m->md.pv_list))
+			vm_page_flag_clear(m, PG_WRITEABLE);
+
+		free_pv_entry(pv);
+	}
+	tte_hash_reset(pmap->pm_hash);
+
+	pmap_invalidate_all(pmap);
+	sched_unpin();
+	PMAP_UNLOCK(pmap);
+	vm_page_unlock_queues();
+}
+
+
+
+void
+pmap_scrub_pages(vm_paddr_t pa, int64_t size)
+{
+	uint64_t bytes_zeroed;
+	while (size > 0) {
+		hv_mem_scrub(pa, size, &bytes_zeroed);
+		pa += bytes_zeroed;
+		size -= bytes_zeroed;
+	}
+}
+static void
+pmap_remove_tte(pmap_t pmap, tte_t tte_data, vm_offset_t va)
+{
+	
+	vm_page_t m;
+
+	if (pmap != kernel_pmap)
+		DPRINTF("pmap_remove_tte(va=0x%lx, pa=0x%lx)\n", va, TTE_GET_PA(tte_data));
+
+	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
+	if (tte_data & VTD_WIRED)
+		pmap->pm_stats.wired_count--;
+
+	pmap->pm_stats.resident_count--;
+	
+	if (tte_data & VTD_MANAGED) {
+		m = PHYS_TO_VM_PAGE(TTE_GET_PA(tte_data));
+		if (tte_data & VTD_W) {
+			if (pmap_track_modified(pmap, va))
+				vm_page_dirty(m);	
+		}
+		if (tte_data & VTD_REF) 
+			vm_page_flag_set(m, PG_REFERENCED);
+		pmap_remove_entry(pmap, m, va);
+	}
+}
+
+/*
+ *	pmap_ts_referenced:
+ *
+ *	Return a count of reference bits for a page, clearing those bits.
+ *	It is not necessary for every reference bit to be cleared, but it
+ *	is necessary that 0 only be returned when there are truly no
+ *	reference bits set.
+ *
+ *	XXX: The exact number of bits to check and clear is a matter that
+ *	should be tested and standardized at some point in the future for
+ *	optimal aging of shared pages.
+ */
+
+int
+pmap_ts_referenced(vm_page_t m)
+{
+	
+	int rv;
+	pv_entry_t pv, pvf, pvn;
+	pmap_t pmap;
+	tte_t otte_data;
+
+	rv = 0;
+	if (m->flags & PG_FICTITIOUS)
+		return (rv);
+
+        sched_pin();
+        mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+        if ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) {
+		
+		pvf = pv;
+
+		do {
+                        pvn = TAILQ_NEXT(pv, pv_list);
+			
+                        TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
+			
+                        TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list);
+			
+                        pmap = pv->pv_pmap;
+                        PMAP_LOCK(pmap);
+			otte_data = tte_hash_clear_bits(pmap->pm_hash, pv->pv_va, VTD_REF);
+			if ((otte_data & VTD_REF) != 0) {
+                                pmap_invalidate_page(pmap, pv->pv_va, TRUE);
+				
+                                rv++;
+                                if (rv > 4) {
+                                        PMAP_UNLOCK(pmap);
+                                        break;
+                                }
+			}
+		
+			PMAP_UNLOCK(pmap);
+		} while ((pv = pvn) != NULL && pv != pvf);
+	}
+	sched_unpin();
+
+	return (rv);
+}
+
+void
+pmap_zero_page(vm_page_t m)
+{
+	hwblkclr((void *)TLB_PHYS_TO_DIRECT(VM_PAGE_TO_PHYS(m)), PAGE_SIZE);
+}
+
+void
+pmap_zero_page_area(vm_page_t m, int off, int size)
+{
+	vm_paddr_t pa;
+	vm_offset_t va;
+		
+	pa = VM_PAGE_TO_PHYS(m);
+	va = TLB_PHYS_TO_DIRECT(pa);
+	
+	bzero((char *)(va + off), size);
+
+}
+
+void
+pmap_zero_page_idle(vm_page_t m)
+{
+	hwblkclr((void *)TLB_PHYS_TO_DIRECT(VM_PAGE_TO_PHYS(m)), PAGE_SIZE);
+}
+
+void
+pmap_set_ctx_panic(uint64_t error, vm_paddr_t tsb_ra, pmap_t pmap)
+{
+	panic("setting ctxnon0 failed ctx=0x%lx hvtsb_ra=0x%lx tsbscratch=0x%lx error=0x%lx",
+	      pmap->pm_context, tsb_ra, pmap->pm_tsbscratch, error);
+	
+}
diff --git a/sys/sun4v/sun4v/prof_machdep.c b/sys/sun4v/sun4v/prof_machdep.c
new file mode 100644
index 000000000000..065a1daa44c2
--- /dev/null
+++ b/sys/sun4v/sun4v/prof_machdep.c
@@ -0,0 +1,83 @@
+/*-
+ * Copyright (c) 1996 Bruce D. Evans.
+ * Copyright (c) 2002 by Thomas Moestl <tmm@FreeBSD.org>.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	from: src/sys/i386/isa/prof_machdep.c,v 1.16 2000/07/04 11:25:19
+ *
+ * $FreeBSD$
+ */
+
+#ifdef GUPROF
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/gmon.h>
+#include <sys/kernel.h>
+#include <sys/sysctl.h>
+
+#include <machine/clock.h>
+#include <machine/profile.h>
+
+int	cputime_bias;
+
+/*
+ * Return the time elapsed since the last call.  The units are machine-
+ * dependent.
+ * XXX: this is not SMP-safe. It should use per-CPU variables; %tick can be
+ * used though.
+ */
+int
+cputime(void)
+{
+	u_long count;
+	int delta;
+	static u_long prev_count;
+
+	count = rd(tick);
+	delta = (int)(count - prev_count);
+	prev_count = count;
+	return (delta);
+}
+
+/*
+ * The start and stop routines need not be here since we turn off profiling
+ * before calling them.  They are here for convenience.
+ */
+void
+startguprof(struct gmonparam *gp)
+{
+
+	gp->profrate = tick_freq;
+	cputime_bias = 0;
+	cputime();
+}
+
+void
+stopguprof(struct gmonparam *gp)
+{
+	/* Nothing to do. */
+}
+
+#endif /* GUPROF */
diff --git a/sys/sun4v/sun4v/rtc.c b/sys/sun4v/sun4v/rtc.c
new file mode 100644
index 000000000000..fd988da85253
--- /dev/null
+++ b/sys/sun4v/sun4v/rtc.c
@@ -0,0 +1,191 @@
+/*-
+ * Copyright (c) 2004 Marius Strobl <marius@FreeBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+/*
+ * The `rtc' device is a MC146818 compatible clock found on the ISA bus
+ * and EBus. The EBus version also has an interrupt property so it could
+ * be used to drive the statclock etc.
+ */
+
+#include "opt_isa.h"
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/bus.h>
+#include <sys/kernel.h>
+#include <sys/lock.h>
+#include <sys/module.h>
+#include <sys/mutex.h>
+#include <sys/resource.h>
+
+#include <dev/ofw/ofw_bus.h>
+
+#include <machine/bus.h>
+#include <machine/resource.h>
+
+#include <sys/rman.h>
+
+#include <isa/isavar.h>
+
+#include <dev/mc146818/mc146818var.h>
+
+#include "clock_if.h"
+
+static devclass_t rtc_devclass;
+
+static int	rtc_attach(device_t dev);
+static int	rtc_ebus_probe(device_t dev);
+#ifdef DEV_ISA
+static int	rtc_isa_probe(device_t dev);
+#endif
+
+static device_method_t rtc_ebus_methods[] = {
+	/* Device interface */
+	DEVMETHOD(device_probe,		rtc_ebus_probe),
+	DEVMETHOD(device_attach,	rtc_attach),
+
+	/* clock interface */
+	DEVMETHOD(clock_gettime,	mc146818_gettime),
+	DEVMETHOD(clock_settime,	mc146818_settime),
+
+	{ 0, 0 }
+};
+
+static driver_t rtc_ebus_driver = {
+	"rtc",
+	rtc_ebus_methods,
+	sizeof(struct mc146818_softc),
+};
+
+DRIVER_MODULE(rtc, ebus, rtc_ebus_driver, rtc_devclass, 0, 0);
+
+#ifdef DEV_ISA
+static device_method_t rtc_isa_methods[] = {
+	/* Device interface */
+	DEVMETHOD(device_probe,		rtc_isa_probe),
+	DEVMETHOD(device_attach,	rtc_attach),
+
+	/* clock interface */
+	DEVMETHOD(clock_gettime,	mc146818_gettime),
+	DEVMETHOD(clock_settime,	mc146818_settime),
+
+	{ 0, 0 }
+};
+
+static driver_t rtc_isa_driver = {
+	"rtc",
+	rtc_isa_methods,
+	sizeof(struct mc146818_softc),
+};
+
+DRIVER_MODULE(rtc, isa, rtc_isa_driver, rtc_devclass, 0, 0);
+#endif
+
+static int
+rtc_ebus_probe(device_t dev)
+{
+ 
+	if (strcmp(ofw_bus_get_name(dev), "rtc") == 0) {
+		device_set_desc(dev, "Real Time Clock");
+		return (0);
+	}
+
+	return (ENXIO);
+}
+
+#ifdef DEV_ISA
+static struct isa_pnp_id rtc_isa_ids[] = {
+	{ 0x000bd041, "AT realtime clock" }, /* PNP0B00 */
+	{ 0 }
+};
+
+static int
+rtc_isa_probe(device_t dev)
+{
+
+	if (ISA_PNP_PROBE(device_get_parent(dev), dev, rtc_isa_ids) == 0) {
+		device_set_desc(dev, "Real Time Clock");
+		return (0);
+	}
+
+	return (ENXIO);
+}
+#endif
+
+static int
+rtc_attach(device_t dev)
+{
+	struct timespec ts;
+	struct mc146818_softc *sc;
+	struct resource *res;
+	int error, rid, rtype;
+
+	sc = device_get_softc(dev);
+
+	mtx_init(&sc->sc_mtx, "rtc_mtx", NULL, MTX_SPIN);
+
+	if (strcmp(device_get_name(device_get_parent(dev)), "isa") == 0)
+		rtype = SYS_RES_IOPORT;
+	else
+		rtype = SYS_RES_MEMORY;
+
+	rid = 0;
+	res = bus_alloc_resource_any(dev, rtype, &rid, RF_ACTIVE);
+	if (res == NULL) {
+		device_printf(dev, "cannot allocate resources\n");
+		error = ENXIO;
+		goto fail_mtx;
+	}
+	sc->sc_bst = rman_get_bustag(res);
+	sc->sc_bsh = rman_get_bushandle(res);
+
+	/* The TOD clock year 0 is 0. */
+	sc->sc_year0 = 0;
+	/* Use default register read/write and century get/set functions. */
+	sc->sc_flag = MC146818_NO_CENT_ADJUST;
+	if ((error = mc146818_attach(dev)) != 0) {
+		device_printf(dev, "cannot attach time of day clock\n");
+		goto fail_res;
+	}
+
+	if (bootverbose) {
+		mc146818_gettime(dev, &ts);
+		device_printf(dev, "current time: %ld.%09ld\n", (long)ts.tv_sec,
+		    ts.tv_nsec);
+        }
+
+	return (0);
+
+ fail_res:
+	bus_release_resource(dev, rtype, rid, res);
+ fail_mtx:
+	mtx_destroy(&sc->sc_mtx);
+
+	return (error);
+}
diff --git a/sys/sun4v/sun4v/rwindow.c b/sys/sun4v/sun4v/rwindow.c
new file mode 100644
index 000000000000..5ea877413bd8
--- /dev/null
+++ b/sys/sun4v/sun4v/rwindow.c
@@ -0,0 +1,110 @@
+/*-
+ * Copyright (c) 1997 Berkeley Software Design, Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Berkeley Software Design Inc's name may not be used to endorse or
+ *    promote products derived from this software without specific prior
+ *    written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	from: BSDI: trap.c,v 1.17.2.9 1999/10/19 15:29:52 cp Exp
+ * $FreeBSD$
+ */
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/ktr.h>
+#include <sys/proc.h>
+
+#include <machine/frame.h>
+#include <machine/pcb.h>
+
+CTASSERT((1 << RW_SHIFT) == sizeof(struct rwindow));
+
+int
+rwindow_load(struct thread *td, struct trapframe *tf, int n)
+{
+	struct rwindow rw;
+	u_long usp;
+	int error;
+	int i;
+
+	CTR3(KTR_TRAP, "rwindow_load: td=%p (%s) n=%d",
+	    td, td->td_proc->p_comm, n);
+
+	/*
+	 * In case current window is still only on-chip, push it out;
+	 * if it cannot get all the way out, we cannot continue either.
+	 */
+	if ((error = rwindow_save(td)) != 0)
+		return (error);
+	usp = tf->tf_out[6];
+	for (i = 0; i < n; i++) {
+		CTR1(KTR_TRAP, "rwindow_load: usp=%#lx", usp);
+		usp += SPOFF;
+		if ((error = (usp & 0x7)) != 0)
+			break;
+		error = copyin((void *)usp, &rw, sizeof rw);
+		usp = rw.rw_in[6];
+	}
+	CTR1(KTR_TRAP, "rwindow_load: error=%d", error);
+	return (error == 0 ? 0 : SIGILL);
+}
+
+int
+rwindow_save(struct thread *td)
+{
+	struct rwindow *rw;
+	struct pcb *pcb;
+	u_long *ausp;
+	u_long usp;
+	int error;
+	int i;
+
+	pcb = td->td_pcb;
+	CTR3(KTR_TRAP, "rwindow_save: td=%p (%s) nsaved=%d", td,
+	    td->td_proc->p_comm, pcb->pcb_nsaved);
+
+	flushw();
+	KASSERT(pcb->pcb_nsaved < MAXWIN,
+	    ("rwindow_save: pcb_nsaved > MAXWIN"));
+	if ((i = pcb->pcb_nsaved) == 0)
+		return (0);
+	ausp = pcb->pcb_rwsp;
+	rw = pcb->pcb_rw;
+	error = 0;
+	do {
+		usp = *ausp;
+		CTR1(KTR_TRAP, "rwindow_save: usp=%#lx", usp);
+		usp += SPOFF;
+		if ((error = (usp & 0x7)) != 0)
+			break;
+		error = copyout(rw, (void *)usp, sizeof *rw);
+		if (error)
+			break;
+		ausp++;
+		rw++;
+	} while (--i > 0);
+	CTR1(KTR_TRAP, "rwindow_save: error=%d", error);
+	if (error == 0)
+		pcb->pcb_nsaved = 0;
+	return (error == 0 ? 0 : SIGILL);
+}
diff --git a/sys/sun4v/sun4v/simdisk.c b/sys/sun4v/sun4v/simdisk.c
new file mode 100644
index 000000000000..2584b593dcf7
--- /dev/null
+++ b/sys/sun4v/sun4v/simdisk.c
@@ -0,0 +1,251 @@
+/*-
+ * Copyright (c) 2006 Kip Macy
+ * Copyright (c) 2001 Benno Rice
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/bio.h>
+#include <sys/kernel.h>
+#include <sys/kthread.h>
+#include <sys/linker.h>
+#include <sys/lock.h>
+#include <sys/malloc.h>
+#include <sys/mutex.h>
+#include <sys/proc.h>
+
+#include <vm/vm.h>
+#include <vm/pmap.h>
+
+#include <geom/geom.h>
+
+#include <dev/ofw/ofw_bus.h>
+#include <dev/ofw/ofw_bus_subr.h>
+#include <dev/ofw/openfirm.h>
+
+
+#include <machine/hypervisor_api.h>
+
+#define	HVD_BLOCKSIZE	512
+
+struct hvd_softc
+{
+        struct bio_queue_head hvd_bio_queue;
+        struct mtx	hvd_queue_mtx;
+	off_t		hvd_mediasize;
+        unsigned	hvd_sectorsize;
+        unsigned	hvd_fwheads;
+        unsigned	hvd_fwsectors;
+        struct proc	*hvd_procp;
+        struct g_geom	*hvd_gp;
+        struct g_provider *hvd_pp;
+} hvd_softc;
+
+static g_init_t g_hvd_init;
+static g_start_t g_hvd_start;
+static g_access_t g_hvd_access;
+
+struct g_class g_hvd_class = {
+	.name = "HVD",
+	.version = G_VERSION,
+	.init = g_hvd_init,
+	.start = g_hvd_start,
+	.access = g_hvd_access,
+};
+
+DECLARE_GEOM_CLASS(g_hvd_class, g_hvd);
+
+
+
+static int
+hvd_startio(struct hvd_softc *sc, struct bio *bp)
+{
+	u_int r;
+	int len, rlen, wlen;
+	uint64_t page_off;
+
+	r = H_EOK;
+	len = 0;
+
+	page_off = bp->bio_offset & PAGE_MASK;
+
+
+        switch (bp->bio_cmd) {
+        case BIO_READ:
+		if (bp->bio_length > (PAGE_SIZE - page_off)) {
+			len = rlen = (PAGE_SIZE - page_off);
+			r = hv_sim_read(bp->bio_offset, vtophys((char *)bp->bio_data), rlen);
+		}
+		for (; len < bp->bio_length && r == H_EOK; len += PAGE_SIZE) {
+			rlen = (bp->bio_length - len) > PAGE_SIZE ? PAGE_SIZE : bp->bio_length - len;
+			r = hv_sim_read(bp->bio_offset + len, vtophys((char *)bp->bio_data + len), rlen);
+
+		}
+                break;
+        case BIO_WRITE:
+		if (bp->bio_length > (PAGE_SIZE - page_off)) {
+			len = wlen = (PAGE_SIZE - page_off);
+			r = hv_sim_write(bp->bio_offset, vtophys((char *)bp->bio_data), wlen);
+		}
+		for (; len < bp->bio_length && r == H_EOK; len += PAGE_SIZE) {
+			wlen = (bp->bio_length - len) > PAGE_SIZE ? PAGE_SIZE : bp->bio_length - len;
+			r = hv_sim_write(bp->bio_offset + len, vtophys((char *)bp->bio_data + len), wlen);
+		}
+                break;
+        }
+	if (r != H_EOK)
+		panic("invalid I/O");
+
+        bp->bio_resid = 0;
+        return (0);
+}
+
+static void
+hvd_kthread(void *arg)
+{
+	struct hvd_softc *sc;
+	struct bio *bp;
+	int error;
+
+        sc = arg;
+        curthread->td_base_pri = PRIBIO;
+
+        for (;;) {
+		mtx_lock(&sc->hvd_queue_mtx);
+		bp = bioq_takefirst(&sc->hvd_bio_queue);
+		if (!bp) {
+			msleep(sc, &sc->hvd_queue_mtx, PRIBIO | PDROP,
+			    "hvdwait", 0);
+                        continue;
+		}
+                mtx_unlock(&sc->hvd_queue_mtx);
+                if (bp->bio_cmd == BIO_GETATTR) {
+			error = EOPNOTSUPP;
+                } else
+			error = hvd_startio(sc, bp);
+
+		if (error != -1) {
+                        bp->bio_completed = bp->bio_length;
+                        g_io_deliver(bp, error);
+                }
+	}
+}
+
+static void
+g_hvd_init(struct g_class *mp __unused)
+{
+	struct hvd_softc *sc;
+        struct g_geom *gp;
+        struct g_provider *pp;
+	int	error;
+	printf("calling g_hvd_init\n");
+
+	sc = (struct hvd_softc *)malloc(sizeof *sc, M_DEVBUF,
+	         M_WAITOK|M_ZERO);
+	bioq_init(&sc->hvd_bio_queue);
+        mtx_init(&sc->hvd_queue_mtx, "hvd bio queue", NULL, MTX_DEF);
+	sc->hvd_mediasize = (off_t)0x20000000;
+	sc->hvd_sectorsize = HVD_BLOCKSIZE;
+	sc->hvd_fwsectors = 0;
+	sc->hvd_fwheads = 0;
+	error = kthread_create(hvd_kthread, sc, &sc->hvd_procp, 0, 0,
+		     "hvd0");
+        if (error != 0) {
+		free(sc, M_DEVBUF);
+                return;
+	}
+
+	gp = g_new_geomf(&g_hvd_class, "hvd0");
+	gp->softc = sc;
+	pp = g_new_providerf(gp, "hvd0");
+	pp->mediasize = sc->hvd_mediasize;
+	pp->sectorsize = sc->hvd_sectorsize;
+	sc->hvd_gp = gp;
+	sc->hvd_pp = pp;
+	g_error_provider(pp, 0);
+}
+
+static void
+g_hvd_start(struct bio *bp)
+{
+        struct hvd_softc *sc;
+#if 0
+	printf("in hvd_start\n");
+#endif
+        sc = bp->bio_to->geom->softc;
+        mtx_lock(&sc->hvd_queue_mtx);
+        bioq_disksort(&sc->hvd_bio_queue, bp);
+        mtx_unlock(&sc->hvd_queue_mtx);
+        wakeup(sc);
+}
+
+static int
+g_hvd_access(struct g_provider *pp, int r, int w, int e)
+{
+
+	if (pp->geom->softc == NULL)
+		return (ENXIO);
+        return (0);
+}
+
+static int
+hvd_probe(device_t dev)
+{
+
+	if (strcmp(ofw_bus_get_name(dev), "disk"))
+		return (ENXIO);
+	
+	device_set_desc(dev, "sun4v virtual disk");	
+
+	return (0);
+}
+
+
+static int
+hvd_attach(device_t dev)
+{
+	return (0);
+}
+
+static device_method_t hvd_methods[] = {
+        DEVMETHOD(device_probe, hvd_probe),
+        DEVMETHOD(device_attach, hvd_attach),
+        {0, 0}
+};
+
+
+static driver_t hvd_driver = {
+        "hvd",
+        hvd_methods,
+	0,
+};
+
+
+static devclass_t hvd_devclass;
+
+DRIVER_MODULE(hvd, vnex, hvd_driver, hvd_devclass, 0, 0);
diff --git a/sys/sun4v/sun4v/support.S b/sys/sun4v/sun4v/support.S
new file mode 100644
index 000000000000..f388afb5dd0a
--- /dev/null
+++ b/sys/sun4v/sun4v/support.S
@@ -0,0 +1,897 @@
+/*-
+ * Copyright (c) 2001 Jake Burkholder.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <machine/asm.h>
+__FBSDID("$FreeBSD$"); 
+	
+#include <machine/asi.h>
+#include <machine/asmacros.h>
+#include <machine/ktr.h>
+#include <machine/pstate.h>
+#include <machine/hypervisorvar.h>
+	
+#include "assym.s"
+
+	.register %g2, #ignore
+	.register %g3, #ignore
+	.register %g6, #ignore
+
+#define	E	/* empty */
+
+/*
+ * Generate load and store instructions for the corresponding width and asi
+ * (or not).  Note that we want to evaluate the macro args before
+ * concatenating, so that E really turns into nothing.
+ */
+#define	_LD(w, a)	ld ## w ## a
+#define	_ST(w, a)	st ## w ## a
+
+#define	LD(w, a)	_LD(w, a)
+#define	ST(w, a)	_ST(w, a)
+
+/*
+ * Common code for copy routines.
+ *
+ * We use large macros to generate functions for each of the copy routines.
+ * This allows the load and store instructions to be generated for the right
+ * operation, asi or not.  It is possible to write an asi independent function
+ * but this would require 2 expensive wrs in the main loop to switch %asi.
+ * It would also screw up profiling (if we ever get it), but may save some I$.
+ * We assume that either one of dasi and sasi is empty, or that they are both
+ * the same (empty or non-empty).  It is up to the caller to set %asi.
+ */
+
+/*
+ * ASI independent implementation of copystr(9).
+ * Used to implement copyinstr() and copystr().
+ *
+ * Return value is in %g1.
+ */
+#define	_COPYSTR(src, dst, len, done, sa, sasi, da, dasi) \
+	brz	len, 4f ; \
+	 mov	src, %g2 ; \
+1:	deccc	1, len ; \
+	bl,a,pn	%xcc, 3f ; \
+	 nop ; \
+	LD(ub, sa) [src] sasi, %g1 ; \
+	ST(b, da) %g1, [dst] dasi ; \
+	brz,pn	%g1, 3f ; \
+	 inc	src ; \
+	ba	%xcc, 1b ; \
+	 inc	dst ; \
+2:	mov	ENAMETOOLONG, %g1 ; \
+3:	sub	src, %g2, %g2 ; \
+	brnz,a	done, 4f ; \
+	 stx	%g2, [done] ; \
+4:
+
+/*
+ * ASI independent implementation of memset(3).
+ * Used to implement bzero(), memset() and aszero().
+ *
+ * If the pattern is non-zero, duplicate it to fill 64 bits.
+ * Store bytes until dst is 8-byte aligned, then store 8 bytes.
+ * It has yet to be determined how much unrolling is beneficial.
+ * Could also read and compare before writing to minimize snoop traffic.
+ *
+ * XXX bzero() should be implemented as
+ * #define bzero(dst, len) (void)memset((dst), 0, (len))
+ * if at all.
+ */
+#define	_MEMSET(dst, pat, len, da, dasi) \
+	brlez,pn len, 5f ; \
+	 and	pat, 0xff, pat ; \
+	brz,pt	pat, 1f ; \
+	 sllx	pat, 8, %g1 ; \
+	or	pat, %g1, pat ; \
+	sllx	pat, 16, %g1 ; \
+	or	pat, %g1, pat ; \
+	sllx	pat, 32, %g1 ; \
+	or	pat, %g1, pat ; \
+	.align 16 ; \
+1:	deccc	1, len ; \
+	bl,pn	%xcc, 5f ; \
+	 btst	7, dst ; \
+	bz,a,pt	%xcc, 2f ; \
+	 inc	1, len ; \
+	ST(b, da) pat, [dst] dasi ; \
+	ba	%xcc, 1b ; \
+	 inc	dst ; \
+	.align 16 ; \
+2:	deccc	32, len ; \
+	bl,a,pn	%xcc, 3f ; \
+	 inc	32, len ; \
+	ST(x, da) pat, [dst] dasi ; \
+	ST(x, da) pat, [dst + 8] dasi ; \
+	ST(x, da) pat, [dst + 16] dasi ; \
+	ST(x, da) pat, [dst + 24] dasi ; \
+	ba	%xcc, 2b ; \
+	 inc	32, dst ; \
+	.align 16 ; \
+3:	deccc	8, len ; \
+	bl,a,pn	%xcc, 4f ; \
+	 inc	8, len ; \
+	ST(x, da) pat, [dst] dasi ; \
+	ba	%xcc, 3b ; \
+	 inc	8, dst ; \
+	.align 16 ; \
+4:	deccc	1, len ; \
+	bl,a,pn	%xcc, 5f ; \
+	 nop ; \
+	ST(b, da) pat, [dst] dasi ; \
+	ba	%xcc, 4b ; \
+	 inc	1, dst ; \
+5:
+
+/*
+ * ASI independent implementation of memcpy(3).
+ * Used to implement bcopy(), copyin(), copyout(), memcpy(), ascopy(),
+ * ascopyfrom() and ascopyto().
+ *
+ * Transfer bytes until dst is 8-byte aligned.  If src is then also 8 byte
+ * aligned, transfer 8 bytes, otherwise finish with bytes.  The unaligned
+ * case could be optimized, but it is expected that this is the uncommon
+ * case and of questionable value.  The code to do so is also rather large
+ * and ugly.  It has yet to be determined how much unrolling is beneficial.
+ *
+ * XXX bcopy() must also check for overlap.  This is stupid.
+ * XXX bcopy() should be implemented as
+ * #define bcopy(src, dst, len) (void)memcpy((dst), (src), (len))
+ * if at all.
+ */
+#define	_MEMCPY(dst, src, len, da, dasi, sa, sasi) \
+1:	deccc	1, len ; \
+	bl,pn	%xcc, 6f ; \
+	 btst	7, dst ; \
+	bz,a,pt	%xcc, 2f ; \
+	 inc	1, len ; \
+	LD(ub, sa) [src] sasi, %g1 ; \
+	ST(b, da) %g1, [dst] dasi ; \
+	inc	1, src ; \
+	ba	%xcc, 1b ; \
+	 inc	1, dst ; \
+	.align 16 ; \
+2:	btst	7, src ; \
+	bz,a,pt	%xcc, 3f ; \
+	 nop ; \
+	ba,a	%xcc, 5f ; \
+	.align 16 ; \
+3:	deccc	32, len ; \
+	bl,a,pn	%xcc, 4f ; \
+	 inc	32, len ; \
+	LD(x, sa) [src] sasi, %g1 ; \
+	LD(x, sa) [src + 8] sasi, %g2 ; \
+	LD(x, sa) [src + 16] sasi, %g3 ; \
+	LD(x, sa) [src + 24] sasi, %g4 ; \
+	ST(x, da) %g1, [dst] dasi ; \
+	ST(x, da) %g2, [dst + 8] dasi ; \
+	ST(x, da) %g3, [dst + 16] dasi ; \
+	ST(x, da) %g4, [dst + 24] dasi ; \
+	inc	32, src ; \
+	ba	%xcc, 3b ; \
+	 inc	32, dst ; \
+	.align 16 ; \
+4:	deccc	8, len ; \
+	bl,a,pn	%xcc, 5f ; \
+	 inc	8, len ; \
+	LD(x, sa) [src] sasi, %g1 ; \
+	ST(x, da) %g1, [dst] dasi ; \
+	inc	8, src ; \
+	ba	%xcc, 4b ; \
+	 inc	8, dst ; \
+	.align 16 ; \
+5:	deccc	1, len ; \
+	bl,a,pn	%xcc, 6f ; \
+	 nop ; \
+	LD(ub, sa) [src] sasi, %g1 ; \
+	ST(b, da) %g1, [dst] dasi ; \
+	inc	src ; \
+	ba	%xcc, 5b ; \
+	 inc	dst ; \
+6:
+
+/*
+ * void ascopy(u_long asi, vm_offset_t src, vm_offset_t dst, size_t len)
+ */
+ENTRY(ascopy)
+	wr	%o0, 0, %asi
+	_MEMCPY(%o2, %o1, %o3, a, %asi, a, %asi)
+	retl
+	 nop
+END(ascopy)
+
+/*
+ * void ascopyfrom(u_long sasi, vm_offset_t src, caddr_t dst, size_t len)
+ */
+ENTRY(ascopyfrom)
+	wr	%o0, 0, %asi
+	_MEMCPY(%o2, %o1, %o3, E, E, a, %asi)
+	retl
+	 nop
+END(ascopyfrom)
+
+/*
+ * void ascopyto(caddr_t src, u_long dasi, vm_offset_t dst, size_t len)
+ */
+ENTRY(ascopyto)
+	wr	%o1, 0, %asi
+	_MEMCPY(%o2, %o0, %o3, a, %asi, E, E)
+	retl
+	 nop
+END(ascopyto)
+
+/*
+ * void aszero(u_long asi, vm_offset_t pa, size_t len)
+ */
+ENTRY(aszero)
+	wr	%o0, 0, %asi
+	_MEMSET(%o1, %g0, %o2, a, %asi)
+	retl
+	 nop
+END(aszero)
+
+/*
+ * int bcmp(const void *b1, const void *b2, size_t len)
+ */
+ENTRY(bcmp)
+	brz,pn	%o2, 2f
+	 clr	%o3
+1:	ldub	[%o0 + %o3], %o4
+	ldub	[%o1 + %o3], %o5
+	cmp	%o4, %o5
+	bne,pn	%xcc, 2f
+	 inc	%o3
+	deccc	%o2
+	bne,pt	%xcc, 1b
+	 nop
+2:	retl
+	 mov	%o2, %o0
+END(bcmp)
+#if 0
+/*
+ * void bcopy(const void *src, void *dst, size_t len)
+ */
+ENTRY(bcopy)
+	/*
+	 * Check for overlap, and copy backwards if so.
+	 */
+	sub	%o1, %o0, %g1
+	cmp	%g1, %o2
+	bgeu,a,pt %xcc, 3f
+	 nop
+
+	/*
+	 * Copy backwards.
+	 */
+	add	%o0, %o2, %o0
+	add	%o1, %o2, %o1
+1:	deccc	1, %o2
+	bl,a,pn	%xcc, 2f
+	 nop
+	dec	1, %o0
+	ldub	[%o0], %g1
+	dec	1, %o1
+	ba	%xcc, 1b
+	 stb	%g1, [%o1]
+2:	retl
+	 nop
+
+	/*
+	 * Do the fast version.
+	 */
+3:	_MEMCPY(%o1, %o0, %o2, E, E, E, E)
+	retl
+	 nop
+END(bcopy)
+
+/*
+ * void bzero(void *b, size_t len)
+ */
+ENTRY(bzero)
+	_MEMSET(%o0, %g0, %o1, E, E)
+	retl
+	 nop
+END(bzero)
+#endif
+/*
+ * int copystr(const void *src, void *dst, size_t len, size_t *done)
+ */
+ENTRY(copystr)
+	_COPYSTR(%o0, %o1, %o2, %o3, E, E, E, E)
+	retl
+	 mov	%g1, %o0
+END(copystr)
+
+/*
+ * void *memcpy(void *dst, const void *src, size_t len)
+ */
+ENTRY(memcpy)
+	mov	%o0, %o3
+	_MEMCPY(%o3, %o1, %o2, E, E, E, E)
+	retl
+	 nop
+END(memcpy)
+
+/*
+ * void *memset(void *b, int c, size_t len)
+ */
+ENTRY(memset)
+	mov	%o0, %o3
+	_MEMSET(%o3, %o1, %o2, E, E)
+	retl
+	 nop
+END(memset)
+
+	.globl	copy_nofault_begin
+copy_nofault_begin:
+	nop
+#if 1
+/*
+ * int copyin(const void *uaddr, void *kaddr, size_t len)
+ */
+ENTRY(copyin)
+	wr	%g0, ASI_AIUS, %asi
+	_MEMCPY(%o1, %o0, %o2, E, E, a, %asi)
+	retl
+	 clr	%o0
+END(copyin)
+#endif
+/*
+ * int copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done)
+ */
+ENTRY(copyinstr)
+	wr	%g0, ASI_AIUS, %asi
+	_COPYSTR(%o0, %o1, %o2, %o3, a, %asi, E, E)
+	retl
+	 mov	%g1, %o0
+END(copyinstr)
+
+/*
+ * int copyout(const void *kaddr, void *uaddr, size_t len)
+ */
+ENTRY(copyout)
+	wr	%g0, ASI_AIUS, %asi
+	_MEMCPY(%o1, %o0, %o2, a, %asi, E, E)
+	retl
+	 clr	%o0
+END(copyout)
+
+	.globl	copy_nofault_end
+copy_nofault_end:
+	nop
+
+ENTRY(copy_fault)
+	retl
+	 mov	EFAULT, %o0
+END(copy_fault)
+
+	.globl	fs_nofault_begin
+fs_nofault_begin:
+	nop
+
+/*
+ * Chatty aliases for fetch, store functions.
+ */
+	.globl	fubyte, fusword, fuword, subyte, susword, suword
+	.set	fubyte, fuword8
+	.set	fusword, fuword16
+	.set	fuword, fuword64
+	.set	subyte, suword8
+	.set	susword, suword16
+	.set	suword, suword64
+
+	.globl	casuptr, fuptr, suptr
+	.set	casuptr, casuword64
+	.set	fuptr, fuword64
+	.set	suptr, suword64
+
+/*
+ * int32_t casuword32(volatile int32_t *p, int32_t e, int32_t s)
+ */
+ENTRY(casuword32)
+	casa	[%o0] ASI_AIUS, %o1, %o2
+	retl
+	 mov	%o2, %o0
+END(casuword32)
+
+/*
+ * int64_t casuword64(volatile int64_t *p, int64_t e, int64_t s)
+ */
+ENTRY(casuword64)
+	casxa	[%o0] ASI_AIUS, %o1, %o2
+	retl
+	 mov	%o2, %o0
+END(casuword64)
+
+/*
+ * int fuword8(const void *base)
+ */
+ENTRY(fuword8)
+	retl
+	 lduba	[%o0] ASI_AIUS, %o0
+END(fuword8)
+
+/*
+ * int fuword16(const void *base)
+ */
+ENTRY(fuword16)
+	retl
+	 lduha	[%o0] ASI_AIUS, %o0
+END(fuword16)
+
+/*
+ * int32_t fuword32(const void *base)
+ */
+ENTRY(fuword32)
+	retl
+	 lduwa	[%o0] ASI_AIUS, %o0
+END(fuword32)
+
+/*
+ * int64_t fuword64(const void *base)
+ */
+ENTRY(fuword64)
+	retl
+	 ldxa	[%o0] ASI_AIUS, %o0
+END(fuword64)
+
+/*
+ * int suword8(const void *base, int word)
+ */
+ENTRY(suword8)
+	stba	%o1, [%o0] ASI_AIUS
+	retl
+	 clr	%o0
+END(suword8)
+
+/*
+ * int suword16(const void *base, int word)
+ */
+ENTRY(suword16)
+	stha	%o1, [%o0] ASI_AIUS
+	retl
+	 clr	%o0
+END(suword16)
+
+/*
+ * int suword32(const void *base, int32_t word)
+ */
+ENTRY(suword32)
+	stwa	%o1, [%o0] ASI_AIUS
+	retl
+	 clr	%o0
+END(suword32)
+
+/*
+ * int suword64(const void *base, int64_t word)
+ */
+ENTRY(suword64)
+	stxa	%o1, [%o0] ASI_AIUS
+	retl
+	 clr	%o0
+END(suword64)
+
+	.globl	fs_nofault_intr_begin
+fs_nofault_intr_begin:
+	nop
+
+/*
+ * int fuswintr(const void *base)
+ */
+ENTRY(fuswintr)
+	retl
+	 lduha	[%o0] ASI_AIUS, %o0
+END(fuswintr)
+
+/*
+ * int suswintr(const void *base, int word)
+ */
+ENTRY(suswintr)
+	stha	%o1, [%o0] ASI_AIUS
+	retl
+	 clr	%o0
+END(suswintr)
+
+	.globl	fs_nofault_intr_end
+fs_nofault_intr_end:
+	nop
+
+	.globl	fs_nofault_end
+fs_nofault_end:
+	nop
+
+ENTRY(fs_fault)
+	retl
+	 mov	-1, %o0
+END(fsfault)
+
+	.globl	fas_nofault_begin
+fas_nofault_begin:
+
+/*
+ * int fasword8(u_long asi, uint64_t addr, uint8_t *val)
+ */
+ENTRY(fasword8)
+	wr	%o0, 0, %asi
+	membar	#Sync
+	lduba	[%o1] %asi, %o3
+	membar	#Sync
+	stb	%o3, [%o2]
+	retl
+	 clr	%o0
+END(fasword8)
+
+/*
+ * int fasword16(u_long asi, uint64_t addr, uint16_t *val)
+ */
+ENTRY(fasword16)
+	wr	%o0, 0, %asi
+	membar	#Sync
+	lduha	[%o1] %asi, %o3
+	membar	#Sync
+	sth	%o3, [%o2]
+	retl
+	 clr	%o0
+END(fasword16)
+
+/*
+ * int fasword32(u_long asi, uint64_t addr, uint32_t *val)
+ */
+ENTRY(fasword32)
+	wr	%o0, 0, %asi
+	membar	#Sync
+	lduwa	[%o1] %asi, %o3
+	membar	#Sync
+	stw	%o3, [%o2]
+	retl
+	 clr	%o0
+END(fasword32)
+
+	.globl	fas_nofault_end
+fas_nofault_end:
+	nop
+
+	.globl	fas_fault
+ENTRY(fas_fault)
+	retl
+	 mov	-1, %o0
+END(fas_fault)
+
+	.globl	fpu_fault_begin
+fpu_fault_begin:
+	nop
+
+	.globl	fpu_fault_end
+fpu_fault_end:
+	nop
+
+	.globl	fpu_fault_size
+	.set	fpu_fault_size, fpu_fault_end - fpu_fault_begin
+
+ENTRY(longjmp)
+	set	1, %g3
+	movrz	%o1, %o1, %g3
+	mov	%o0, %g1
+	ldx	[%g1 + _JB_FP], %g2
+1:	cmp	%fp, %g2
+	bl,a,pt	%xcc, 1b
+	 restore
+	bne,pn	%xcc, 2f
+	 ldx	[%g1 + _JB_SP], %o2
+	cmp	%o2, %sp
+	blt,pn	%xcc, 2f
+	 movge	%xcc, %o2, %sp
+	ldx	[%g1 + _JB_PC], %o7
+	retl
+	 mov	%g3, %o0
+2:	PANIC("longjmp botch", %l1)
+END(longjmp)
+
+ENTRY(setjmp)
+	stx	%sp, [%o0 + _JB_SP]
+	stx	%o7, [%o0 + _JB_PC]
+	stx	%fp, [%o0 + _JB_FP]
+	retl
+	 clr	%o0
+END(setjmp)
+
+/*
+ * void openfirmware(cell_t args[])
+ */
+ENTRY(openfirmware)
+	save	%sp, -CCFSZ, %sp
+	SET(ofw_vec, %l7, %l6)
+	ldx	[%l6], %l6
+	rdpr	%pil, %l7
+	wrpr	%g0, PIL_TICK, %pil
+	call	%l6
+	 mov	%i0, %o0
+	wrpr	%l7, 0, %pil
+	ret
+	 restore %o0, %g0, %o0
+END(openfirmware)
+
+#ifdef notyet
+/* SUN4V_FIXME - uses a now illegal ASI */
+/*
+ * void ofw_exit(cell_t args[])
+ */
+ENTRY(openfirmware_exit)
+	save	%sp, -CCFSZ, %sp
+	flushw
+	wrpr	%g0, PIL_TICK, %pil
+	SET(ofw_tba, %l7, %l5)
+	ldx	[%l5], %l5
+	wrpr	%l5, 0, %tba			! restore the ofw trap table
+	SET(ofw_vec, %l7, %l6)
+	ldx	[%l6], %l6
+	SET(kstack0 + KSTACK_PAGES * PAGE_SIZE - PCB_SIZEOF, %l7, %l0)
+	sub	%l0, SPOFF, %fp			! setup a stack in a locked page
+	sub	%l0, SPOFF + CCFSZ, %sp
+	mov     AA_DMMU_PCXR, %l3		! set context 0
+	stxa    %g0, [%l3] ASI_DMMU
+	membar	#Sync
+	wrpr	%g0, 0, %tl			! force trap level 0
+	call	%l6
+	 mov	%i0, %o0
+	! never to return
+END(openfirmware_exit)
+
+#endif
+ENTRY(set_mmfsa_scratchpad)
+	stxa	%o0, [%g0]ASI_SCRATCHPAD
+	retl
+	  nop
+END(set_mmfsa_scratchpad)
+ENTRY(set_pcpu_scratchpad)
+	mov     SCRATCH_REG_PCPU, %g2
+	stxa	%o0, [%g0 + %g2]ASI_SCRATCHPAD
+	retl
+	  nop
+END(set_pcpu_scratchpad)
+ENTRY(set_hash_kernel_scratchpad)
+	mov     SCRATCH_REG_HASH_KERNEL, %g2
+	stxa	%o0, [%g0 + %g2]ASI_SCRATCHPAD
+	retl
+	  nop
+END(set_hash_kernel_scratchpad)
+ENTRY(set_tsb_kernel_scratchpad)
+	mov     SCRATCH_REG_TSB_KERNEL, %g2
+	stxa	%o0, [%g0 + %g2]ASI_SCRATCHPAD
+	retl
+	  nop
+END(set_tsb_kernel_scratchpad)
+
+ENTRY(set_hash_user_scratchpad)
+	mov     SCRATCH_REG_HASH_USER, %g2
+	stxa	%o0, [%g0 + %g2]ASI_SCRATCHPAD
+	retl
+	  nop
+END(set_hash_user_scratchpad)
+ENTRY(set_tsb_user_scratchpad)
+	mov     SCRATCH_REG_TSB_USER, %g2
+	stxa	%o0, [%g0 + %g2]ASI_SCRATCHPAD
+	retl
+	  nop
+END(set_tsb_user_scratchpad)
+	
+		
+/*
+ * %o0 ra - real address
+ * [%o1] lo - lower 64-bits
+ * [%o2] hi - upper 64-bits
+ */
+	
+ENTRY(load_real_dw)
+	save	%sp, -SA(MINFRAME64), %sp
+	ldda [%g0 + %i0]ASI_LDTD_REAL, %l0
+	stx %l0, [%i1]
+      	stx %l1, [%i2]
+	restore
+	retl
+	  nop
+END(load_real_dw)
+
+/*
+ * %o0 = vaddr
+ * %o1 = ctxnum
+ */		
+ENTRY(invlpg)
+	mov	%o0, %o2
+	mov	%o1, %o3
+	mov	%g0, %o0
+	mov	%g0, %o1
+	mov	MAP_ITLB | MAP_DTLB, %o4
+	mov	MMU_DEMAP_PAGE, %o5
+	ta	FAST_TRAP
+	brz,pt	%o0, 1f
+	  nop
+	ba	panic_bad_hcall
+	  mov	MMU_DEMAP_PAGE, %o1
+1:
+	retl
+	  nop
+END(invlpg)
+
+
+/*
+ * %o0 = ctxnum
+ * 
+ */		
+ENTRY(invlctx)
+	cmp	%o0, %g0
+	bne	%xcc, 2f
+	  nop
+	MAGIC_TRAP_ON
+	MAGIC_EXIT
+2:		
+	mov	%o0, %o2
+	mov	%g0, %o0
+	mov	%g0, %o1
+	mov	MAP_ITLB | MAP_DTLB, %o3
+	mov	MMU_DEMAP_CTX, %o5
+	ta	FAST_TRAP
+	brz,pt	%o0, 1f
+	  nop
+	ba	panic_bad_hcall
+	  mov	MMU_DEMAP_CTX, %o1
+1:
+	retl
+	  nop
+END(invlctx)
+
+ENTRY(invltlb)
+	mov	%g0, %o0
+	mov	%g0, %o1
+	mov	MAP_ITLB | MAP_DTLB, %o2
+	mov	MMU_DEMAP_ALL, %o5
+	ta	FAST_TRAP
+	brz,pt	%o0, 1f
+	  nop
+	ba	panic_bad_hcall
+	  mov	MMU_DEMAP_ALL, %o1
+1:
+	retl
+	  nop
+END(invltlb)
+
+! %o0 = pa
+! %o1 = size
+	
+ENTRY(bzerophyspage)
+	save	%sp, -SA(MINFRAME64), %sp
+	mov	8, %l1
+	mov	16, %l2
+	mov	24, %l3
+	mov	32, %l4
+	mov	40, %l5
+	mov	48, %l6
+	mov	56, %l7
+1:	stxa	%g0, [%i0]ASI_REAL
+	stxa	%g0, [%i0 + %l1]ASI_REAL
+	stxa	%g0, [%i0 + %l2]ASI_REAL
+	stxa	%g0, [%i0 + %l3]ASI_REAL
+	stxa	%g0, [%i0 + %l4]ASI_REAL
+	stxa	%g0, [%i0 + %l5]ASI_REAL
+	stxa	%g0, [%i0 + %l6]ASI_REAL
+	stxa	%g0, [%i0 + %l7]ASI_REAL
+	sub	%i1, 64, %i1
+	brnz,pt	%i1, 1b
+	  add   %i0, 64, %i0
+	membar #Sync
+	ret
+          restore
+
+END(bzerophyspage)
+
+ENTRY(init_mondo)
+	ldx	[PCPU(MONDO_DATA)], %g2
+	stx	%o0, [%g2]
+        stx     %o1, [%g2+8]
+        stx     %o2, [%g2+0x10]
+        stx     %o3, [%g2+0x18]
+        stx     %g0, [%g2+0x20]
+        stx     %g0, [%g2+0x28]
+        stx     %g0, [%g2+0x30]
+        stx     %g0, [%g2+0x38]
+        retl
+	 membar #Sync 
+END(init_mondo)
+
+
+ENTRY(init_mondo_queue)			
+        mov     CPU_MONDO_QUEUE_TAIL, %g2     
+        ldxa    [%g2]ASI_QUEUE, %g2     
+        mov     CPU_MONDO_QUEUE_HEAD, %g6     
+        stxa    %g2, [%g6]ASI_QUEUE
+        mov     DEV_MONDO_QUEUE_TAIL, %g2     
+        ldxa    [%g2]ASI_QUEUE, %g2     
+        mov     DEV_MONDO_QUEUE_HEAD, %g6     
+        stxa    %g2, [%g6]ASI_QUEUE 
+	retl
+	  membar #Sync    
+END(init_mondo_queue)
+	
+#ifdef GPROF
+
+ENTRY(user)
+	nop
+
+ENTRY(btrap)
+	nop
+
+ENTRY(etrap)
+	nop
+
+ENTRY(bintr)
+	nop
+
+ENTRY(eintr)
+	nop
+
+
+/*
+ * XXX including sys/gmon.h in genassym.c is not possible due to uintfptr_t
+ * badness.
+ */
+#define	GM_STATE	0x0
+#define	GMON_PROF_OFF	3
+#define	GMON_PROF_HIRES	4
+
+	.globl	_mcount
+	.set	_mcount, __cyg_profile_func_enter
+
+ENTRY(__cyg_profile_func_enter)
+	SET(_gmonparam, %o3, %o2)
+	lduw    [%o2 + GM_STATE], %o3
+	cmp     %o3, GMON_PROF_OFF
+	be,a,pn %icc, 1f
+	 nop
+	SET(mcount, %o3, %o2)
+	jmpl	%o2, %g0
+	 nop
+1:      retl
+	 nop
+END(__cyg_profile_func_enter)
+
+#ifdef GUPROF
+
+ENTRY(__cyg_profile_func_exit)
+	SET(_gmonparam, %o3, %o2)
+	lduw    [%o2 + GM_STATE], %o3
+	cmp     %o3, GMON_PROF_HIRES
+	be,a,pn %icc, 1f
+	 nop
+	SET(mexitcount, %o3, %o2)
+	jmpl	%o2, %g0
+	 nop
+1:      retl
+	 nop
+END(__cyg_profile_func_exit)
+
+#endif /* GUPROF */
+
+#endif /* GPROF */
diff --git a/sys/sun4v/sun4v/swtch.S b/sys/sun4v/sun4v/swtch.S
new file mode 100644
index 000000000000..9c3711947220
--- /dev/null
+++ b/sys/sun4v/sun4v/swtch.S
@@ -0,0 +1,255 @@
+/*-
+ * Copyright (c) 2001 Jake Burkholder.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <machine/asm.h>
+__FBSDID("$FreeBSD$");
+
+#include <machine/asi.h>
+#include <machine/asmacros.h>
+#include <machine/ktr.h>
+#include <machine/tstate.h>
+#include <machine/hypervisorvar.h>
+
+#include "assym.s"
+
+	.register	%g2, #ignore
+	.register	%g3, #ignore
+
+#define	PCB_REG	%g6
+
+#define	MEMBAR_PRE	membar	#LoadStore|#StoreStore
+#define MEMBAR_POST	membar	#LoadLoad
+
+#define ATOMIC_CLEAR_INT_BIT(addr, old, bit, new, label) \
+	MEMBAR_PRE		; \
+	ld	[addr], old	; \
+label:	andn	old, bit, new	; \
+	cas	[addr], old, new ; \
+	cmp	old, new	; \
+	bne,a,pn	%icc, label ## b ; \
+	  ld	[addr], old	; \
+	MEMBAR_POST
+	
+#define ATOMIC_SET_INT_BIT(addr, old, bit, new, label) \
+	MEMBAR_PRE		; \
+	ld	[addr], old	; \
+label:	or	old, bit, new	; \
+	cas	[addr], old, new ; \
+	cmp	old, new	; \
+	bne,a,pn	%icc, label ## b ; \
+	  ld	[addr], old	; \
+	MEMBAR_POST	
+/*
+ * void cpu_throw(struct thread *old, struct thread *new)
+ */
+ENTRY(cpu_throw)
+	save	%sp, -CCFSZ, %sp
+	flushw
+	ba	%xcc, .Lsw1
+	 mov	%i1, %i0
+END(cpu_throw)
+	
+/*
+ * void cpu_switch(struct thread *old, struct thread *new)
+ */
+ENTRY(cpu_switch)
+	save	%sp, -CCFSZ, %sp
+	ldx	[PCPU(CURPCB)], PCB_REG
+	mov	%i1, %i0
+
+	/*
+	 * If the current thread was using floating point in the kernel, save
+	 * its context.  The userland floating point context has already been
+	 * saved in that case.
+	 */
+	rd	%fprs, %l2
+	andcc	%l2, FPRS_FEF, %g0
+	bz,a,pt %xcc, 1f
+	 nop
+	call	savefpctx
+	 add	PCB_REG, PCB_KFP, %o0
+	ba,a	%xcc, 2f
+	 nop
+
+	/*
+	 * If the current thread was using floating point in userland, save
+	 * its context.
+	 */
+1:	sub	PCB_REG, TF_SIZEOF, %l2
+	ldx	[%l2 + TF_FPRS], %l3
+	andcc	%l3, FPRS_FEF, %g0
+	bz,a,pt	%xcc, 2f
+	 nop
+	call	savefpctx
+	 add	PCB_REG, PCB_UFP, %o0
+	andn	%l3, FPRS_FEF, %l3
+	stx	%l3, [%l2 + TF_FPRS]
+
+	ldx	[PCB_REG + PCB_FLAGS], %l3
+	or	%l3, PCB_FEF, %l3
+	stx	%l3, [PCB_REG + PCB_FLAGS]
+
+	/*
+	 * Flush the windows out to the stack and save the current frame
+	 * pointer and program counter.
+	 */
+2:	flushw
+	stx	%fp, [PCB_REG + PCB_SP]
+	stx	%i7, [PCB_REG + PCB_PC]
+
+
+	/*
+	 * Load the new thread's frame pointer and program counter, and set
+	 * the current thread and pcb.
+	 */
+.Lsw1:
+#if KTR_COMPILE & KTR_PROC & 0
+	CATR(KTR_PROC, "cpu_switch: new td=%p pc=%#lx fp=%#lx"
+	    , %g1, %g2, %g3, 7, 8, 9)
+	stx	%i0, [%g1 + KTR_PARM1]
+	ldx	[%i0 + TD_PCB], %g2
+	ldx	[%g2 + PCB_PC], %g3
+	stx	%g3, [%g1 + KTR_PARM2]
+	ldx	[%g2 + PCB_SP], %g3
+	stx	%g3, [%g1 + KTR_PARM3]
+9:
+#endif
+	
+	ldx	[%i0 + TD_PCB], %i1
+
+	stx	%i0, [PCPU(CURTHREAD)]
+	stx	%i1, [PCPU(CURPCB)]
+
+	mov	%i1, PCB_REG				! load in new PCB
+
+	ldx	[PCB_REG + PCB_SP], %fp
+	ldx	[PCB_REG + PCB_PC], %i7
+	sub	%fp, CCFSZ, %sp
+
+ 	/*
+	 * Point to the pmaps of the new process, and of the last non-kernel
+	 * process to run.
+	 */
+	ldx	[%i0 + TD_PROC], %i2
+	ldx	[PCPU(PMAP)], %l2
+	ldx	[%i2 + P_VMSPACE], %i5
+	add	%i5, VM_PMAP, %i2
+
+#if KTR_COMPILE & KTR_PROC & 0
+	CATR(KTR_PROC, "cpu_switch: new pmap=%p old pmap=%p"
+	    , %g1, %g2, %g3, 7, 8, 9)
+	stx	%i2, [%g1 + KTR_PARM1]
+	stx	%l2, [%g1 + KTR_PARM2]
+9:
+#endif
+	/*
+	 * If they are the same we are done.
+	 */
+	cmp	%l2, %i2			! current pmap == new pmap?
+	be,a,pn %xcc, 5f
+	  nop
+	ldx	[%i2 + PM_CONTEXT], %l5		! new context
+	ldx	[%l2 + PM_CONTEXT], %l3		! old context
+	cmp	%g0, %l5
+	be,pn	%xcc, 5f			! new context == kernel?
+	  ld	[PCPU(CPUMASK)], %l4
+
+	brz	%l3, 10f			! old context == kernel?
+	  nop
+	
+	/*
+	 * Mark the old pmap as no longer active on this CPU
+	 */
+	add	%l2, PM_ACTIVE, %l2
+	ATOMIC_CLEAR_INT_BIT(%l2, %l3, %l4, %l6, 8)	
+	
+10:
+	add	%i2,  PM_ACTIVE, %i3
+	ATOMIC_SET_INT_BIT(%i3, %l3, %l4, %l6, 11)
+
+	add	%i2, PM_TLBACTIVE, %i3
+	ATOMIC_SET_INT_BIT(%i3, %l3, %l4, %l6, 12)
+	
+	mov	SCRATCH_REG_HASH_USER, %l6
+	mov	SCRATCH_REG_TSB_USER, %l7
+	ldx	[%i2 + PM_HASHSCRATCH], %l3	
+	ldx	[%i2 + PM_TSBSCRATCH], %l4
+	stx	%i2, [PCPU(PMAP)]
+
+	SET_SCRATCH(%l6,%l3)			! XXX we're assuming the 
+	SET_SCRATCH(%l7,%l4)			! scratch values <= 32 bits
+
+
+	ldx	[%i2 + PM_TSB_RA], %l3
+
+	mov	MMU_CID_S, %l6
+
+	mov	1, %o0
+	mov	%l3, %o1
+	mov	MMU_TSB_CTXNON0, %o5
+	ta	FAST_TRAP
+	cmp	%g0, %o0
+	be	%xcc, 4f
+	  nop
+	MAGIC_TRAP_ON
+	MAGIC_EXIT
+	
+4:	/*
+	 * install the new secondary context number in the cpu.
+	 */
+	
+	SET_MMU_CONTEXT(%l6, %l5)
+	membar #Sync
+	/*
+	 * Done.  Return and load the new process's window from the stack.
+	 */
+5:	ret
+	 restore
+END(cpu_switch)
+
+ENTRY(savectx)
+	save	%sp, -CCFSZ, %sp
+	flushw
+	call	savefpctx
+	 add	%i0, PCB_UFP, %o0
+	stx	%fp, [%i0 + PCB_SP]
+	stx	%i7, [%i0 + PCB_PC]
+	ret
+	 restore %g0, 0, %o0
+END(savectx)
+
+/*
+ * void savefpctx(uint32_t *);
+ */
+ENTRY(savefpctx)
+	wr	%g0, FPRS_FEF, %fprs
+	std	%f0, [%o0 + (0 * 64)] 
+	std	%f16, [%o0 + (1 * 64)] 
+	std	%f32, [%o0 + (2 * 64)] 
+	std	%f48, [%o0 + (3 * 64)] 
+	retl
+	 wr	%g0, 0, %fprs
+END(savefpctx)
diff --git a/sys/sun4v/sun4v/sys_machdep.c b/sys/sun4v/sun4v/sys_machdep.c
new file mode 100644
index 000000000000..9ac5fbc212a4
--- /dev/null
+++ b/sys/sun4v/sun4v/sys_machdep.c
@@ -0,0 +1,130 @@
+/*-
+ * Copyright (c) 2001 Jake Burkholder.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/lock.h>
+#include <sys/malloc.h>
+#include <sys/mutex.h>
+#include <sys/proc.h>
+#include <sys/sysproto.h>
+
+#include <machine/md_var.h>
+#include <machine/utrap.h>
+#include <machine/sysarch.h>
+
+static int sparc_sigtramp_install(struct thread *td, char *args);
+static int sparc_utrap_install(struct thread *td, char *args);
+
+#ifndef	_SYS_SYSPROTO_H_
+struct sysarch_args {
+	int	op;
+	char	*parms;
+};
+#endif
+
+int
+sysarch(struct thread *td, struct sysarch_args *uap)
+{
+	int error;
+
+	mtx_lock(&Giant);
+	switch (uap->op) {
+	case SPARC_SIGTRAMP_INSTALL:
+		error = sparc_sigtramp_install(td, uap->parms);
+		break;
+	case SPARC_UTRAP_INSTALL:
+		error = sparc_utrap_install(td, uap->parms);
+		break;
+	default:
+		error = EINVAL;
+		break;
+	}
+	mtx_unlock(&Giant);
+	return (error);
+}
+
+static int
+sparc_sigtramp_install(struct thread *td, char *args)
+{
+	struct sparc_sigtramp_install_args sia;
+	struct proc *p;
+	int error;
+
+	p = td->td_proc;
+	if ((error = copyin(args, &sia, sizeof(sia))) != 0)
+		return (error);
+	if (sia.sia_old != NULL) {
+		if (suword(sia.sia_old, (long)p->p_md.md_sigtramp) != 0)
+			return (EFAULT);
+	}
+	p->p_md.md_sigtramp = sia.sia_new;
+	return (0);
+}
+
+static int
+sparc_utrap_install(struct thread *td, char *args)
+{
+	struct sparc_utrap_install_args uia;
+	struct sparc_utrap_args ua;
+	struct md_utrap *ut;
+	int error;
+	int i;
+
+	ut = td->td_proc->p_md.md_utrap;
+	if ((error = copyin(args, &uia, sizeof(uia))) != 0)
+		return (error);
+	if (uia.num < 0 || uia.num > UT_MAX ||
+	    (uia.handlers == NULL && uia.num > 0))
+		return (EINVAL);
+	for (i = 0; i < uia.num; i++) {
+		if ((error = copyin(&uia.handlers[i], &ua, sizeof(ua))) != 0)
+			return (error);
+		if (ua.type != UTH_NOCHANGE &&
+		    (ua.type < 0 || ua.type >= UT_MAX))
+			return (EINVAL);
+		if (ua.old_deferred != NULL) {
+			if ((error = suword(ua.old_deferred, 0)) != 0)
+				return (error);
+		}
+		if (ua.old_precise != NULL) {
+			error = suword(ua.old_precise,
+			    ut != NULL ? (long)ut->ut_precise[ua.type] : 0);
+			if (error != 0)
+				return (error);
+		}
+		if (ua.type != UTH_NOCHANGE) {
+			if (ut == NULL) {
+				ut = utrap_alloc();
+				td->td_proc->p_md.md_utrap = ut;
+			}
+			ut->ut_precise[ua.type] = ua.new_precise;
+		}
+	}
+	return (0);
+}
diff --git a/sys/sun4v/sun4v/t1_copy.S b/sys/sun4v/sun4v/t1_copy.S
new file mode 100644
index 000000000000..23bdeeb4f386
--- /dev/null
+++ b/sys/sun4v/sun4v/t1_copy.S
@@ -0,0 +1,1599 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License").  You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http:	//www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information:	 Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#include <machine/asm.h>
+__FBSDID("$FreeBSD$")
+	
+#include <machine/asi.h>
+#include <machine/asmacros.h>
+#include <machine/ktr.h>
+#include <machine/pstate.h>
+#include <machine/trap.h>
+#include <machine/tstate.h>
+#include <machine/wstate.h>
+#include <machine/hypervisorvar.h>
+#include <machine/errata.h>
+
+       .register %g2,#ignore
+       .register %g3,#ignore
+       .register %g6,#ignore
+       .register %g7,#ignore
+		
+	
+/*
+ * This define is to align data for the unaligned source cases.
+ * The data1, data2 and data3 is merged into data1 and data2.
+ * The data3 is preserved for next merge.
+ */
+#define	ALIGN_DATA(data1, data2, data3, lshift, rshift, tmp)	\
+	sllx	data1, lshift, data1				;\
+	srlx	data2, rshift, tmp				;\
+	or	data1, tmp, data1				;\
+	sllx	data2, lshift, data2				;\
+	srlx	data3, rshift, tmp				;\
+	or	data2, tmp, data2
+/*
+ * This macro is to align the data. Basically it merges
+ * data1 and data2 to form double word.
+ */
+#define	ALIGN_DATA_EW(data1, data2, lshift, rshift, tmp)	\
+	sllx	data1, lshift, data1				;\
+	srlx	data2, rshift, tmp				;\
+	or	data1, tmp, data1
+
+
+
+	
+	
+/*
+ * DGDEF and DGDEF2 provide global data declarations.
+ *
+ * DGDEF provides a word aligned word of storage.
+ *
+ * DGDEF2 allocates "sz" bytes of storage with **NO** alignment.  This
+ * implies this macro is best used for byte arrays.
+ *
+ * DGDEF3 allocates "sz" bytes of storage with "algn" alignment.
+ */
+#define DGDEF2(name, sz) \
+        .section        ".data"	;  \
+        .global name	;  \
+        .type   name, @object ;  \
+        .size   name, sz;  \
+name:
+
+#define DGDEF3(name, sz, algn) \
+        .section        ".data"	;  \
+        .align  algn	;  \
+        .global name	;  \
+        .type   name, @object ;  \
+        .size   name, sz;  \
+name:
+
+#define DGDEF(name)     DGDEF3(name, 4, 4)
+	
+.align  4
+DGDEF(hw_copy_limit_1)
+.word   0x100
+DGDEF(hw_copy_limit_2)
+.word   0x200
+DGDEF(hw_copy_limit_4)
+.word   0x400
+DGDEF(hw_copy_limit_8)
+.word   0x400
+.align  64
+.section ".text"
+	
+
+#if defined(lint)
+
+/*ARGSUSED*/
+void
+ovbcopy(const void *from, void *to, size_t count)
+{}
+
+#else	/* lint */
+
+ENTRY(bcopy)
+	tst	%o2			! check count
+	bgu,a	%xcc, 1f		! nothing to do or bad arguments
+	subcc	%o0, %o1, %o3		! difference of from and to address
+
+	retl				! return
+	  nop
+1:
+	bneg,a	%xcc, 2f
+	neg	%o3			! if < 0, make it positive
+2:	cmp	%o2, %o3		! cmp size and abs(from - to)
+	bleu	%xcc, novbcopy		! if size <= abs(diff): use bcopy,
+	  nop
+	cmp	%o0, %o1		! compare from and to addresses
+	blu	%xcc, ov_bkwd		! if from < to, copy backwards
+	  nop
+	!
+	! Copy forwards.
+	!
+ov_fwd:
+	ldub	[%o0], %o3		! read from address
+	inc	%o0			! inc from address
+	stb	%o3, [%o1]		! write to address
+	deccc	%o2			! dec count
+	bgu	%xcc, ov_fwd		! loop till done
+	  inc	%o1			! inc to address
+
+	retl				! return
+	nop
+	!
+	! Copy backwards.
+	!
+ov_bkwd:
+	deccc	%o2			! dec count
+	ldub	[%o0 + %o2], %o3	! get byte at end of src
+	bgu	%xcc, ov_bkwd		! loop till done
+	  stb	%o3, [%o1 + %o2]	! delay slot, store at end of dst
+
+	retl				! return
+	nop
+	SET_SIZE(bcopy)
+
+#endif	/* lint */
+
+
+		
+/*
+ * Copy a block of storage - must not overlap (from + len <= to).
+ */
+ENTRY(novbcopy)
+
+	save	%sp, -SA(MINFRAME), %sp
+
+do_copy:
+	cmp	%i2, 12			! for small counts
+	blu	%xcc, bytecp		! just copy bytes
+	  nop
+
+	cmp	%i2, 128		! for less than 128 bytes
+	blu,pn	%xcc, bcb_punt		! no block st/quad ld
+	  nop
+#if 0
+	set	use_hw_bcopy, %o2
+	ld	[%o2], %o2
+	tst	%o2
+	bz	bcb_punt
+	  nop
+#endif
+	subcc	%i1, %i0, %i3
+	bneg,a,pn %xcc, 1f
+	  neg	%i3
+1:
+	/*
+	 * Compare against 256 since we should be checking block addresses
+	 * and (dest & ~63) - (src & ~63) can be 3 blocks even if
+	 * src = dest + (64 * 3) + 63.
+	 */
+	cmp	%i3, 256
+	blu,pn	%xcc, bcb_punt
+	  nop
+
+	/*
+	 * Copy that reach here have at least 2 blocks of data to copy.
+	 */
+do_blockcopy:
+	! Swap src/dst since the code below is memcpy code
+	! and memcpy/bcopy have different calling sequences
+	mov	%i1, %i5
+	mov	%i0, %i1
+	mov	%i5, %i0
+
+	andcc	%i0, 0x3f, %i3		! is dst aligned on a 64 bytes
+	bz	%xcc, chksrc		! dst is already double aligned
+	  sub	%i3, 0x40, %i3
+	neg	%i3			! bytes till dst 64 bytes aligned
+	sub	%i2, %i3, %i2		! update i2 with new count
+
+1:	ldub	[%i1], %i4
+	stb	%i4, [%i0]
+	inc	%i1
+	deccc	%i3
+	bgu	%xcc, 1b
+	  inc	%i0
+
+	! Now Destination is block (64 bytes) aligned
+chksrc:
+	andn	%i2, 0x3f, %i3		! %i3 count is multiple of block size
+	sub	%i2, %i3, %i2		! Residue bytes in %i2
+
+	wr	%g0, ASI_LDSTBI_P, %asi
+
+	andcc	%i1, 0xf, %o2		! is src quadword aligned
+	bz,pn	%xcc, blkcpy		! src offset in %o2
+	nop
+	cmp	%o2, 0x8
+	bg	cpy_upper_double
+	nop
+	bl	cpy_lower_double
+	nop
+
+	! Falls through when source offset is equal to 8 i.e.
+	! source is double word aligned.
+	! In this case no shift/merge of data is required
+	sub	%i1, %o2, %i1		! align the src at 16 bytes.
+	andn	%i1, 0x3f, %l0		! %l0 has block aligned source
+	prefetch [%l0+0x0], #one_read
+	ldda	[%i1+0x0]%asi, %l2
+loop0:
+	ldda	[%i1+0x10]%asi, %l4
+	prefetch [%l0+0x40], #one_read
+
+	stxa	%l3, [%i0+0x0]%asi
+	stxa	%l4, [%i0+0x8]%asi
+
+	ldda	[%i1+0x20]%asi, %l2
+	stxa	%l5, [%i0+0x10]%asi
+	stxa	%l2, [%i0+0x18]%asi
+
+	ldda	[%i1+0x30]%asi, %l4
+	stxa	%l3, [%i0+0x20]%asi
+	stxa	%l4, [%i0+0x28]%asi
+
+	ldda	[%i1+0x40]%asi, %l2
+	stxa	%l5, [%i0+0x30]%asi
+	stxa	%l2, [%i0+0x38]%asi
+
+	add	%l0, 0x40, %l0
+	add	%i1, 0x40, %i1
+	subcc	%i3, 0x40, %i3
+	bgu,pt	%xcc, loop0
+	  add	%i0, 0x40, %i0
+	ba	blkdone
+	add	%i1, %o2, %i1		! increment the source by src offset
+					! the src offset was stored in %o2
+
+cpy_lower_double:
+	sub	%i1, %o2, %i1		! align the src at 16 bytes.
+	sll	%o2, 3, %o0		! %o0 left shift
+	mov	0x40, %o1
+	sub	%o1, %o0, %o1		! %o1 right shift = (64 - left shift)
+	andn	%i1, 0x3f, %l0		! %l0 has block aligned source
+	prefetch [%l0+0x0], #one_read
+	ldda	[%i1+0x0]%asi, %l2	! partial data in %l2 and %l3 has
+					! complete data
+loop1:
+	ldda	[%i1+0x10]%asi, %l4	! %l4 has partial data for this read.
+	ALIGN_DATA(%l2, %l3, %l4, %o0, %o1, %l6)	! merge %l2, %l3 and %l4
+							! into %l2 and %l3
+	prefetch [%l0+0x40], #one_read
+	stxa	%l2, [%i0+0x0]%asi
+	stxa	%l3, [%i0+0x8]%asi
+
+	ldda	[%i1+0x20]%asi, %l2
+	ALIGN_DATA(%l4, %l5, %l2, %o0, %o1, %l6)	! merge %l2 with %l5 and
+	stxa	%l4, [%i0+0x10]%asi			! %l4 from previous read
+	stxa	%l5, [%i0+0x18]%asi			! into %l4 and %l5
+
+	! Repeat the same for next 32 bytes.
+
+	ldda	[%i1+0x30]%asi, %l4
+	ALIGN_DATA(%l2, %l3, %l4, %o0, %o1, %l6)
+	stxa	%l2, [%i0+0x20]%asi
+	stxa	%l3, [%i0+0x28]%asi
+
+	ldda	[%i1+0x40]%asi, %l2
+	ALIGN_DATA(%l4, %l5, %l2, %o0, %o1, %l6)
+	stxa	%l4, [%i0+0x30]%asi
+	stxa	%l5, [%i0+0x38]%asi
+
+	add	%l0, 0x40, %l0
+	add	%i1, 0x40, %i1
+	subcc	%i3, 0x40, %i3
+	bgu,pt	%xcc, loop1
+	  add	%i0, 0x40, %i0
+	ba	blkdone
+	add	%i1, %o2, %i1		! increment the source by src offset
+					! the src offset was stored in %o2
+
+cpy_upper_double:
+	sub	%i1, %o2, %i1		! align the src at 16 bytes.
+	mov	0x8, %o0
+	sub	%o2, %o0, %o0
+	sll	%o0, 3, %o0		! %o0 left shift
+	mov	0x40, %o1
+	sub	%o1, %o0, %o1		! %o1 right shift = (64 - left shift)
+	andn	%i1, 0x3f, %l0		! %l0 has block aligned source
+	prefetch [%l0+0x0], #one_read
+	ldda	[%i1+0x0]%asi, %l2	! partial data in %l3 for this read and
+					! no data in %l2
+loop2:
+	ldda	[%i1+0x10]%asi, %l4	! %l4 has complete data and %l5 has
+					! partial
+	ALIGN_DATA(%l3, %l4, %l5, %o0, %o1, %l6)	! merge %l3, %l4 and %l5
+							! into %l3 and %l4
+	prefetch [%l0+0x40], #one_read
+	stxa	%l3, [%i0+0x0]%asi
+	stxa	%l4, [%i0+0x8]%asi
+
+	ldda	[%i1+0x20]%asi, %l2
+	ALIGN_DATA(%l5, %l2, %l3, %o0, %o1, %l6)	! merge %l2 and %l3 with
+	stxa	%l5, [%i0+0x10]%asi			! %l5 from previous read
+	stxa	%l2, [%i0+0x18]%asi			! into %l5 and %l2
+
+	! Repeat the same for next 32 bytes.
+
+	ldda	[%i1+0x30]%asi, %l4
+	ALIGN_DATA(%l3, %l4, %l5, %o0, %o1, %l6)
+	stxa	%l3, [%i0+0x20]%asi
+	stxa	%l4, [%i0+0x28]%asi
+
+	ldda	[%i1+0x40]%asi, %l2
+	ALIGN_DATA(%l5, %l2, %l3, %o0, %o1, %l6)
+	stxa	%l5, [%i0+0x30]%asi
+	stxa	%l2, [%i0+0x38]%asi
+
+	add	%l0, 0x40, %l0
+	add	%i1, 0x40, %i1
+	subcc	%i3, 0x40, %i3
+	bgu,pt	%xcc, loop2
+	  add	%i0, 0x40, %i0
+	ba	blkdone
+	add	%i1, %o2, %i1		! increment the source by src offset
+					! the src offset was stored in %o2
+
+
+	! Both Source and Destination are block aligned.
+	! Do fast copy using ASI_LDSTBI_P
+blkcpy:
+	prefetch [%i1+0x0], #one_read
+1:
+	ldda	[%i1+0x0]%asi, %l0
+	ldda	[%i1+0x10]%asi, %l2
+	prefetch [%i1+0x40], #one_read
+
+	stxa	%l0, [%i0+0x0]%asi
+	ldda	[%i1+0x20]%asi, %l4
+	ldda	[%i1+0x30]%asi, %l6
+
+	stxa	%l1, [%i0+0x8]%asi
+	stxa	%l2, [%i0+0x10]%asi
+	stxa	%l3, [%i0+0x18]%asi
+	stxa	%l4, [%i0+0x20]%asi
+	stxa	%l5, [%i0+0x28]%asi
+	stxa	%l6, [%i0+0x30]%asi
+	stxa	%l7, [%i0+0x38]%asi
+
+	add	%i1, 0x40, %i1
+	subcc	%i3, 0x40, %i3
+	bgu,pt	%xcc, 1b
+	  add	%i0, 0x40, %i0
+
+blkdone:
+	tst	%i2
+	bz,pt	%xcc, blkexit
+	nop
+
+residue:
+	ldub	[%i1], %i4
+	stb	%i4, [%i0]
+	inc	%i1
+	deccc	%i2
+	bgu	%xcc, residue
+	  inc	%i0
+
+blkexit:
+	membar	#Sync				! sync error barrier
+	ret
+	restore	%g0, 0, %o0
+
+bcb_punt:
+	!
+	! use aligned transfers where possible
+	!
+	xor	%i0, %i1, %o4		! xor from and to address
+	btst	7, %o4			! if lower three bits zero
+	bz	aldoubcp		! can align on double boundary
+	  nop	! assembler complaints about label
+
+	xor	%i0, %i1, %o4		! xor from and to address
+	btst	3, %o4			! if lower two bits zero
+	bz	alwordcp		! can align on word boundary
+	btst	3, %i0			! delay slot, from address unaligned?
+	!
+	! use aligned reads and writes where possible
+	! this differs from wordcp in that it copes
+	! with odd alignment between source and destnation
+	! using word reads and writes with the proper shifts
+	! in between to align transfers to and from memory
+	! i0 - src address, i1 - dest address, i2 - count
+	! i3, i4 - tmps for used generating complete word
+	! i5 (word to write)
+	! l0 size in bits of upper part of source word (US)
+	! l1 size in bits of lower part of source word (LS = 32 - US)
+	! l2 size in bits of upper part of destination word (UD)
+	! l3 size in bits of lower part of destination word (LD = 32 - UD)
+	! l4 number of bytes leftover after aligned transfers complete
+	! l5 the number 32
+	!
+	mov	32, %l5			! load an oft-needed constant
+	bz	align_dst_only
+	  btst	3, %i1			! is destnation address aligned?
+	clr	%i4			! clear registers used in either case
+	bz	align_src_only
+	  clr	%l0
+	!
+	! both source and destination addresses are unaligned
+	!
+1:					! align source
+	ldub	[%i0], %i3		! read a byte from source address
+	add	%i0, 1, %i0		! increment source address
+	or	%i4, %i3, %i4		! or in with previous bytes (if any)
+	btst	3, %i0			! is source aligned?
+	add	%l0, 8, %l0		! increment size of upper source (US)
+	bnz,a	1b
+	  sll	%i4, 8, %i4		! make room for next byte
+
+	sub	%l5, %l0, %l1		! generate shift left count (LS)
+	sll	%i4, %l1, %i4		! prepare to get rest
+	ld	[%i0], %i3		! read a word
+	add	%i0, 4, %i0		! increment source address
+	srl	%i3, %l0, %i5		! upper src bits into lower dst bits
+	or	%i4, %i5, %i5		! merge
+	mov	24, %l3			! align destination
+1:
+	srl	%i5, %l3, %i4		! prepare to write a single byte
+	stb	%i4, [%i1]		! write a byte
+	add	%i1, 1, %i1		! increment destination address
+	sub	%i2, 1, %i2		! decrement count
+	btst	3, %i1			! is destination aligned?
+	bnz,a	1b
+	  sub	%l3, 8, %l3		! delay slot, decrement shift count (LD)
+	sub	%l5, %l3, %l2		! generate shift left count (UD)
+	sll	%i5, %l2, %i5		! move leftover into upper bytes
+	cmp	%l2, %l0		! cmp # reqd to fill dst w old src left
+	bgu	%xcc, more_needed	! need more to fill than we have
+	  nop
+
+	sll	%i3, %l1, %i3		! clear upper used byte(s)
+	srl	%i3, %l1, %i3
+	! get the odd bytes between alignments
+	sub	%l0, %l2, %l0		! regenerate shift count
+	sub	%l5, %l0, %l1		! generate new shift left count (LS)
+	and	%i2, 3, %l4		! must do remaining bytes if count%4 > 0
+	andn	%i2, 3, %i2		! # of aligned bytes that can be moved
+	srl	%i3, %l0, %i4
+	or	%i5, %i4, %i5
+	st	%i5, [%i1]		! write a word
+	subcc	%i2, 4, %i2		! decrement count
+	bz	%xcc, unalign_out
+	  add	%i1, 4, %i1		! increment destination address
+
+	b	2f
+	sll	%i3, %l1, %i5		! get leftover into upper bits
+more_needed:
+	sll	%i3, %l0, %i3		! save remaining byte(s)
+	srl	%i3, %l0, %i3
+	sub	%l2, %l0, %l1		! regenerate shift count
+	sub	%l5, %l1, %l0		! generate new shift left count
+	sll	%i3, %l1, %i4		! move to fill empty space
+	b	3f
+	or	%i5, %i4, %i5		! merge to complete word
+	!
+	! the source address is aligned and destination is not
+	!
+align_dst_only:
+	ld	[%i0], %i4		! read a word
+	add	%i0, 4, %i0		! increment source address
+	mov	24, %l0			! initial shift alignment count
+1:
+	srl	%i4, %l0, %i3		! prepare to write a single byte
+	stb	%i3, [%i1]		! write a byte
+	add	%i1, 1, %i1		! increment destination address
+	sub	%i2, 1, %i2		! decrement count
+	btst	3, %i1			! is destination aligned?
+	bnz,a	1b
+	  sub	%l0, 8, %l0		! delay slot, decrement shift count
+xfer:
+	sub	%l5, %l0, %l1		! generate shift left count
+	sll	%i4, %l1, %i5		! get leftover
+3:
+	and	%i2, 3, %l4		! must do remaining bytes if count%4 > 0
+	andn	%i2, 3, %i2		! # of aligned bytes that can be moved
+2:
+	ld	[%i0], %i3		! read a source word
+	add	%i0, 4, %i0		! increment source address
+	srl	%i3, %l0, %i4		! upper src bits into lower dst bits
+	or	%i5, %i4, %i5		! merge with upper dest bits (leftover)
+	st	%i5, [%i1]		! write a destination word
+	subcc	%i2, 4, %i2		! decrement count
+	bz	%xcc, unalign_out	! check if done
+	  add	%i1, 4, %i1		! increment destination address
+	b	2b			! loop
+	sll	%i3, %l1, %i5		! get leftover
+unalign_out:
+	tst	%l4			! any bytes leftover?
+	bz	%xcc, cpdone
+	  nop
+1:
+	sub	%l0, 8, %l0		! decrement shift
+	srl	%i3, %l0, %i4		! upper src byte into lower dst byte
+	stb	%i4, [%i1]		! write a byte
+	subcc	%l4, 1, %l4		! decrement count
+	bz	%xcc, cpdone		! done?
+	add	%i1, 1, %i1		! increment destination
+	tst	%l0			! any more previously read bytes
+	bnz	%xcc, 1b		! we have leftover bytes
+	  mov	%l4, %i2		! delay slot, mv cnt where dbytecp wants
+	b	dbytecp			! let dbytecp do the rest
+	  sub	%i0, %i1, %i0		! i0 gets the difference of src and dst
+	!
+	! the destination address is aligned and the source is not
+	!
+align_src_only:
+	ldub	[%i0], %i3		! read a byte from source address
+	add	%i0, 1, %i0		! increment source address
+	or	%i4, %i3, %i4		! or in with previous bytes (if any)
+	btst	3, %i0			! is source aligned?
+	add	%l0, 8, %l0		! increment shift count (US)
+	bnz,a	align_src_only
+	  sll	%i4, 8, %i4		! make room for next byte
+	b,a	xfer
+	!
+	! if from address unaligned for double-word moves,
+	! move bytes till it is, if count is < 56 it could take
+	! longer to align the thing than to do the transfer
+	! in word size chunks right away
+	!
+aldoubcp:
+	cmp	%i2, 56			! if count < 56, use wordcp, it takes
+	blu,a	%xcc, alwordcp		! longer to align doubles than words
+	  mov	3, %o0			! mask for word alignment
+	call	alignit			! copy bytes until aligned
+	mov	7, %o0			! mask for double alignment
+	  !
+	! source and destination are now double-word aligned
+	! i3 has aligned count returned by alignit
+	!
+	and	%i2, 7, %i2		! unaligned leftover count
+	sub	%i0, %i1, %i0		! i0 gets the difference of src and dst
+5:
+	ldx	[%i0+%i1], %o4		! read from address
+	stx	%o4, [%i1]		! write at destination address
+	subcc	%i3, 8, %i3		! dec count
+	bgu	%xcc, 5b
+	  add	%i1, 8, %i1		! delay slot, inc to address
+	cmp	%i2, 4			! see if we can copy a word
+	blu	%xcc, dbytecp		! if 3 or less bytes use bytecp
+	  nop
+	!
+	! for leftover bytes we fall into wordcp, if needed
+	!
+wordcp:
+	and	%i2, 3, %i2		! unaligned leftover count
+5:
+	ld	[%i0+%i1], %o4		! read from address
+	st	%o4, [%i1]		! write at destination address
+	subcc	%i3, 4, %i3		! dec count
+	bgu	%xcc, 5b
+	  add	%i1, 4, %i1		! delay slot, inc to address
+	b,a	dbytecp
+
+	! we come here to align copies on word boundaries
+alwordcp:
+	call	alignit		! go word-align it
+	  mov	3, %o0			! bits that must be zero to be aligned
+	b	wordcp
+	  sub	%i0, %i1, %i0		! i0 gets the difference of src and dst
+
+	!
+	! byte copy, works with any alignment
+	!
+bytecp:
+	b	dbytecp
+	  sub	%i0, %i1, %i0		! i0 gets difference of src and dst
+
+	!
+	! differenced byte copy, works with any alignment
+	! assumes dest in %i1 and (source - dest) in %i0
+	!
+1:
+	stb	%o4, [%i1]		! write to address
+	inc	%i1			! inc to address
+dbytecp:
+	deccc	%i2			! dec count
+	bgeu,a	%xcc, 1b		! loop till done
+	ldub	[%i0+%i1], %o4		! read from address
+cpdone:
+	membar	#Sync				! sync error barrier
+	ret
+	  restore %g0, 0, %o0		! return (0)
+
+/*
+ * Common code used to align transfers on word and doubleword
+ * boudaries.  Aligns source and destination and returns a count
+ * of aligned bytes to transfer in %i3
+ */
+1:
+	inc	%i0			! inc from
+	stb	%o4, [%i1]		! write a byte
+	inc	%i1			! inc to
+	dec	%i2			! dec count
+alignit:
+	btst	%o0, %i0		! %o0 is bit mask to check for alignment
+	bnz,a	1b
+	  ldub	[%i0], %o4		! read next byte
+
+	retl
+	andn	%i2, %o0, %i3		! return size of aligned bytes
+END(novbcopy)
+
+
+/*
+ * hwblkclr - clears block-aligned, block-multiple-sized regions that are
+ * longer than 256 bytes in length using Niagara's block stores/quad store.
+ * If the criteria for using this routine are not met then it calls bzero
+ * and returns 1.  Otherwise 0 is returned indicating success.
+ * Caller is responsible for ensuring use_hw_bzero is true and that
+ * kpreempt_disable() has been called.
+ */
+#ifdef lint
+/*ARGSUSED*/
+int
+hwblkclr(void *addr, size_t len)
+{ 
+	return(0);
+}
+#else /* lint */
+	! %i0 - start address
+	! %i1 - length of region (multiple of 64)
+
+ENTRY(hwblkclr)
+	save	%sp, -SA(MINFRAME), %sp
+
+	! Must be block-aligned
+	andcc	%i0, 0x3f, %g0
+	bnz,pn	%xcc, 1f
+	  nop
+
+	! ... and must be 256 bytes or more
+	cmp	%i1, 0x100
+	blu,pn	%xcc, 1f
+	  nop
+
+	! ... and length must be a multiple of 64
+	andcc	%i1, 0x3f, %g0
+	bz,pn	%xcc, pz_doblock
+	wr	%g0, ASI_LDSTBI_P, %asi
+
+1:	! punt, call bzero but notify the caller that bzero was used
+	mov	%i0, %o0
+	call	bzero
+	  mov	%i1, %o1
+	ret
+	  restore	%g0, 1, %o0	! return (1) - did not use block operations
+
+	! Already verified that there are at least 256 bytes to set
+pz_doblock:
+	stxa	%g0, [%i0+0x0]%asi
+	stxa	%g0, [%i0+0x40]%asi
+	stxa	%g0, [%i0+0x80]%asi
+	stxa	%g0, [%i0+0xc0]%asi
+
+	stxa	%g0, [%i0+0x8]%asi
+	stxa	%g0, [%i0+0x10]%asi
+	stxa	%g0, [%i0+0x18]%asi
+	stxa	%g0, [%i0+0x20]%asi
+	stxa	%g0, [%i0+0x28]%asi
+	stxa	%g0, [%i0+0x30]%asi
+	stxa	%g0, [%i0+0x38]%asi
+
+	stxa	%g0, [%i0+0x48]%asi
+	stxa	%g0, [%i0+0x50]%asi
+	stxa	%g0, [%i0+0x58]%asi
+	stxa	%g0, [%i0+0x60]%asi
+	stxa	%g0, [%i0+0x68]%asi
+	stxa	%g0, [%i0+0x70]%asi
+	stxa	%g0, [%i0+0x78]%asi
+
+	stxa	%g0, [%i0+0x88]%asi
+	stxa	%g0, [%i0+0x90]%asi
+	stxa	%g0, [%i0+0x98]%asi
+	stxa	%g0, [%i0+0xa0]%asi
+	stxa	%g0, [%i0+0xa8]%asi
+	stxa	%g0, [%i0+0xb0]%asi
+	stxa	%g0, [%i0+0xb8]%asi
+
+	stxa	%g0, [%i0+0xc8]%asi
+	stxa	%g0, [%i0+0xd0]%asi
+	stxa	%g0, [%i0+0xd8]%asi
+	stxa	%g0, [%i0+0xe0]%asi
+	stxa	%g0, [%i0+0xe8]%asi
+	stxa	%g0, [%i0+0xf0]%asi
+	stxa	%g0, [%i0+0xf8]%asi
+
+	sub	%i1, 0x100, %i1
+	cmp	%i1, 0x100
+	bgu,pt	%xcc, pz_doblock
+	  add	%i0, 0x100, %i0
+
+2:
+	! Check if more than 64 bytes to set
+	cmp	%i1,0x40
+	blu	%xcc, pz_finish
+	  nop
+
+3:
+	stxa	%g0, [%i0+0x0]%asi
+	stxa	%g0, [%i0+0x8]%asi
+	stxa	%g0, [%i0+0x10]%asi
+	stxa	%g0, [%i0+0x18]%asi
+	stxa	%g0, [%i0+0x20]%asi
+	stxa	%g0, [%i0+0x28]%asi
+	stxa	%g0, [%i0+0x30]%asi
+	stxa	%g0, [%i0+0x38]%asi
+
+	subcc	%i1, 0x40, %i1
+	bgu,pt	%xcc, 3b
+	  add	%i0, 0x40, %i0
+
+pz_finish:
+	membar	#Sync
+	ret
+	  restore	%g0, 0, %o0		! return (bzero or not)
+END(hwblkclr)
+#endif	/* lint */
+
+#if defined(lint)
+
+/* ARGSUSED */
+void
+bzero(void *addr, size_t count)
+{}
+
+#else	/* lint */
+
+ENTRY(bzero)
+	wr	%g0, ASI_P, %asi
+
+	cmp	%o1, 7
+	blu,pn	%xcc, byteclr
+	  nop
+
+	cmp	%o1, 15
+	blu,pn	%xcc, wdalign
+	  nop
+
+	andcc	%o0, 7, %o3		! is add aligned on a 8 byte bound
+	bz,pt	%xcc, blkalign		! already double aligned
+	  sub	%o3, 8, %o3		! -(bytes till double aligned)
+	add	%o1, %o3, %o1		! update o1 with new count
+
+1:
+	stba	%g0, [%o0]%asi
+	inccc	%o3
+	bl,pt	%xcc, 1b
+	  inc	%o0
+
+	! Now address is double aligned
+blkalign:
+	cmp	%o1, 0x80		! check if there are 128 bytes to set
+	blu,pn	%xcc, bzero_small
+	  mov	%o1, %o3
+#if 0
+	sethi	%hi(use_hw_bzero), %o2
+	ld	[%o2 + %lo(use_hw_bzero)], %o2
+	tst	%o2
+	bz	%xcc, bzero_small
+	  mov	%o1, %o3
+#endif
+	rd	%asi, %o3
+	wr	%g0, ASI_LDSTBI_P, %asi
+	cmp	%o3, ASI_P
+	bne,a	%xcc, algnblk
+	  wr	%g0, ASI_LDSTBI_AIUS, %asi
+
+algnblk:
+	andcc	%o0, 0x3f, %o3		! is block aligned?
+	bz,pt	%xcc, bzero_blk
+	  sub	%o3, 0x40, %o3		! -(bytes till block aligned)
+	add	%o1, %o3, %o1		! o1 is the remainder
+	
+	! Clear -(%o3) bytes till block aligned
+1:
+	stxa	%g0, [%o0]%asi
+	addcc	%o3, 8, %o3
+	bl,pt	%xcc, 1b
+ 	  add	%o0, 8, %o0
+
+bzero_blk:
+	and	%o1, 0x3f, %o3		! calc bytes left after blk clear
+	andn	%o1, 0x3f, %o4		! calc size of blocks in bytes
+
+	cmp	%o4, 0x100		! 256 bytes or more
+	blu,pn	%xcc, 3f
+	  nop
+
+2:
+	stxa	%g0, [%o0+0x0]%asi
+	stxa	%g0, [%o0+0x40]%asi
+	stxa	%g0, [%o0+0x80]%asi
+	stxa	%g0, [%o0+0xc0]%asi
+
+	stxa	%g0, [%o0+0x8]%asi
+	stxa	%g0, [%o0+0x10]%asi
+	stxa	%g0, [%o0+0x18]%asi
+	stxa	%g0, [%o0+0x20]%asi
+	stxa	%g0, [%o0+0x28]%asi
+	stxa	%g0, [%o0+0x30]%asi
+	stxa	%g0, [%o0+0x38]%asi
+
+	stxa	%g0, [%o0+0x48]%asi
+	stxa	%g0, [%o0+0x50]%asi
+	stxa	%g0, [%o0+0x58]%asi
+	stxa	%g0, [%o0+0x60]%asi
+	stxa	%g0, [%o0+0x68]%asi
+	stxa	%g0, [%o0+0x70]%asi
+	stxa	%g0, [%o0+0x78]%asi
+
+	stxa	%g0, [%o0+0x88]%asi
+	stxa	%g0, [%o0+0x90]%asi
+	stxa	%g0, [%o0+0x98]%asi
+	stxa	%g0, [%o0+0xa0]%asi
+	stxa	%g0, [%o0+0xa8]%asi
+	stxa	%g0, [%o0+0xb0]%asi
+	stxa	%g0, [%o0+0xb8]%asi
+
+	stxa	%g0, [%o0+0xc8]%asi
+	stxa	%g0, [%o0+0xd0]%asi
+	stxa	%g0, [%o0+0xd8]%asi
+	stxa	%g0, [%o0+0xe0]%asi
+	stxa	%g0, [%o0+0xe8]%asi
+	stxa	%g0, [%o0+0xf0]%asi
+	stxa	%g0, [%o0+0xf8]%asi
+
+	sub	%o4, 0x100, %o4
+	cmp	%o4, 0x100
+	bgu,pt	%xcc, 2b
+	  add	%o0, 0x100, %o0
+
+3:
+	! ... check if 64 bytes to set
+	cmp	%o4, 0x40
+	blu	%xcc, bzero_blk_done
+	  nop
+
+4:
+	stxa	%g0, [%o0+0x0]%asi
+	stxa	%g0, [%o0+0x8]%asi
+	stxa	%g0, [%o0+0x10]%asi
+	stxa	%g0, [%o0+0x18]%asi
+	stxa	%g0, [%o0+0x20]%asi
+	stxa	%g0, [%o0+0x28]%asi
+	stxa	%g0, [%o0+0x30]%asi
+	stxa	%g0, [%o0+0x38]%asi
+
+	subcc	%o4, 0x40, %o4
+	bgu,pt	%xcc, 3b
+	  add	%o0, 0x40, %o0
+
+bzero_blk_done:
+	membar	#Sync
+	!
+	! Undo asi register setting.
+	!
+	rd	%asi, %o4
+	wr	%g0, ASI_P, %asi
+	cmp	%o4, ASI_LDSTBI_P
+	bne,a	%xcc, bzero_small
+	  wr	%g0, ASI_AIUS, %asi
+
+bzero_small:
+	! Set the remaining doubles
+	subcc	%o3, 8, %o3		! Can we store any doubles?
+	blu,pn	%xcc, byteclr
+	  and	%o1, 7, %o1		! calc bytes left after doubles
+
+dbclr:
+	stxa	%g0, [%o0]%asi		! Clear the doubles
+	subcc	%o3, 8, %o3
+	bgeu,pt	%xcc, dbclr
+	  add	%o0, 8, %o0
+
+	ba	byteclr
+ 	  nop
+
+wdalign:			
+	andcc	%o0, 3, %o3		! is add aligned on a word boundary
+	bz,pn	%xcc, wdclr
+	andn	%o1, 3, %o3		! create word sized count in %o3
+
+	dec	%o1			! decrement count
+	stba	%g0, [%o0]%asi		! clear a byte
+	ba	wdalign
+	  inc	%o0			! next byte
+
+wdclr:
+	sta	%g0, [%o0]%asi		! 4-byte clearing loop
+	subcc	%o3, 4, %o3
+	bnz,pt	%xcc, wdclr
+	  inc	4, %o0
+
+	and	%o1, 3, %o1		! leftover count, if any
+
+byteclr:
+	! Set the leftover bytes
+	brz	%o1, bzero_exit
+	nop
+
+7:
+	deccc	%o1			! byte clearing loop
+	stba	%g0, [%o0]%asi
+	bgu,pt	%xcc, 7b
+	  inc	%o0
+
+bzero_exit:
+	retl
+	  clr	%o0			! return (0)
+
+END(bzero)
+#endif	/* lint */
+
+
+#if 0
+#define SMALL_LIMIT 7	
+#if defined(lint)
+
+/*ARGSUSED*/
+int
+copyin(const void *uaddr, void *kaddr, size_t count)
+{ return (0); }
+
+#else	/* lint */
+
+ENTRY(copyin)
+	!
+	! Check the length and bail if zero.
+	!
+	tst	%o2
+	bnz,pt	%xcc, 1f
+	  nop
+	retl
+	  clr	%o0
+#if 0
+1:
+	sethi	%hi(copyio_fault), %o4
+	or	%o4, %lo(copyio_fault), %o4
+	sethi	%hi(copyio_fault_nowindow), %o3
+	ldn	[THREAD_REG + T_LOFAULT], SAVED_LOFAULT
+	or	%o3, %lo(copyio_fault_nowindow), %o3
+	membar	#Sync
+	stn	%o3, [THREAD_REG + T_LOFAULT]
+
+	mov	%o0, SAVE_SRC
+	mov	%o1, SAVE_DST
+	mov	%o2, SAVE_COUNT
+#endif
+	!
+	! Check to see if we're more than SMALL_LIMIT.
+	!
+	subcc	%o2, SMALL_LIMIT, %o3
+	bgu,a,pt %xcc, dci_ns
+	  or	%o0, %o1, %o3
+	!
+	! What was previously ".small_copyin"
+	!
+dcibcp:
+	sub	%g0, %o2, %o3		! setup for copy loop
+	add	%o0, %o2, %o0
+	add	%o1, %o2, %o1
+	ba,pt	%xcc, dcicl
+	lduba	[%o0 + %o3]ASI_AIUS, %o4
+	!
+	! %o0 and %o1 point at the end and remain pointing at the end
+	! of their buffers. We pull things out by adding %o3 (which is
+	! the negation of the length) to the buffer end which gives us
+	! the curent location in the buffers. By incrementing %o3 we walk
+	! through both buffers without having to bump each buffer's
+	! pointer. A very fast 4 instruction loop.
+	!
+	.align 16
+dcicl:
+	stb	%o4, [%o1 + %o3]
+	inccc	%o3
+	bl,a,pt %xcc, dcicl
+	lduba	[%o0 + %o3]ASI_AIUS, %o4
+	!
+	! We're done. Go home.
+	!	
+	membar	#Sync
+	retl
+	  clr	%o0
+	!
+	! Try aligned copies from here.
+	!
+dci_ns:
+	!
+	! See if we're single byte aligned. If we are, check the
+	! limit for single byte copies. If we're smaller, or equal,
+	! bounce to the byte for byte copy loop. Otherwise do it in
+	! HW (if enabled).
+	!
+	btst	1, %o3
+	bz,a,pt	%icc, dcih8
+	btst	7, %o3
+	!
+	! We're single byte aligned.
+	!
+	sethi	%hi(hw_copy_limit_1), %o3
+	ld	[%o3 + %lo(hw_copy_limit_1)], %o3
+	!
+	! Is HW copy on? If not do everything byte for byte.
+	!
+	tst	%o3
+	bz,pn	%icc, dcibcp
+	  subcc	%o3, %o2, %o3
+	!
+	! Are we bigger than the HW limit? If not
+	! go to byte for byte.
+	!
+	bge,pt	%xcc, dcibcp
+	  nop
+	!
+	! We're big enough and copy is on. Do it with HW.
+	!
+	ba,pt	%xcc, big_copyin
+	nop
+dcih8:
+	!
+	! 8 byte aligned?
+	!
+	bnz,a	%xcc, dcih4
+	btst	3, %o3
+	!
+	! We're eight byte aligned.
+	!
+	sethi	%hi(hw_copy_limit_8), %o3
+	ld	[%o3 + %lo(hw_copy_limit_8)], %o3
+	!
+	! Is HW assist on? If not, do it with the aligned copy.
+	!
+	tst	%o3
+	bz,pn	%icc, dcis8
+	subcc	%o3, %o2, %o3
+	bge	%xcc, dcis8
+	nop
+	ba,pt	%xcc, big_copyin
+	nop
+dcis8:
+	!
+	! Housekeeping for copy loops. Uses same idea as in the byte for
+	! byte copy loop above.
+	!
+	add	%o0, %o2, %o0
+	add	%o1, %o2, %o1
+	sub	%g0, %o2, %o3
+	ba,pt	%xcc, didebc
+	srl	%o2, 3, %o2		! Number of 8 byte chunks to copy
+	!
+	! 4 byte aligned?
+	!
+dcih4:
+	bnz	%xcc, dcih2
+	sethi	%hi(hw_copy_limit_4), %o3
+	ld	[%o3 + %lo(hw_copy_limit_4)], %o3
+	!
+	! Is HW assist on? If not, do it with the aligned copy.
+	!
+	tst	%o3
+	bz,pn	%icc, dcis4
+	  subcc	%o3, %o2, %o3
+	!
+	! We're negative if our size is less than or equal to hw_copy_limit_4.
+	!
+	bge	%xcc, dcis4
+	  nop
+	ba,pt	%xcc, big_copyin
+	  nop
+dcis4:
+	!
+	! Housekeeping for copy loops. Uses same idea as in the byte
+	! for byte copy loop above.
+	!
+	add	%o0, %o2, %o0
+	add	%o1, %o2, %o1
+	sub	%g0, %o2, %o3
+	ba,pt	%xcc, didfbc
+	  srl	%o2, 2, %o2		! Number of 4 byte chunks to copy
+dcih2:
+	!
+	! We're two byte aligned. Check for "smallness"
+	! done in delay at .dcih4
+	!
+	bleu,pt	%xcc, dcis2
+	sethi	%hi(hw_copy_limit_2), %o3
+	ld	[%o3 + %lo(hw_copy_limit_2)], %o3
+	!
+	! Is HW assist on? If not, do it with the aligned copy.
+	!
+	tst	%o3
+	bz,pn	%icc, dcis2
+	  subcc	%o3, %o2, %o3
+	!
+	! Are we larger than the HW limit?
+	!
+	bge	%xcc, dcis2
+	nop
+	!
+	! HW assist is on and we're large enough to use it.
+	!
+	ba,pt	%xcc, big_copyin
+	nop
+	!
+	! Housekeeping for copy loops. Uses same idea as in the byte
+	! for byte copy loop above.
+	!
+dcis2:
+	add	%o0, %o2, %o0
+	add	%o1, %o2, %o1
+	sub	%g0, %o2, %o3
+	ba,pt	%xcc, didtbc
+	srl	%o2, 1, %o2		! Number of 2 byte chunks to copy
+	!
+small_copyin:
+	!
+	! Why are we doing this AGAIN? There are certain conditions in
+	! big copyin that will cause us to forgo the HW assisted copys
+	! and bounce back to a non-hw assisted copy. This dispatches
+	! those copies. Note that we branch around this in the main line
+	! code.
+	!
+	! We make no check for limits or HW enablement here. We've
+	! already been told that we're a poster child so just go off
+	! and do it.
+	!
+	or	%o0, %o1, %o3
+	btst	1, %o3
+	bnz	%icc, dcibcp		! Most likely
+	  btst	7, %o3
+	bz	%icc, dcis8
+	  btst	3, %o3
+	bz	%icc, dcis4
+	  nop
+	ba,pt	%xcc, dcis2
+	  nop
+	!
+	! Eight byte aligned copies. A steal from the original .small_copyin
+	! with modifications. %o2 is number of 8 byte chunks to copy. When
+	! done, we examine %o3. If this is < 0, we have 1 - 7 bytes more
+	! to copy.
+	!
+	.align 32
+didebc:
+	ldxa	[%o0 + %o3]ASI_AIUS, %o4
+	deccc	%o2
+	stx	%o4, [%o1 + %o3]
+	bg,pt	%xcc, didebc
+	addcc	%o3, 8, %o3
+	!
+	! End of copy loop. Most 8 byte aligned copies end here.
+	!
+	bz,pt	%xcc, dcifh
+	nop
+	!
+	! Something is left. Do it byte for byte.
+	!
+	ba,pt	%xcc, dcicl
+	lduba	[%o0 + %o3]ASI_AIUS, %o4
+	!
+	! 4 byte copy loop. %o2 is number of 4 byte chunks to copy.
+	!
+	.align 32
+didfbc:
+	lduwa	[%o0 + %o3]ASI_AIUS, %o4
+	deccc	%o2
+	st	%o4, [%o1 + %o3]
+	bg,pt	%xcc, didfbc
+	addcc	%o3, 4, %o3
+	!
+	! End of copy loop. Most 4 byte aligned copies end here.
+	!
+	bz,pt	%xcc, dcifh
+	nop
+	!
+	! Something is left. Do it byte for byte.
+	!
+	ba,pt	%xcc, dcicl
+	lduba	[%o0 + %o3]ASI_AIUS, %o4
+	!
+	! 2 byte aligned copy loop. %o2 is number of 2 byte chunks to
+	! copy.
+	!
+	.align 32
+didtbc:
+	lduha	[%o0 + %o3]ASI_AIUS, %o4
+	deccc	%o2
+	sth	%o4, [%o1 + %o3]
+	bg,pt	%xcc, didtbc
+	  addcc	%o3, 2, %o3
+	!
+	! End of copy loop. Most 2 byte aligned copies end here.
+	!
+	bz,pt	%xcc, dcifh
+	  nop
+	!
+	! Deal with the last byte
+	!
+	lduba	[%o0 + %o3]ASI_AIUS, %o4
+	stb	%o4, [%o1 + %o3]
+dcifh:
+	membar	#Sync
+	retl
+	  clr	%o0
+
+big_copyin:
+	!
+	! We're going off to do a block copy.
+	! Switch fault hendlers and grab a window. We
+	! don't do a membar #Sync since we've done only
+	! kernel data to this point.
+	!
+	save	%sp, -SA(MINFRAME), %sp
+
+	! Copy in that reach here are larger than 256 bytes. The
+	! hw_copy_limit_1 is set to 256. Never set this limit less
+	! 128 bytes.
+do_blockcopyin:
+
+	! Swap src/dst since the code below is memcpy code
+	! and memcpy/bcopy have different calling sequences
+	mov	%i1, %i5
+	mov	%i0, %i1
+	mov	%i5, %i0
+
+	andcc	%i0, 7, %i3		! is dst double aligned
+	bz	%xcc, copyin_blkcpy
+	  sub	%i3, 8, %i3
+	neg	%i3			! bytes till double aligned
+	sub	%i2, %i3, %i2		! update %i2 with new count
+
+	! Align Destination on double-word boundary
+
+1:	lduba	[%i1]ASI_AIUS, %i4
+	inc	%i1
+	stb	%i4, [%i0]
+	deccc	%i3
+	bgu	%xcc, 1b
+	  inc	%i0
+
+copyin_blkcpy:
+	andcc	%i0, 63, %i3
+	bz,pn	%xcc, copyin_blalign	! now block aligned
+	sub	%i3, 64, %i3
+	neg	%i3			! bytes till block aligned
+	sub	%i2, %i3, %i2		! update %i2 with new count
+
+	! Copy %i3 bytes till dst is block (64 byte) aligned. use
+	! double word copies.
+
+	andcc	%i1, 7, %g1		! is src aligned on a 8 bytes
+	bz	%xcc, ci_dbcopy		! %g1 has source offset (last 3-bits)
+	sll	%g1, 3, %l1		! left shift
+	mov	0x40, %l2
+	sub	%l2, %l1, %l2		! right shift = (64 - left shift)
+
+	! Now use double word copies to align destination.
+ci_double:
+	sub	%i1, %g1, %i1		! align the src at 8 bytes.
+	ldxa	[%i1]ASI_AIUS, %o2
+2:
+	add	%i1, 0x8, %i1
+	ldxa	[%i1]ASI_AIUS, %o4
+	ALIGN_DATA_EW(%o2, %o4, %l1, %l2, %o3)
+	stx	%o2, [%i0]
+	mov	%o4, %o2
+	subcc	%i3, 0x8, %i3
+	bgu,pt	%xcc, 2b
+	add	%i0, 0x8, %i0
+	ba	copyin_blalign
+	add	%i1, %g1, %i1
+
+	! Both source and destination are double aligned.
+	! No shift and merge of data required in this case.
+ci_dbcopy:
+	ldxa	[%i1]ASI_AIUS, %o2
+	stx	%o2, [%i0]
+	add	%i1, 0x8, %i1
+	subcc	%i3, 0x8, %i3
+	bgu,pt	%xcc, ci_dbcopy
+	add	%i0, 0x8, %i0
+
+copyin_blalign:
+	andn	%i2, 0x3f, %i3		! %i3 count is multiple of block size
+	sub	%i2, %i3, %i2		! Residue bytes in %i2
+
+	wr	%g0, ASI_LDSTBI_P, %asi
+
+	andcc	%i1, 0xf, %o2		! is src quadword aligned
+	bz,pn	%xcc, ci_blkcpy		! src offset in %o2 (last 4-bits)
+	  nop
+	cmp	%o2, 0x8
+	bg	ci_upper_double
+	  nop
+	bl	ci_lower_double
+	  nop
+
+	! Falls through when source offset is equal to 8 i.e.
+	! source is double word aligned.
+	! In this case no shift/merge of data is required
+
+	sub	%i1, %o2, %i1		! align the src at 16 bytes.
+	andn	%i1, 0x3f, %l0		! %l0 has block aligned source
+	prefetch [%l0+0x0], #one_read
+	ldda	[%i1]ASI_LDSTBI_AIUS, %l2
+ci_loop0:
+	add	%i1, 0x10, %i1
+	ldda	[%i1]ASI_LDSTBI_AIUS, %l4
+
+	prefetch [%l0+0x40], #one_read
+
+	stxa	%l3, [%i0+0x0]%asi
+	stxa	%l4, [%i0+0x8]%asi
+
+	add	%i1, 0x10, %i1
+	ldda	[%i1]ASI_LDSTBI_AIUS, %l2
+
+	stxa	%l5, [%i0+0x10]%asi
+	stxa	%l2, [%i0+0x18]%asi
+
+	add	%i1, 0x10, %i1
+	ldda	[%i1]ASI_LDSTBI_AIUS, %l4
+
+	stxa	%l3, [%i0+0x20]%asi
+	stxa	%l4, [%i0+0x28]%asi
+
+	add	%i1, 0x10, %i1
+	ldda	[%i1]ASI_LDSTBI_AIUS, %l2
+
+	stxa	%l5, [%i0+0x30]%asi
+	stxa	%l2, [%i0+0x38]%asi
+
+	add	%l0, 0x40, %l0
+	subcc	%i3, 0x40, %i3
+	bgu,pt	%xcc, ci_loop0
+	  add	%i0, 0x40, %i0
+	ba	ci_blkdone
+	  add	%i1, %o2, %i1		! increment the source by src offset
+					! the src offset was stored in %o2
+
+ci_lower_double:
+
+	sub	%i1, %o2, %i1		! align the src at 16 bytes.
+	sll	%o2, 3, %o0		! %o0 left shift
+	mov	0x40, %o1
+	sub	%o1, %o0, %o1		! %o1 right shift = (64 - left shift)
+	andn	%i1, 0x3f, %l0		! %l0 has block aligned source
+	prefetch [%l0+0x0], #one_read
+	ldda	[%i1]ASI_LDSTBI_AIUS, %l2	! partial data in %l2
+							! and %l3 has complete
+							! data
+ci_loop1:
+	add	%i1, 0x10, %i1
+	ldda	[%i1]ASI_LDSTBI_AIUS, %l4	! %l4 has partial data
+							! for this read.
+	ALIGN_DATA(%l2, %l3, %l4, %o0, %o1, %l6)	! merge %l2, %l3 and %l4
+							! into %l2 and %l3
+
+	prefetch [%l0+0x40], #one_read
+
+	stxa	%l2, [%i0+0x0]%asi
+	stxa	%l3, [%i0+0x8]%asi
+
+	add	%i1, 0x10, %i1
+	ldda	[%i1]ASI_LDSTBI_AIUS, %l2
+	ALIGN_DATA(%l4, %l5, %l2, %o0, %o1, %l6)	! merge %l2 with %l5 and
+							! %l4 from previous read
+							! into %l4 and %l5
+	stxa	%l4, [%i0+0x10]%asi
+	stxa	%l5, [%i0+0x18]%asi
+
+	! Repeat the same for next 32 bytes.
+
+	add	%i1, 0x10, %i1
+	ldda	[%i1]ASI_LDSTBI_AIUS, %l4
+	ALIGN_DATA(%l2, %l3, %l4, %o0, %o1, %l6)
+
+	stxa	%l2, [%i0+0x20]%asi
+	stxa	%l3, [%i0+0x28]%asi
+
+	add	%i1, 0x10, %i1
+	ldda	[%i1]ASI_LDSTBI_AIUS, %l2
+	ALIGN_DATA(%l4, %l5, %l2, %o0, %o1, %l6)
+
+	stxa	%l4, [%i0+0x30]%asi
+	stxa	%l5, [%i0+0x38]%asi
+
+	add	%l0, 0x40, %l0
+	subcc	%i3, 0x40, %i3
+	bgu,pt	%xcc, ci_loop1
+	  add	%i0, 0x40, %i0
+	ba	ci_blkdone
+	  add	%i1, %o2, %i1		! increment the source by src offset
+					! the src offset was stored in %o2
+
+ci_upper_double:
+
+	sub	%i1, %o2, %i1		! align the src at 16 bytes.
+	sub	%o2, 0x8, %o0
+	sll	%o0, 3, %o0		! %o0 left shift
+	mov	0x40, %o1
+	sub	%o1, %o0, %o1		! %o1 right shift = (64 - left shift)
+	andn	%i1, 0x3f, %l0		! %l0 has block aligned source
+	prefetch [%l0+0x0], #one_read
+	ldda	[%i1]ASI_LDSTBI_AIUS, %l2	! partial data in %l3
+							! for this read and
+							! no data in %l2
+ci_loop2:
+	add	%i1, 0x10, %i1
+	ldda	[%i1]ASI_LDSTBI_AIUS, %l4	! %l4 has complete data
+							! and %l5 has partial
+	ALIGN_DATA(%l3, %l4, %l5, %o0, %o1, %l6)	! merge %l3, %l4 and %l5
+							! into %l3 and %l4
+	prefetch [%l0+0x40], #one_read
+
+	stxa	%l3, [%i0+0x0]%asi
+	stxa	%l4, [%i0+0x8]%asi
+
+	add	%i1, 0x10, %i1
+	ldda	[%i1]ASI_LDSTBI_AIUS, %l2
+	ALIGN_DATA(%l5, %l2, %l3, %o0, %o1, %l6)	! merge %l2 and %l3 with
+							! %l5 from previous read
+							! into %l5 and %l2
+
+	stxa	%l5, [%i0+0x10]%asi
+	stxa	%l2, [%i0+0x18]%asi
+
+	! Repeat the same for next 32 bytes.
+
+	add	%i1, 0x10, %i1
+	ldda	[%i1]ASI_LDSTBI_AIUS, %l4
+	ALIGN_DATA(%l3, %l4, %l5, %o0, %o1, %l6)
+
+	stxa	%l3, [%i0+0x20]%asi
+	stxa	%l4, [%i0+0x28]%asi
+
+	add	%i1, 0x10, %i1
+	ldda	[%i1]ASI_LDSTBI_AIUS, %l2
+	ALIGN_DATA(%l5, %l2, %l3, %o0, %o1, %l6)
+
+	stxa	%l5, [%i0+0x30]%asi
+	stxa	%l2, [%i0+0x38]%asi
+
+	add	%l0, 0x40, %l0
+	subcc	%i3, 0x40, %i3
+	bgu,pt	%xcc, ci_loop2
+	  add	%i0, 0x40, %i0
+	ba	ci_blkdone
+	  add	%i1, %o2, %i1		! increment the source by src offset
+					! the src offset was stored in %o2
+
+
+	! Do fast copy using ASI_LDSTBI_P
+ci_blkcpy:
+
+	andn	%i1, 0x3f, %o0		! %o0 has block aligned source
+	prefetch [%o0+0x0], #one_read
+1:
+	ldda	[%i1]ASI_LDSTBI_AIUS, %l0
+	add	%i1, 0x10, %i1
+	ldda	[%i1]ASI_LDSTBI_AIUS, %l2
+	add	%i1, 0x10, %i1
+
+	prefetch [%o0+0x40], #one_read
+
+	stxa	%l0, [%i0+0x0]%asi
+
+	ldda	[%i1]ASI_LDSTBI_AIUS, %l4
+	add	%i1, 0x10, %i1
+	ldda	[%i1]ASI_LDSTBI_AIUS, %l6
+	add	%i1, 0x10, %i1
+
+	stxa	%l1, [%i0+0x8]%asi
+	stxa	%l2, [%i0+0x10]%asi
+	stxa	%l3, [%i0+0x18]%asi
+	stxa	%l4, [%i0+0x20]%asi
+	stxa	%l5, [%i0+0x28]%asi
+	stxa	%l6, [%i0+0x30]%asi
+	stxa	%l7, [%i0+0x38]%asi
+
+	add	%o0, 0x40, %o0
+	subcc	%i3, 0x40, %i3
+	bgu,pt	%xcc, 1b
+	  add	%i0, 0x40, %i0
+
+ci_blkdone:
+	membar	#Sync
+
+	! Copy as much rest of the data as double word copy.
+ci_dwcp:
+	cmp	%i2, 0x8		! Not enough bytes to copy as double
+	blu	%xcc, ci_dbdone
+	  nop
+
+	andn	%i2, 0x7, %i3		! %i3 count is multiple of 8 bytes size
+	sub	%i2, %i3, %i2		! Residue bytes in %i2
+
+	andcc	%i1, 7, %g1		! is src aligned on a 8 bytes
+	bz	%xcc, ci_cpy_db
+	  nop
+
+	sll	%g1, 3, %l0		! left shift
+	mov	0x40, %l1
+	sub	%l1, %l0, %l1		! right shift = (64 - left shift)
+
+ci_cpy_dbwd:
+	sub	%i1, %g1, %i1		! align the src at 8 bytes.
+	ldxa	[%i1]ASI_AIUS, %o2
+3:
+	add	%i1, 0x8, %i1
+	ldxa	[%i1]ASI_AIUS, %o4
+	ALIGN_DATA_EW(%o2, %o4, %l0, %l1, %o3)
+	stx	%o2, [%i0]
+	mov	%o4, %o2
+	subcc	%i3, 0x8, %i3
+	bgu,pt	%xcc, 3b
+	  add	%i0, 0x8, %i0
+	ba	ci_dbdone
+	  add	%i1, %g1, %i1
+
+ci_cpy_db:
+	ldxa	[%i1]ASI_AIUS, %o2
+	stx	%o2, [%i0]
+	add	%i1, 0x8, %i1
+	subcc	%i3, 0x8, %i3
+	bgu,pt	%xcc, ci_cpy_db
+	add	%i0, 0x8, %i0
+
+ci_dbdone:
+	tst	%i2
+	bz,pt	%xcc, copyin_exit
+	  nop
+
+	! Copy the residue as byte copy
+ci_residue:
+	lduba	[%i1]ASI_AIUS, %i4
+	stb	%i4, [%i0]
+	inc	%i1
+	deccc	%i2
+	bgu	%xcc, ci_residue
+	  inc	%i0
+
+copyin_exit:
+	membar	#Sync
+	ret
+	  restore	%g0, 0, %o0
+END(copyin)
+
+#endif	/* lint */
+#endif
+
diff --git a/sys/sun4v/sun4v/tick.c b/sys/sun4v/sun4v/tick.c
new file mode 100644
index 000000000000..0f0e1aac6bf2
--- /dev/null
+++ b/sys/sun4v/sun4v/tick.c
@@ -0,0 +1,196 @@
+/*-
+ * Copyright (c) 2001 Jake Burkholder.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/kernel.h>
+#include <sys/systm.h>
+#include <sys/bus.h>
+#include <sys/interrupt.h>
+#include <sys/pcpu.h>
+#include <sys/sysctl.h>
+#include <sys/timetc.h>
+
+#include <machine/clock.h>
+#include <machine/cpu.h>
+#include <machine/frame.h>
+#include <machine/intr_machdep.h>
+#include <machine/tick.h>
+#include <machine/ver.h>
+
+#ifdef DEBUG
+#include <sys/proc.h>
+#endif
+
+#define	TICK_GRACE	10000
+
+SYSCTL_NODE(_machdep, OID_AUTO, tick, CTLFLAG_RD, 0, "tick statistics");
+
+static int adjust_edges = 0;
+SYSCTL_INT(_machdep_tick, OID_AUTO, adjust_edges, CTLFLAG_RD, &adjust_edges,
+    0, "total number of times tick interrupts got more than 12.5% behind");
+
+static int adjust_excess = 0;
+SYSCTL_INT(_machdep_tick, OID_AUTO, adjust_excess, CTLFLAG_RD, &adjust_excess,
+    0, "total number of ignored tick interrupts");
+
+static int adjust_missed = 0;
+SYSCTL_INT(_machdep_tick, OID_AUTO, adjust_missed, CTLFLAG_RD, &adjust_missed,
+    0, "total number of missed tick interrupts");
+
+static int adjust_ticks = 0;
+SYSCTL_INT(_machdep_tick, OID_AUTO, adjust_ticks, CTLFLAG_RD, &adjust_ticks,
+    0, "total number of tick interrupts with adjustment");
+
+static void tick_hardclock(struct trapframe *);
+
+static uint64_t
+tick_cputicks(void)
+{
+
+	return (rd(tick));
+}
+
+void
+cpu_initclocks(void)
+{
+
+	stathz = hz;
+	tick_start();
+}
+
+static __inline void
+tick_process(struct trapframe *tf)
+{
+
+	if (PCPU_GET(cpuid) == 0)
+		hardclock(TRAPF_USERMODE(tf), TRAPF_PC(tf));
+	else
+		hardclock_cpu(TRAPF_USERMODE(tf));
+	if (profprocs != 0)
+		profclock(TRAPF_USERMODE(tf), TRAPF_PC(tf));
+
+	statclock(TRAPF_USERMODE(tf));
+}
+
+static void
+tick_hardclock(struct trapframe *tf)
+{
+	u_long adj, s, tick, ref;
+	long delta;
+	int count;
+
+#ifdef DEBUG	
+	if (curthread->td_critnest > 2 || curthread->td_critnest < 1)
+		panic("nested hardclock %d\n", curthread->td_critnest);
+#endif
+	/*
+	 * The sequence of reading the TICK register, calculating the value
+	 * of the next tick and writing it to the TICK_CMPR register must not
+	 * be interrupted, not even by an IPI, otherwise a value that is in
+	 * the past could be written in the worst case, causing hardclock to
+	 * stop.
+	 */
+	adj = PCPU_GET(tickadj);
+	s = intr_disable_all();
+	tick = rd(tick);
+	wrtickcmpr(tick + tick_increment - adj, 0);
+	intr_restore_all(s);
+
+	ref = PCPU_GET(tickref);
+	delta = tick - ref;
+	count = 0;
+	while (delta >= tick_increment) {
+		tick_process(tf);
+		delta -= tick_increment;
+		ref += tick_increment;
+		if (adj != 0)
+			adjust_ticks++;
+		count++;
+	}
+	if (count > 0) {
+		adjust_missed += count - 1;
+		if (delta > (tick_increment >> 3)) {
+			if (adj == 0)
+				adjust_edges++;
+			adj = tick_increment >> 4;
+		} else
+			adj = 0;
+	} else {
+		adj = 0;
+		adjust_excess++;
+	}
+	PCPU_SET(tickref, ref);
+	PCPU_SET(tickadj, adj);
+
+}
+
+void
+tick_init(u_long clock)
+{
+
+	tick_freq = clock;
+	tick_MHz = clock / 1000000;
+	tick_increment = clock / hz;
+	/*
+	 * Avoid stopping of hardclock in terms of a lost tick interrupt
+	 * by ensuring that the tick period is at least TICK_GRACE ticks.
+	 */
+	printf("tick_freq=%ld hz=%d tick_increment=%ld\n",
+	       tick_freq, hz, tick_increment);
+
+#ifndef SIMULATOR
+	if (tick_increment < TICK_GRACE)
+		panic("%s: HZ too high, decrease to at least %ld", __func__,
+		    clock / TICK_GRACE);
+#endif
+	set_cputicker(tick_cputicks, tick_freq, 0);
+}
+
+void
+tick_start(void)
+{
+	u_long base, s;
+
+	if (PCPU_GET(cpuid) == 0)
+		intr_setup(PIL_TICK, tick_hardclock, -1, NULL, NULL);
+
+	/*
+	 * Try to make the tick interrupts as synchronously as possible on
+	 * all CPUs to avoid inaccuracies for migrating processes. Leave out
+	 * one tick to make sure that it is not missed.
+	 */
+	PCPU_SET(tickadj, 0);
+	s = intr_disable_all();
+	base = rd(tick);
+	base = roundup(base, tick_increment);
+	PCPU_SET(tickref, base);
+	wrtickcmpr(base + tick_increment, 0);
+	intr_restore_all(s);
+}
+
diff --git a/sys/sun4v/sun4v/tlb.c b/sys/sun4v/sun4v/tlb.c
new file mode 100644
index 000000000000..4d8211f408d9
--- /dev/null
+++ b/sys/sun4v/sun4v/tlb.c
@@ -0,0 +1,74 @@
+/*-
+ * Copyright (c) 2001 Jake Burkholder.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include "opt_pmap.h"
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/ktr.h>
+#include <sys/linker_set.h>
+#include <sys/pcpu.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
+#include <sys/smp.h>
+#include <sys/sysctl.h>
+
+#include <vm/vm.h>
+#include <vm/pmap.h>
+
+#include <machine/pmap.h>
+#include <machine/smp.h>
+#include <machine/tlb.h>
+
+PMAP_STATS_VAR(tlb_ncontext_demap);
+PMAP_STATS_VAR(tlb_npage_demap);
+PMAP_STATS_VAR(tlb_nrange_demap);
+
+tlb_flush_user_t *tlb_flush_user;
+
+/*
+ * Some tlb operations must be atomic, so no interrupt or trap can be allowed
+ * while they are in progress. Traps should not happen, but interrupts need to
+ * be explicitely disabled. critical_enter() cannot be used here, since it only
+ * disables soft interrupts.
+ */
+
+void
+tlb_context_demap(struct pmap *pm)
+{
+}
+
+void
+tlb_page_demap(struct pmap *pm, vm_offset_t va)
+{
+}
+
+void
+tlb_range_demap(struct pmap *pm, vm_offset_t start, vm_offset_t end)
+{
+}
diff --git a/sys/sun4v/sun4v/trap.c b/sys/sun4v/sun4v/trap.c
new file mode 100644
index 000000000000..cf0ecf79bc6a
--- /dev/null
+++ b/sys/sun4v/sun4v/trap.c
@@ -0,0 +1,703 @@
+/*-
+ * Copyright (c) 2001, Jake Burkholder
+ * Copyright (C) 1994, David Greenman
+ * Copyright (c) 1990, 1993
+ *      The Regents of the University of California.  All rights reserved.
+ *
+ * This code is derived from software contributed to Berkeley by
+ * the University of Utah, and William Jolitz.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ *    must display the following acknowledgement:
+ *      This product includes software developed by the University of
+ *      California, Berkeley and its contributors.
+ * 4. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *      from: @(#)trap.c        7.4 (Berkeley) 5/13/91
+ * 	from: FreeBSD: src/sys/i386/i386/trap.c,v 1.197 2001/07/19
+ * $FreeBSD$
+ */
+
+#include "opt_ddb.h"
+#include "opt_ktr.h"
+#include "opt_ktrace.h"
+
+#include <sys/param.h>
+#include <sys/kdb.h>
+#include <sys/kernel.h>
+#include <sys/bus.h>
+#include <sys/interrupt.h>
+#include <sys/ktr.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
+#include <sys/systm.h>
+#include <sys/pioctl.h>
+#include <sys/ptrace.h>
+#include <sys/proc.h>
+#include <sys/smp.h>
+#include <sys/signalvar.h>
+#include <sys/syscall.h>
+#include <sys/sysctl.h>
+#include <sys/sysent.h>
+#include <sys/vmmeter.h>
+#ifdef KTRACE
+#include <sys/uio.h>
+#include <sys/ktrace.h>
+#endif
+
+#include <vm/vm.h>
+#include <vm/pmap.h>
+#include <vm/vm_extern.h>
+#include <vm/vm_param.h>
+#include <vm/vm_kern.h>
+#include <vm/vm_map.h>
+#include <vm/vm_page.h>
+
+#include <machine/clock.h>
+#include <machine/cpu.h>
+#include <machine/frame.h>
+#include <machine/intr_machdep.h>
+#include <machine/pcb.h>
+#include <machine/smp.h>
+#include <machine/trap.h>
+#include <machine/tstate.h>
+#include <machine/tte.h>
+#include <machine/tlb.h>
+#include <machine/tsb.h>
+#include <machine/watch.h>
+#include <machine/wstate.h>
+
+#include <machine/md_var.h>
+#include <machine/hypervisor_api.h>
+
+void trap(struct trapframe *tf, int64_t type, uint64_t data);
+void syscall(struct trapframe *tf);
+
+vm_paddr_t mmu_fault_status_area;
+
+static int trap_pfault(struct thread *td, struct trapframe *tf, int64_t type, uint64_t data);
+
+extern char copy_fault[];
+extern char copy_nofault_begin[];
+extern char copy_nofault_end[];
+
+extern char fs_fault[];
+extern char fs_nofault_begin[];
+extern char fs_nofault_end[];
+extern char fs_nofault_intr_begin[];
+extern char fs_nofault_intr_end[];
+
+extern char fas_fault[];
+extern char fas_nofault_begin[];
+extern char fas_nofault_end[];
+
+extern char *syscallnames[];
+
+static int trap_conversion[256];
+
+const char *trap_msg[] = {
+	"reserved",
+	"instruction access exception",
+	"instruction access error",
+	"instruction access protection",
+	"illtrap instruction",
+	"illegal instruction",
+	"privileged opcode",
+	"floating point disabled",
+	"floating point exception ieee 754",
+	"floating point exception other",
+	"tag overflow",
+	"division by zero",
+	"data access exception",
+	"data access error",
+	"data access protection",
+	"memory address not aligned",
+	"privileged action",
+	"async data error",
+	"trap instruction 16",
+	"trap instruction 17",
+	"trap instruction 18",
+	"trap instruction 19",
+	"trap instruction 20",
+	"trap instruction 21",
+	"trap instruction 22",
+	"trap instruction 23",
+	"trap instruction 24",
+	"trap instruction 25",
+	"trap instruction 26",
+	"trap instruction 27",
+	"trap instruction 28",
+	"trap instruction 29",
+	"trap instruction 30",
+	"trap instruction 31",
+	"fast instruction access mmu miss",
+	"fast data access mmu miss",
+	"interrupt",
+	"physical address watchpoint",
+	"virtual address watchpoint",
+	"corrected ecc error",
+	"spill",
+	"fill",
+	"fill",
+	"breakpoint",
+	"clean window",
+	"range check",
+	"fix alignment",
+	"integer overflow",
+	"syscall",
+	"restore physical watchpoint",
+	"restore virtual watchpoint",
+	"kernel stack fault",
+	"resumable error",
+	"nonresumable error"
+};
+
+const int trap_sig[] = {
+	SIGILL,			/* reserved */
+	SIGILL,			/* instruction access exception */
+	SIGILL,			/* instruction access error */
+	SIGILL,			/* instruction access protection */
+	SIGILL,			/* illtrap instruction */
+	SIGILL,			/* illegal instruction */
+	SIGBUS,			/* privileged opcode */
+	SIGFPE,			/* floating point disabled */
+	SIGFPE,			/* floating point exception ieee 754 */
+	SIGFPE,			/* floating point exception other */
+	SIGEMT,			/* tag overflow */
+	SIGFPE,			/* division by zero */
+	SIGILL,			/* data access exception */
+	SIGILL,			/* data access error */
+	SIGBUS,			/* data access protection */
+	SIGBUS,			/* memory address not aligned */
+	SIGBUS,			/* privileged action */
+	SIGBUS,			/* async data error */
+	SIGILL,			/* trap instruction 16 */
+	SIGILL,			/* trap instruction 17 */
+	SIGILL,			/* trap instruction 18 */
+	SIGILL,			/* trap instruction 19 */
+	SIGILL,			/* trap instruction 20 */
+	SIGILL,			/* trap instruction 21 */
+	SIGILL,			/* trap instruction 22 */
+	SIGILL,			/* trap instruction 23 */
+	SIGILL,			/* trap instruction 24 */
+	SIGILL,			/* trap instruction 25 */
+	SIGILL,			/* trap instruction 26 */
+	SIGILL,			/* trap instruction 27 */
+	SIGILL,			/* trap instruction 28 */
+	SIGILL,			/* trap instruction 29 */
+	SIGILL,			/* trap instruction 30 */
+	SIGILL,			/* trap instruction 31 */
+	SIGSEGV,		/* floating point not implemented */
+                                /* should be SIGFPE but other signals currently cause problems */
+	SIGSEGV,		/* fast data access mmu miss */
+	-1,			/* interrupt */
+	-1,			/* physical address watchpoint */
+	-1,			/* virtual address watchpoint */
+	-1,			/* corrected ecc error */
+	SIGILL,			/* spill */
+	SIGILL,			/* fill */
+	SIGILL,			/* fill */
+	SIGTRAP,		/* breakpoint */
+	SIGILL,			/* clean window */
+	SIGILL,			/* range check */
+	SIGILL,			/* fix alignment */
+	SIGILL,			/* integer overflow */
+	SIGSYS,			/* syscall */
+	-1,			/* restore physical watchpoint */
+	-1,			/* restore virtual watchpoint */
+	-1,			/* kernel stack fault */
+};
+
+CTASSERT(sizeof(struct trapframe) == 256);
+
+int debugger_on_signal = 0;
+#ifdef DEBUG
+SYSCTL_INT(_debug, OID_AUTO, debugger_on_signal, CTLFLAG_RW,
+    &debugger_on_signal, 0, "");
+#endif
+
+void 
+trap_init(void)
+{
+	vm_paddr_t mmfsa;
+	int i;
+
+	mmfsa = mmu_fault_status_area + (MMFSA_SIZE*PCPU_GET(cpuid));
+	set_wstate(WSTATE_KERN);
+	set_mmfsa_scratchpad(mmfsa);
+
+	init_mondo_queue();
+	OF_set_mmfsa_traptable(&tl0_base, mmfsa);
+	for (i = 0; i < 128; i++)
+		trap_conversion[i] = i;
+	for (i = 128; i < 256; i++)
+		trap_conversion[i] = 0;
+	trap_conversion[0x31] = 35;
+	trap_conversion[0x34] = 15;
+	trap_conversion[0x9] = 34;
+	trap_conversion[0x6c] = 14;
+
+}
+
+void
+trap(struct trapframe *tf, int64_t type, uint64_t data)
+{
+	struct thread *td;
+	struct proc *p;
+	int error, sig, ctx;
+	uint64_t trapno;
+	register_t addr;
+	ksiginfo_t ksi;
+
+	td = PCPU_GET(curthread);
+
+	CTR4(KTR_TRAP, "trap: %p type=%s (%s) pil=%#lx", td,
+	    trap_msg[trap_conversion[trapno]],
+	    (TRAPF_USERMODE(tf) ? "user" : "kernel"), rdpr(pil));
+
+	PCPU_LAZY_INC(cnt.v_trap);
+
+	trapno = (type & TRAP_MASK);
+	ctx = (type >> TRAP_CTX_SHIFT);
+
+	if (((tf->tf_tstate & TSTATE_PRIV) == 0) || (ctx != 0)) {
+		KASSERT(td != NULL, ("trap: curthread NULL"));
+		KASSERT(td->td_proc != NULL, ("trap: curproc NULL"));
+
+		p = td->td_proc;
+		td->td_pticks = 0;
+		td->td_frame = tf;
+		addr = tf->tf_tpc;
+		if (td->td_ucred != p->p_ucred)
+			cred_update_thread(td);
+
+		switch (trapno) {
+		case T_DATA_MISS:
+		case T_DATA_PROTECTION:
+			addr = TLB_TAR_VA(data);
+		case T_INSTRUCTION_MISS:
+			sig = trap_pfault(td, tf, trapno, data);
+			break;
+		case T_FILL:
+			sig = rwindow_load(td, tf, 2);
+			break;
+		case T_FILL_RET:
+			sig = rwindow_load(td, tf, 1);
+			break;
+		case T_SPILL:
+			sig = rwindow_save(td);
+			break;
+		case T_DATA_EXCEPTION:
+		case T_DATA_ERROR:
+		case T_MEM_ADDRESS_NOT_ALIGNED:
+			addr = data;
+			sig = trap_sig[trap_conversion[trapno]];
+			break;
+
+		default:
+			if (trapno < 0 || trapno >= T_MAX ||
+			    trap_sig[trapno] == -1)
+				panic("trap: bad trap type");
+			sig = trap_sig[trap_conversion[trapno]];
+			break;
+		}
+
+		if (sig != 0) {
+			/* Translate fault for emulators. */
+			if (p->p_sysent->sv_transtrap != NULL) {
+				sig = p->p_sysent->sv_transtrap(sig,
+				    trapno);
+			}
+			if (debugger_on_signal &&
+			    (sig == 4 || sig == 10 || sig == 11))
+				kdb_enter("trapsig");
+#ifdef VERBOSE
+			if (sig == 4 || sig == 10 || sig == 11)
+				printf("trap: %ld:%s: 0x%lx at 0x%lx on cpu=%d sig=%d\n", trapno, 
+				       trap_msg[trap_conversion[trapno]], data, tf->tf_tpc, curcpu, sig);
+#endif
+			/* XXX I've renumbered the traps to largely reflect what the hardware uses
+			 * so this will need to be re-visited
+			 */
+			ksiginfo_init_trap(&ksi);
+			ksi.ksi_signo = sig;
+			ksi.ksi_code = (int)trap_conversion[trapno]; /* XXX not POSIX */
+			ksi.ksi_addr = (void *)addr;
+			ksi.ksi_trapno = (int)trap_conversion[trapno];
+			trapsignal(td, &ksi);
+		}
+
+		userret(td, tf);
+		mtx_assert(&Giant, MA_NOTOWNED);
+ 	} else {
+		KASSERT((type & T_KERNEL) != 0,
+		    ("trap: kernel trap isn't"));
+
+#ifdef KDB
+		if (kdb_active) {
+			kdb_reenter();
+			return;
+		}
+#endif
+
+		switch (trapno) {
+#ifdef KDB
+		case T_BREAKPOINT:
+		case T_KSTACK_FAULT:
+			error = (kdb_trap(trapno, 0, tf) == 0);
+			TF_DONE(tf);
+			break;
+#endif
+		case T_DATA_MISS:
+		case T_DATA_PROTECTION:
+		case T_INSTRUCTION_MISS:
+			error = trap_pfault(td, tf, trapno, data);
+			break;
+		case T_DATA_EXCEPTION:
+			printf("data exception on 0x%lx at 0x%lx\n", data, tf->tf_tpc);
+			printf("trap: %ld=%s: 0x%lx at 0x%lx:0x%lx\n", trapno, 
+			       trap_msg[trap_conversion[trapno]], data, tf->tf_tpc, tf->tf_tnpc);
+               case T_ILLEGAL_INSTRUCTION:
+                       if (tf->tf_tpc > KERNBASE) {
+                               printf("illinstr: 0x%lx\n", tf->tf_tpc);
+                               printf("illinstr: 0x%x\n", *((uint32_t *)tf->tf_tpc));
+                       }
+		case T_DATA_ERROR:
+		case T_ALIGNMENT:
+			if (tf->tf_asi == ASI_AIUS) {
+				if (tf->tf_tpc >= (u_long)copy_nofault_begin &&
+				    tf->tf_tpc <= (u_long)copy_nofault_end) {
+					tf->tf_tpc = (u_long)copy_fault;
+					tf->tf_tnpc = tf->tf_tpc + 4;
+					error = 0;
+					break;
+				}
+				
+				printf("ASI_AIUS but bad tpc\n");
+			} 
+			if (tf->tf_tpc >= (u_long)fs_nofault_begin &&
+			    tf->tf_tpc <= (u_long)fs_nofault_end) {
+				tf->tf_tpc = (u_long)fs_fault;
+				tf->tf_tnpc = tf->tf_tpc + 4;
+				error = 0;
+					break;
+			}
+			printf("asi=0x%lx\n", tf->tf_asi);
+			error = 1;	
+			break;
+		default:
+			printf("unchecked trap 0x%lx asi=0x%lx\n", trapno, tf->tf_asi);
+			error = 1;
+			break;
+		}
+
+		if (error != 0)
+			panic("trap: %ld=%s: 0x%lx at 0x%lx:0x%lx error=%d asi=0x%lx", 
+			      trapno, trap_msg[trap_conversion[trapno]], data, tf->tf_tpc, 
+			      tf->tf_tnpc, error, tf->tf_asi);
+	}
+	CTR1(KTR_TRAP, "trap: td=%p return", td);
+}
+
+static int
+trap_pfault(struct thread *td, struct trapframe *tf, int64_t type, uint64_t data)
+{
+	struct vmspace *vm;
+	struct pcb *pcb;
+	struct proc *p;
+	vm_offset_t va;
+	vm_prot_t prot;
+	u_long ctx;
+	int flags;
+	int rv;
+
+	if (td == NULL)
+		return (-1);
+	KASSERT(td->td_pcb != NULL, ("trap_pfault: pcb NULL"));
+	KASSERT(td->td_proc != NULL, ("trap_pfault: curproc NULL"));
+	KASSERT(td->td_proc->p_vmspace != NULL, ("trap_pfault: vmspace NULL"));
+
+	p = td->td_proc;
+
+	rv = KERN_SUCCESS;
+	ctx = TLB_TAR_CTX(data);
+	pcb = td->td_pcb;
+	type = type & ~T_KERNEL;
+	va = TLB_TAR_VA(data);
+
+#if 0
+	printf("trap_pfault(type=%ld, data=0x%lx, tpc=0x%lx, ctx=0x%lx)\n", 
+	       type, data, tf->tf_tpc, ctx);
+#endif
+
+	CTR4(KTR_TRAP, "trap_pfault: td=%p pm_ctx=%#lx va=%#lx ctx=%#lx",
+	    td, p->p_vmspace->vm_pmap.pm_context[PCPU_GET(cpuid)], va, ctx);
+
+	if (type == T_DATA_PROTECTION) {
+		prot = VM_PROT_WRITE;
+		flags = VM_FAULT_DIRTY;
+	} else {
+		if (type == T_DATA_MISS)
+			prot = VM_PROT_READ;
+		else
+			prot = VM_PROT_READ | VM_PROT_EXECUTE;
+		flags = VM_FAULT_NORMAL;
+	}
+
+	if (ctx != TLB_CTX_KERNEL) {
+		if ((tf->tf_tstate & TSTATE_PRIV) != 0 &&
+		    (tf->tf_tpc >= (u_long)fs_nofault_intr_begin &&
+		     tf->tf_tpc <= (u_long)fs_nofault_intr_end)) {
+			tf->tf_tpc = (u_long)fs_fault;
+			tf->tf_tnpc = tf->tf_tpc + 4;
+			return (0);
+		}
+
+		/*
+		 * This is a fault on non-kernel virtual memory.
+		 */
+		vm = p->p_vmspace;
+
+		/*
+		 * Keep swapout from messing with us during this
+		 * critical time.
+		 */
+		PROC_LOCK(p);
+		++p->p_lock;
+		PROC_UNLOCK(p);
+
+		/* Fault in the user page. */
+		rv = vm_fault(&vm->vm_map, va, prot, flags);
+
+		/*
+		 * Now the process can be swapped again.
+		 */
+		PROC_LOCK(p);
+		--p->p_lock;
+		PROC_UNLOCK(p);
+	} else {
+		/*
+		 * This is a fault on kernel virtual memory.  Attempts to
+		 * access kernel memory from user mode cause privileged
+		 * action traps, not page fault.
+		 */
+		KASSERT(tf->tf_tstate & TSTATE_PRIV,
+		    ("trap_pfault: fault on nucleus context from user mode"));
+
+		/*
+		 * Don't have to worry about process locking or stacks in the
+		 * kernel.
+		 */
+		rv = vm_fault(kernel_map, va, prot, VM_FAULT_NORMAL);
+	}
+
+	CTR3(KTR_TRAP, "trap_pfault: return td=%p va=%#lx rv=%d",
+	    td, va, rv);
+	if (rv == KERN_SUCCESS)
+		return (0);
+	if (ctx != TLB_CTX_KERNEL && (tf->tf_tstate & TSTATE_PRIV) != 0) {
+		if (tf->tf_tpc >= (u_long)fs_nofault_begin &&
+		    tf->tf_tpc <= (u_long)fs_nofault_end) {
+			tf->tf_tpc = (u_long)fs_fault;
+			tf->tf_tnpc = tf->tf_tpc + 4;
+			return (0);
+		}
+		if (tf->tf_tpc >= (u_long)copy_nofault_begin &&
+		    tf->tf_tpc <= (u_long)copy_nofault_end) {
+			tf->tf_tpc = (u_long)copy_fault;
+			tf->tf_tnpc = tf->tf_tpc + 4;
+			return (0);
+		}
+	}
+	return ((rv == KERN_PROTECTION_FAILURE) ? SIGBUS : SIGSEGV);
+}
+
+/* Maximum number of arguments that can be passed via the out registers. */
+#define	REG_MAXARGS	6
+
+/*
+ * Syscall handler. The arguments to the syscall are passed in the o registers
+ * by the caller, and are saved in the trap frame. The syscall number is passed
+ * in %g1 (and also saved in the trap frame).
+ */
+void
+syscall(struct trapframe *tf)
+{
+	struct sysent *callp;
+	struct thread *td;
+	register_t args[8];
+	register_t *argp;
+	struct proc *p;
+	u_long code;
+	u_long tpc;
+	int reg;
+	int regcnt;
+	int narg;
+	int error;
+
+	td = PCPU_GET(curthread);
+	KASSERT(td != NULL, ("trap: curthread NULL"));
+	KASSERT(td->td_proc != NULL, ("trap: curproc NULL"));
+
+	p = td->td_proc;
+
+	PCPU_LAZY_INC(cnt.v_syscall);
+
+	narg = 0;
+	error = 0;
+	reg = 0;
+	regcnt = REG_MAXARGS;
+
+	td->td_pticks = 0;
+	td->td_frame = tf;
+	if (td->td_ucred != p->p_ucred)
+		cred_update_thread(td);
+	code = tf->tf_global[1];
+
+	/*
+	 * For syscalls, we don't want to retry the faulting instruction
+	 * (usually), instead we need to advance one instruction.
+	 */
+	tpc = tf->tf_tpc;
+	TF_DONE(tf);
+
+	if (p->p_sysent->sv_prepsyscall) {
+		/*
+		 * The prep code is MP aware.
+		 */
+#if 0
+		(*p->p_sysent->sv_prepsyscall)(tf, args, &code, &params);
+#endif	
+	} else 	if (code == SYS_syscall || code == SYS___syscall) {
+		code = tf->tf_out[reg++];
+		regcnt--;
+	}
+
+ 	if (p->p_sysent->sv_mask)
+ 		code &= p->p_sysent->sv_mask;
+
+ 	if (code >= p->p_sysent->sv_size)
+ 		callp = &p->p_sysent->sv_table[0];
+  	else
+ 		callp = &p->p_sysent->sv_table[code];
+
+	narg = callp->sy_narg;
+
+	if (narg <= regcnt) {
+		argp = &tf->tf_out[reg];
+		error = 0;
+	} else {
+		KASSERT(narg <= sizeof(args) / sizeof(args[0]),
+		    ("Too many syscall arguments!"));
+		argp = args;
+		bcopy(&tf->tf_out[reg], args, sizeof(args[0]) * regcnt);
+		error = copyin((void *)(tf->tf_out[6] + SPOFF +
+		    offsetof(struct frame, fr_pad[6])),
+		    &args[regcnt], (narg - regcnt) * sizeof(args[0]));
+	}
+
+	CTR5(KTR_SYSC, "syscall: td=%p %s(%#lx, %#lx, %#lx)", td,
+	    syscallnames[code], argp[0], argp[1], argp[2]);
+
+	/*
+	 * Try to run the syscall without the MP lock if the syscall
+	 * is MP safe.
+	 */
+#ifdef KTRACE
+	if (KTRPOINT(td, KTR_SYSCALL))
+		ktrsyscall(code, narg, argp);
+#endif
+	if (error == 0) {
+		td->td_retval[0] = 0;
+		td->td_retval[1] = 0;
+
+		STOPEVENT(p, S_SCE, narg);	/* MP aware */
+
+		PTRACESTOP_SC(p, td, S_PT_SCE);
+
+		error = (*callp->sy_call)(td, argp);
+
+		CTR5(KTR_SYSC, "syscall: p=%p error=%d %s return %#lx %#lx ", p,
+		    error, syscallnames[code], td->td_retval[0],
+		    td->td_retval[1]);
+	}
+	
+	/*
+	 * MP SAFE (we may or may not have the MP lock at this point)
+	 */
+	switch (error) {
+	case 0:
+		tf->tf_out[0] = td->td_retval[0];
+		tf->tf_out[1] = td->td_retval[1];
+		tf->tf_tstate &= ~TSTATE_XCC_C;
+		break;
+
+	case ERESTART:
+		/*
+		 * Undo the tpc advancement we have done above, we want to
+		 * reexecute the system call.
+		 */
+		tf->tf_tpc = tpc;
+		tf->tf_tnpc -= 4;
+		break;
+
+	case EJUSTRETURN:
+		break;
+
+	default:
+ 		if (p->p_sysent->sv_errsize) {
+ 			if (error >= p->p_sysent->sv_errsize)
+  				error = -1;	/* XXX */
+   			else
+  				error = p->p_sysent->sv_errtbl[error];
+		}
+		tf->tf_out[0] = error;
+		tf->tf_tstate |= TSTATE_XCC_C;
+		break;
+	}
+
+	/*
+	 * Handle reschedule and other end-of-syscall issues
+	 */
+	userret(td, tf);
+
+#ifdef KTRACE
+	if (KTRPOINT(td, KTR_SYSRET))
+		ktrsysret(code, error, td->td_retval[0]);
+#endif
+	/*
+	 * This works because errno is findable through the
+	 * register set.  If we ever support an emulation where this
+	 * is not the case, this code will need to be revisited.
+	 */
+	STOPEVENT(p, S_SCX, code);
+
+	PTRACESTOP_SC(p, td, S_PT_SCX);
+
+	WITNESS_WARN(WARN_PANIC, NULL, "System call %s returning",
+	    (code >= 0 && code < SYS_MAXSYSCALL) ? syscallnames[code] : "???");
+	mtx_assert(&sched_lock, MA_NOTOWNED);
+	mtx_assert(&Giant, MA_NOTOWNED);
+}
diff --git a/sys/sun4v/sun4v/trap_trace.S b/sys/sun4v/sun4v/trap_trace.S
new file mode 100644
index 000000000000..03341c90751f
--- /dev/null
+++ b/sys/sun4v/sun4v/trap_trace.S
@@ -0,0 +1,38 @@
+/*-
+ * Copyright (c) 2006 John Birrell jb@freebsd.org
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <machine/asm.h>
+__FBSDID("$FreeBSD$")				
+	
+#include "opt_trap_trace.h"
+
+#include <machine/param.h>
+
+	.data
+	.globl	trap_trace_entry
+	.align	64
+trap_trace_entry:
+	.skip	MAXCPU * TRAP_TRACE_ENTRIES * 64
diff --git a/sys/sun4v/sun4v/tsb.c b/sys/sun4v/sun4v/tsb.c
new file mode 100644
index 000000000000..83ed9260a164
--- /dev/null
+++ b/sys/sun4v/sun4v/tsb.c
@@ -0,0 +1,313 @@
+/*-
+ * Copyright (c) 2006 Kip Macy
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+
+#include "opt_ddb.h"
+#include "opt_pmap.h"
+
+#include <sys/param.h>
+#include <sys/queue.h>
+#include <sys/ktr.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
+#include <sys/smp.h>
+#include <sys/sysctl.h>
+#include <sys/systm.h>
+
+#include <vm/vm.h> 
+#include <vm/vm_extern.h> 
+#include <vm/vm_page.h>
+#include <vm/vm_pageout.h>
+
+#include <machine/cpufunc.h>
+#include <machine/hypervisorvar.h>
+#include <machine/smp.h>
+#include <machine/mmu.h>
+#include <machine/tte.h>
+#include <machine/tte_hash.h>
+#include <machine/tsb.h>
+#include <machine/vmparam.h>
+#include <machine/tlb.h>
+
+CTASSERT(sizeof(tte_t) == sizeof(uint64_t));
+#define TSB_MASK(tsb) ((tsb->hvtsb_ntte) - 1)
+/* make TSB start off at the same size as the hash */
+#define TSB_SIZE       8
+
+#ifdef DEBUG_TSB
+#define DPRINTF printf
+#else
+#define DPRINTF(...)
+#endif
+
+void tsb_sysinit(void);
+
+vm_paddr_t
+tsb_init(hv_tsb_info_t *hvtsb, uint64_t *scratchval)
+{
+	vm_page_t m;
+	int i;
+	uint64_t tsb_pages;	
+
+	m = NULL;
+	while (m == NULL) {
+		m = vm_page_alloc_contig(TSB_SIZE, phys_avail[0], 
+					 phys_avail[1], TSB_SIZE*PAGE_SIZE, (1UL<<34));
+		if (m == NULL) {
+			printf("vm_page_alloc_contig failed - waiting to retry\n");
+			VM_WAIT;
+		}
+	}
+	if ((VM_PAGE_TO_PHYS(m) & (TSB_SIZE*PAGE_SIZE - 1)) != 0)
+	    panic("vm_page_alloc_contig allocated unaligned pages: 0x%lx",
+		  VM_PAGE_TO_PHYS(m));
+
+	hvtsb->hvtsb_idxpgsz = TTE8K;
+	hvtsb->hvtsb_assoc = 1;
+	hvtsb->hvtsb_ntte = (TSB_SIZE*PAGE_SIZE >> TTE_SHIFT);
+	hvtsb->hvtsb_ctx_index = -1;    /* TSBs aren't shared so if we don't 
+					 * set the context in the TTEs we can 
+					 * simplify miss handling slightly
+					 */
+	hvtsb->hvtsb_pgszs = TSB8K;
+	hvtsb->hvtsb_rsvd = 0;
+	hvtsb->hvtsb_pa = VM_PAGE_TO_PHYS(m);
+
+	for (i = 0; i < TSB_SIZE; i++, m++) 
+		if ((m->flags & PG_ZERO) == 0)
+			pmap_zero_page(m);
+
+	tsb_pages = hvtsb->hvtsb_ntte >> (PAGE_SHIFT - TTE_SHIFT);
+	*scratchval = TLB_PHYS_TO_DIRECT(hvtsb->hvtsb_pa) | tsb_pages;
+
+	return vtophys(hvtsb);
+}
+
+void
+tsb_deinit(hv_tsb_info_t *hvtsb)
+{
+	vm_page_t m, tm;
+	int i;
+	
+
+	m = PHYS_TO_VM_PAGE((vm_paddr_t)hvtsb->hvtsb_pa);
+	for (i = 0, tm = m; i < TSB_SIZE; i++, m++) {
+		tm->wire_count--;
+		atomic_subtract_int(&cnt.v_wire_count, 1);
+	}
+	vm_page_release_contig(m, TSB_SIZE);
+}
+
+
+void
+tsb_assert_invalid(hv_tsb_info_t *tsb, vm_offset_t va)
+{
+	vm_paddr_t tsb_load_pa;
+	uint64_t tsb_index, tsb_shift, tte_tag, tte_data;
+	tsb_shift = TTE_PAGE_SHIFT(tsb->hvtsb_idxpgsz);
+	tsb_index = (va >> tsb_shift) & TSB_MASK(tsb);
+	tsb_load_pa = tsb->hvtsb_pa + 2*tsb_index*sizeof(uint64_t);
+	load_real_dw(tsb_load_pa, &tte_tag, &tte_data);
+	if (tte_tag == 0 && tte_data == 0)
+		return;
+	printf("tsb_shift=0x%lx tsb_index=0x%lx\n", tsb_shift, tsb_index);
+	printf("tte_tag=0x%lx tte_data=0x%lx TSB_MASK=%lx\n", tte_tag, tte_data, (uint64_t)TSB_MASK(tsb));
+	panic("non-zero entry found where not expected");
+
+}
+
+void 
+tsb_set_tte_real(hv_tsb_info_t *tsb, vm_offset_t va, uint64_t tte_data, uint64_t ctx)
+{
+	vm_paddr_t tsb_store_pa;
+	uint64_t tsb_index, tsb_shift, tte_tag;
+	DPRINTF("tsb_set_tte va: 0x%lx idxpgsz: %x\n", va, tsb->hvtsb_idxpgsz);
+
+	tsb_shift = TTE_PAGE_SHIFT(tsb->hvtsb_idxpgsz);
+
+	DPRINTF("tsb_shift: 0x%lx\n", tsb_shift);
+	tsb_index = (va >> tsb_shift) & TSB_MASK(tsb);
+	DPRINTF("tsb_index_absolute: 0x%lx tsb_index: 0x%lx\n", (va >> tsb_shift), tsb_index);
+	tsb_store_pa = tsb->hvtsb_pa + 2*tsb_index*sizeof(uint64_t);
+
+	tte_data &= ~VTD_V;
+	/* store new value with valid bit cleared 
+	 * to avoid invalid intermediate value;
+	 */
+	store_real(tsb_store_pa + sizeof(uint64_t), tte_data);
+	tte_data |= VTD_V;
+
+	tte_tag = (ctx << TTARGET_CTX_SHIFT) | (va >> TTARGET_VA_SHIFT);
+	store_real(tsb_store_pa, tte_tag); 
+	store_real(tsb_store_pa + sizeof(uint64_t), tte_data);
+} 
+
+
+void 
+tsb_set_tte(hv_tsb_info_t *tsb, vm_offset_t va, uint64_t tte_data, uint64_t ctx)
+{
+
+	uint64_t tsb_index, tsb_shift, tte_tag;
+	tte_t *entry;
+
+	tsb_shift = TTE_PAGE_SHIFT(tsb->hvtsb_idxpgsz);
+	tsb_index = (va >> tsb_shift) & TSB_MASK(tsb);
+	entry = (tte_t *)TLB_PHYS_TO_DIRECT(tsb->hvtsb_pa + 2*tsb_index*sizeof(uint64_t));
+	tte_tag = (ctx << TTARGET_CTX_SHIFT) | (va >> TTARGET_VA_SHIFT);
+	/* store new value with valid bit cleared 
+	 * to avoid invalid intermediate value;
+	 */
+	*(entry + 1) = 0;
+	membar(StoreLoad);
+	*(entry) = tte_tag;
+	*(entry + 1) = tte_data;
+	membar(Sync);
+} 
+
+
+void 
+tsb_clear(hv_tsb_info_t *tsb)
+{
+	hwblkclr((void *)TLB_PHYS_TO_DIRECT(tsb->hvtsb_pa), tsb->hvtsb_ntte << TTE_SHIFT);
+}
+
+void 
+tsb_clear_tte(hv_tsb_info_t *tsb, vm_offset_t va)
+{
+	tte_t *entry;
+	uint64_t tsb_index, tsb_shift;
+
+	tsb_shift = TTE_PAGE_SHIFT(tsb->hvtsb_idxpgsz);
+	tsb_index = (va >> tsb_shift) & TSB_MASK(tsb);
+	entry = (tte_t *)TLB_PHYS_TO_DIRECT(tsb->hvtsb_pa + 2*tsb_index*sizeof(uint64_t));
+	
+	*(entry + 1) = 0;
+
+	membar(Sync);
+}
+
+void
+tsb_clear_range(hv_tsb_info_t *tsb, vm_offset_t sva, vm_offset_t eva)
+{
+	vm_offset_t tva;
+	uint64_t tsb_index, tsb_shift, tsb_mask;
+	tte_t *entry;
+
+	tsb_mask = TSB_MASK(tsb);
+	tsb_shift = TTE_PAGE_SHIFT(tsb->hvtsb_idxpgsz);
+
+	for (tva = sva; tva < eva; tva += PAGE_SIZE) {
+		tsb_index = (tva >> tsb_shift) & tsb_mask;
+		entry = (tte_t *)TLB_PHYS_TO_DIRECT(tsb->hvtsb_pa + 2*tsb_index*sizeof(uint64_t));
+		*(entry + 1) = 0;
+	}
+
+	membar(Sync);
+}
+
+tte_t
+tsb_get_tte(hv_tsb_info_t *tsb, vm_offset_t va)
+{
+	tte_t *entry;
+	uint64_t tsb_index, tsb_shift, tte_tag, tte_data;
+
+	tsb_shift = TTE_PAGE_SHIFT(tsb->hvtsb_idxpgsz);
+	tsb_index = (va >> tsb_shift) & TSB_MASK(tsb);
+	entry = (tte_t *)TLB_PHYS_TO_DIRECT(tsb->hvtsb_pa + 2*tsb_index*sizeof(uint64_t));
+	tte_tag = *(entry);
+	tte_data = *(entry + 1);
+
+	if ((tte_tag << TTARGET_VA_SHIFT) == (va & ~PAGE_MASK_4M))
+		return tte_data;
+
+	return (0UL);
+}
+#if 0
+tte_t
+tsb_get_tte_real(hv_tsb_info_t *tsb, vm_offset_t va)
+{
+	vm_paddr_t tsb_load_pa;
+	uint64_t tsb_index, tsb_shift, tte_tag, tte_data;
+
+	DPRINTF("tsb_get_tte va: 0x%lx\n", va);
+	tsb_shift = TTE_PAGE_SHIFT(tsb->hvtsb_idxpgsz);
+	DPRINTF("tsb_shift: %lx\n", tsb_shift);
+	tsb_index = (va >> tsb_shift) & TSB_MASK(tsb);
+	DPRINTF("tsb_index_absolute: %lx tsb_index: %lx\n", (va >> tsb_shift), tsb_index);
+	tsb_load_pa = tsb->hvtsb_pa + 2*tsb_index*sizeof(uint64_t);
+	DPRINTF("load_real_dw - ra: %lx &tte_tag: %p &tte_data: %p \n", tsb_load_pa, &tte_tag, &tte_data);
+	load_real_dw(tsb_load_pa, &tte_tag, &tte_data);
+	DPRINTF("tte_data: %lx ctx: %lx  va: %lx\n", tte_data, tte_tag >> TTARGET_CTX_SHIFT, 
+		tte_tag << TTARGET_VA_SHIFT);
+	if ((tte_tag << TTARGET_VA_SHIFT) == (va & ~PAGE_MASK_4M))
+		return tte_data;
+
+	return (0UL);
+}
+#endif
+
+tte_t
+tsb_lookup_tte(vm_offset_t va, uint64_t ctx)
+{
+	tte_t tte_data;
+
+	tte_data = 0;
+
+	if ((tte_data = tsb_get_tte(&kernel_td[TSB4M_INDEX], va)) != 0)
+		goto done;
+
+	/*
+	 * handle user data 
+	 */
+done:
+	return tte_data;
+}
+
+uint64_t
+tsb_set_scratchpad_kernel(hv_tsb_info_t *tsb)
+{
+	uint64_t tsb_pages, tsb_scratch;	
+	tsb_pages = tsb->hvtsb_ntte >> (PAGE_SHIFT - TTE_SHIFT);
+	tsb_scratch = TLB_PHYS_TO_DIRECT(tsb->hvtsb_pa) | tsb_pages;
+	
+	set_tsb_kernel_scratchpad(tsb_scratch);
+	membar(Sync);
+	return tsb_scratch;
+}
+
+uint64_t
+tsb_set_scratchpad_user(hv_tsb_info_t *tsb)
+{
+	uint64_t tsb_pages, tsb_scratch;	
+	tsb_pages = tsb->hvtsb_ntte >> (PAGE_SHIFT - TTE_SHIFT);
+	tsb_scratch = TLB_PHYS_TO_DIRECT(tsb->hvtsb_pa) | tsb_pages;
+	set_tsb_user_scratchpad(tsb_scratch);
+	membar(Sync);
+	return tsb_scratch;
+}
diff --git a/sys/sun4v/sun4v/tte.c b/sys/sun4v/sun4v/tte.c
new file mode 100644
index 000000000000..1325ac88f616
--- /dev/null
+++ b/sys/sun4v/sun4v/tte.c
@@ -0,0 +1,152 @@
+/*-
+ * Copyright (c) 2006 Kip Macy
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include "opt_ddb.h"
+#include "opt_pmap.h"
+
+#include <sys/param.h>
+#include <sys/queue.h>
+#include <sys/ktr.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
+#include <sys/proc.h>
+#include <sys/sched.h>
+#include <sys/smp.h>
+#include <sys/sysctl.h>
+#include <sys/systm.h>
+
+#include <vm/vm.h> 
+#include <vm/vm_page.h>
+
+#include <machine/cpufunc.h>
+#include <machine/smp.h>
+#include <machine/mmu.h>
+#include <machine/tte.h>
+#include <machine/cpu.h>
+#include <machine/tte_hash.h>
+
+void 
+tte_clear_phys_bit(vm_page_t m, uint64_t flags)
+{
+	pv_entry_t pv;
+
+	if ((m->flags & PG_FICTITIOUS) ||
+	    (flags == VTD_SW_W && (m->flags & PG_WRITEABLE) == 0))
+		return;
+	sched_pin();
+	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+	/*
+	 * Loop over all current mappings setting/clearing as appropos If
+	 * setting RO do we need to clear the VAC?
+	 */
+	TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
+		tte_t otte_data;
+		/*
+		 * don't write protect pager mappings
+		 */
+		if (flags == VTD_SW_W) {
+			if (!pmap_track_modified(pv->pv_pmap, pv->pv_va))
+				continue;
+			flags = (VTD_SW_W|VTD_W);
+		}
+		otte_data = tte_hash_clear_bits(pv->pv_pmap->pm_hash, pv->pv_va, flags);
+
+		if (otte_data & flags) {
+			if (otte_data & VTD_W) 
+				vm_page_dirty(m);
+
+			pmap_invalidate_page(pv->pv_pmap, pv->pv_va, TRUE);
+		}
+		    
+		
+	}
+	if (flags & VTD_SW_W)
+		vm_page_flag_clear(m, PG_WRITEABLE);
+	sched_unpin();
+}
+
+boolean_t 
+tte_get_phys_bit(vm_page_t m, uint64_t flags)
+{
+
+	pv_entry_t pv;
+	pmap_t pmap;
+	boolean_t rv;
+	
+	rv = FALSE;
+	if (m->flags & PG_FICTITIOUS)
+		return (rv);
+
+	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+	TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
+		tte_t otte_data;
+
+		pmap = pv->pv_pmap;
+		otte_data = tte_hash_lookup(pmap->pm_hash, pv->pv_va);
+		rv = ((otte_data & flags) != 0);
+		if (rv)
+			break;
+	}
+
+	return (rv);
+}
+
+void 
+tte_clear_virt_bit(struct pmap *pmap, vm_offset_t va, uint64_t flags)
+{
+	tte_t otte_data;
+	
+	if (flags == VTD_SW_W) {
+		if (!pmap_track_modified(pmap, va))
+			return;
+		flags = (VTD_SW_W|VTD_W);
+	}
+
+	otte_data = tte_hash_clear_bits(pmap->pm_hash, va, flags);
+
+	if (otte_data & flags) 
+		pmap_invalidate_page(pmap, va, TRUE);
+}
+
+void 
+tte_set_virt_bit(struct pmap *pmap, vm_offset_t va, uint64_t flags)
+{
+	UNIMPLEMENTED;
+}
+
+boolean_t 
+tte_get_virt_bit(struct pmap *pmap, vm_offset_t va, uint64_t flags)
+{
+	tte_t tte_data;
+	
+	tte_data = tte_hash_lookup(pmap->pm_hash, va);
+	
+	return ((tte_data & flags) == flags);
+}
diff --git a/sys/sun4v/sun4v/tte_hash.c b/sys/sun4v/sun4v/tte_hash.c
new file mode 100644
index 000000000000..5a47b93d9ada
--- /dev/null
+++ b/sys/sun4v/sun4v/tte_hash.c
@@ -0,0 +1,548 @@
+/*-
+ * Copyright (c) 2006 Kip Macy
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/queue.h>
+#include <sys/ktr.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
+#include <sys/proc.h>
+#include <sys/smp.h>
+#include <sys/sysctl.h>
+#include <sys/systm.h>
+#ifdef DEBUG
+#include <sys/kdb.h>
+#endif
+#include <vm/vm.h> 
+#include <vm/vm_object.h>
+#include <vm/vm_page.h>
+#include <vm/vm_kern.h>
+#include <vm/vm_pageout.h>
+#include <vm/vm_extern.h>
+#include <vm/uma.h> 
+
+#include <machine/cpufunc.h>
+#include <machine/hypervisorvar.h>
+#include <machine/smp.h>
+#include <machine/mmu.h>
+#include <machine/tte.h>
+#include <machine/vmparam.h>
+#include <machine/tlb.h>
+#include <machine/tte_hash.h>
+
+#define HASH_SIZE        (1 << HASH_ENTRY_SHIFT)
+#define HASH_MASK(th)    ((th->th_size<<(PAGE_SHIFT-THE_SHIFT))-1)
+#define NULL_TAG         0
+#define MAGIC_VALUE      0xcafebabe
+
+struct tte_hash_entry;
+struct of_field;
+
+#define MAX_FRAGMENT_ENTRIES ((PAGE_SIZE / sizeof(struct tte_hash_entry)) - 1)
+
+typedef struct tte_hash_field_ {
+	uint64_t tag;
+	uint64_t data;
+} tte_hash_field, *tte_hash_field_t;
+
+struct of_field {
+	int16_t          count;
+	uint8_t          lock;
+	uint8_t          pad;
+	uint32_t         flags;
+	struct tte_hash_entry *next;
+};
+
+typedef struct tte_hash_entry {
+	tte_hash_field the_fields[HASH_ENTRIES];
+	struct of_field of;
+} *tte_hash_entry_t;
+
+struct fragment_header {
+	struct tte_hash_fragment *fh_next;
+	uint8_t fh_count;
+	uint8_t fh_free_head;
+	uint8_t pad[sizeof(struct tte_hash_entry) - 10];
+};
+
+CTASSERT(sizeof(struct fragment_header) == sizeof(struct tte_hash_entry));
+
+struct tte_hash {
+	uint16_t th_size;               /* size in pages */
+	uint16_t th_context;            /* TLB context   */
+	uint32_t th_entries;            /* # pages held  */
+	tte_hash_entry_t th_hashtable;   /* hash of TTEs  */
+	struct tte_hash_fragment *th_fhhead;
+	struct tte_hash_fragment *th_fhtail;
+};
+
+struct tte_hash_fragment {
+	struct fragment_header thf_head;
+	struct tte_hash_entry  thf_entries[MAX_FRAGMENT_ENTRIES];
+};
+
+CTASSERT(sizeof(struct tte_hash_fragment) == PAGE_SIZE);
+
+
+static struct tte_hash kernel_tte_hash;
+/*
+ * Data for the tte_hash allocation mechanism
+ */
+static uma_zone_t thzone;
+static struct vm_object thzone_obj;
+static int tte_hash_count = 0, tte_hash_max = 0;
+
+extern uint64_t hash_bucket_lock(tte_hash_field_t fields);
+extern void hash_bucket_unlock(tte_hash_field_t fields, uint64_t s);
+
+static tte_hash_t
+get_tte_hash(void)
+{
+	tte_hash_t th;
+
+	th = uma_zalloc(thzone, M_NOWAIT);
+
+	KASSERT(th != NULL, ("tte_hash allocation failed"));
+	tte_hash_count++;
+	return th;
+
+}
+
+static __inline void
+free_tte_hash(tte_hash_t th)
+{
+	tte_hash_count--;
+	uma_zfree(thzone, th);
+}
+
+void 
+tte_hash_init(void)
+{
+	thzone = uma_zcreate("TTE_HASH", sizeof(struct tte_hash), NULL, NULL, 
+	    NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM | UMA_ZONE_NOFREE);
+	tte_hash_max = maxproc;
+	uma_zone_set_obj(thzone, &thzone_obj, tte_hash_max);
+}
+
+tte_hash_t
+tte_hash_kernel_create(vm_offset_t va, uint64_t size, vm_paddr_t fragment_page)
+{
+	tte_hash_t th;
+		
+	th = &kernel_tte_hash;
+	th->th_size = (size >> PAGE_SHIFT);
+	th->th_entries = 0;
+	th->th_context = 0;
+	th->th_hashtable = (tte_hash_entry_t)va;
+	th->th_fhtail = th->th_fhhead = (void *)TLB_PHYS_TO_DIRECT(fragment_page);
+
+	return th;
+}
+
+static inline vm_page_t
+alloc_zeroed_page(void)
+{
+	vm_page_t m;
+	static int color;
+
+	m = NULL;
+
+	while (m == NULL) {
+		m = vm_page_alloc(NULL, color++,
+		    VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED |
+		    VM_ALLOC_ZERO);
+
+		if (m == NULL) 
+			VM_WAIT;
+	}
+
+	if ((m->flags & PG_ZERO) == 0)
+		pmap_zero_page(m);
+
+	return (m);
+}
+
+tte_hash_t
+tte_hash_create(uint64_t context, uint64_t *scratchval)
+{
+	tte_hash_t th;
+	vm_page_t m, tm;
+	int i;
+	
+	th = get_tte_hash();
+	
+	th->th_size = HASH_SIZE;
+	th->th_entries = 0;
+	th->th_context = (uint16_t)context;
+	m = NULL;
+
+	while (m == NULL) {
+		m = vm_page_alloc_contig(HASH_SIZE, phys_avail[0], 
+					 phys_avail[1], PAGE_SIZE, (1UL<<34));
+		if (m == NULL) {
+			printf("vm_page_alloc_contig failed - waiting to retry\n");
+			VM_WAIT;
+		}
+	}
+	for (i = 0, tm = m; i < HASH_SIZE; i++, tm++) 
+		if ((tm->flags & PG_ZERO) == 0)
+			pmap_zero_page(tm);
+
+	th->th_hashtable = (void *)TLB_PHYS_TO_DIRECT(VM_PAGE_TO_PHYS(m));
+	m = alloc_zeroed_page();
+
+
+	th->th_fhtail = th->th_fhhead = (void *)TLB_PHYS_TO_DIRECT(VM_PAGE_TO_PHYS(m));
+	KASSERT(th->th_fhtail != NULL, ("th->th_fhtail == NULL"));
+
+	*scratchval = (uint64_t)((vm_offset_t)th->th_hashtable) | ((vm_offset_t)th->th_size);
+
+	return (th);
+}
+
+void
+tte_hash_destroy(tte_hash_t th)
+{
+	panic("FIXME");
+
+	free_tte_hash(th);
+}
+
+void
+tte_hash_reset(tte_hash_t th)
+{
+	struct tte_hash_fragment *fh;
+	vm_page_t m;
+
+	for (fh = th->th_fhhead->thf_head.fh_next; fh != NULL; fh = fh->thf_head.fh_next) {
+		m = PHYS_TO_VM_PAGE((vm_paddr_t)TLB_DIRECT_TO_PHYS((vm_offset_t)fh));
+		m->wire_count--;
+		vm_page_free(m);
+	}
+	fh = th->th_fhtail = th->th_fhhead;
+	hwblkclr(th->th_fhhead, PAGE_SIZE); 
+
+#ifdef UNMANAGED_PAGES_ARE_TRACKED
+	if (th->th_entries != 0)
+		panic("%d remaining entries", th->th_entries);
+#else
+	hwblkclr(th->th_hashtable, th->th_size*PAGE_SIZE); 	
+#endif
+}
+
+static __inline void
+tte_hash_set_field(tte_hash_field_t field, uint64_t tag, tte_t tte)
+{
+	field->tag = tag;
+	field->data = tte | (field->data & VTD_LOCK);
+}
+
+static __inline tte_hash_entry_t 
+find_entry(tte_hash_t th, vm_offset_t va, int page_shift)
+{
+	uint64_t hash_index;
+
+	hash_index = (va >> page_shift) & HASH_MASK(th);
+	return (&th->th_hashtable[hash_index]);
+}
+
+static __inline tte_hash_entry_t 
+tte_hash_lookup_last_entry(tte_hash_entry_t entry)
+{
+
+	while (entry->of.next) 
+		entry = entry->of.next;
+
+	return (entry);
+}
+
+static tte_hash_entry_t 
+tte_hash_allocate_fragment_entry(tte_hash_t th)
+{
+	struct tte_hash_fragment *fh;
+	tte_hash_entry_t newentry;
+	vm_page_t m;
+	
+	fh = th->th_fhtail;
+	if (fh->thf_head.fh_free_head == MAX_FRAGMENT_ENTRIES) {
+		m = alloc_zeroed_page();
+
+		fh->thf_head.fh_next = (void *)TLB_PHYS_TO_DIRECT(VM_PAGE_TO_PHYS(m));
+		fh = th->th_fhtail = (void *)TLB_PHYS_TO_DIRECT(VM_PAGE_TO_PHYS(m));
+		fh->thf_head.fh_free_head = 1;
+#ifdef NOISY_DEBUG
+		printf("new fh=%p \n", fh);
+#endif
+	} 
+	newentry = &fh->thf_entries[fh->thf_head.fh_free_head];
+
+	fh->thf_head.fh_free_head++;
+	fh->thf_head.fh_count++; 
+
+	return (newentry);
+}
+
+/*
+ * if a match for va is found the tte value is returned 
+ * and if field is non-null field will point to that entry
+ * 
+ * 
+ */
+static __inline tte_t 
+tte_hash_lookup_inline(tte_hash_entry_t entry, tte_t tte_tag, boolean_t insert)
+{
+	int i;
+	tte_t tte_data;
+	tte_hash_field_t fields;
+
+	tte_data = 0;
+	do { 
+		fields = entry->the_fields;
+		for (i = 0; i < entry->of.count; i++) {
+			if (fields[i].tag == tte_tag) {
+				tte_data = (fields[i].data & ~VTD_LOCK);
+				PCPU_SET(lookup_field, (u_long)&fields[i]);
+				goto done;
+			}
+		}
+#ifdef DEBUG
+	if (entry->of.next && entry->of.flags != MAGIC_VALUE)
+		panic("overflow pointer not null without flags set entry= %p next=%p flags=0x%x count=%d", 
+		      entry, entry->of.next, entry->of.flags, entry->of.count);
+#endif
+		entry = entry->of.next;
+	} while (entry);
+
+done:
+	return (tte_data);
+}
+
+
+static __inline void
+tte_hash_lookup_last_inline(tte_hash_entry_t entry)
+{
+
+	tte_hash_field_t fields;
+
+	fields = entry->the_fields;
+
+	while (entry->of.next && (entry->of.next->of.count > 1))
+		entry = entry->of.next;
+
+	if (entry->of.next && entry->of.next->of.count == 1) {
+		PCPU_SET(last_field, (u_long)&entry->of.next->the_fields[0]);
+		entry->of.next = NULL;
+		entry->of.flags = 0;
+	} else {
+#ifdef DEBUG
+		if (entry->of.count == 0)
+			panic("count zero");
+#endif
+		PCPU_SET(last_field, (u_long)&entry->the_fields[--entry->of.count]);
+	}
+}
+
+tte_t
+tte_hash_clear_bits(tte_hash_t th, vm_offset_t va, uint64_t flags)
+{
+	uint64_t s;
+	tte_hash_entry_t entry;
+	tte_t otte_data, tte_tag;
+
+	/* XXX - only handle 8K pages for now */
+	entry = find_entry(th, va, PAGE_SHIFT);
+
+	tte_tag = (((uint64_t)th->th_context << TTARGET_CTX_SHIFT)|(va >> TTARGET_VA_SHIFT));
+	
+	s = hash_bucket_lock(entry->the_fields);
+	if((otte_data = tte_hash_lookup_inline(entry, tte_tag, FALSE)) != 0)
+		tte_hash_set_field((tte_hash_field_t)PCPU_GET(lookup_field), 
+				   ((tte_hash_field_t)PCPU_GET(lookup_field))->tag, 
+				   ((tte_hash_field_t)PCPU_GET(lookup_field))->data & ~flags);
+
+	hash_bucket_unlock(entry->the_fields, s);
+
+	return (otte_data);
+}
+
+tte_t
+tte_hash_delete(tte_hash_t th, vm_offset_t va)
+{
+	uint64_t s;
+	tte_hash_entry_t entry;
+	tte_t tte_data, tte_tag;
+
+	/* XXX - only handle 8K pages for now */
+	entry = find_entry(th, va, PAGE_SHIFT);
+
+	tte_tag = (((uint64_t)th->th_context << TTARGET_CTX_SHIFT)|(va >> TTARGET_VA_SHIFT));
+
+	s  = hash_bucket_lock(entry->the_fields);
+	
+	if ((tte_data = tte_hash_lookup_inline(entry, tte_tag, FALSE)) == 0) 
+		goto done;
+
+	tte_hash_lookup_last_inline(entry);
+
+#ifdef DEBUG
+	if (((tte_hash_field_t)PCPU_GET(last_field))->tag == 0) {
+		hash_bucket_unlock(entry->the_fields, s);
+		panic("lookup_last failed for va=0x%lx\n", va);
+	}
+#endif
+	/* move last field's values in to the field we are deleting */
+	if (PCPU_GET(lookup_field) != PCPU_GET(last_field)) 
+		tte_hash_set_field((tte_hash_field_t)PCPU_GET(lookup_field), 
+				   ((tte_hash_field_t)PCPU_GET(last_field))->tag, 
+				   ((tte_hash_field_t)PCPU_GET(last_field))->data);
+	
+	tte_hash_set_field((tte_hash_field_t)PCPU_GET(last_field), 0, 0);
+done:	
+	hash_bucket_unlock(entry->the_fields, s);
+	if (tte_data) 
+		th->th_entries--;
+
+	return (tte_data);
+}
+
+void
+tte_hash_insert(tte_hash_t th, vm_offset_t va, tte_t tte_data)
+{
+
+	tte_hash_entry_t entry, lentry, newentry;
+	tte_t tte_tag;
+	uint64_t s;
+
+#ifdef DEBUG
+	if (tte_hash_lookup(th, va) != 0) 
+		panic("mapping for va=0x%lx already exists", va);
+#endif
+	entry = find_entry(th, va, PAGE_SHIFT);
+	tte_tag = (((uint64_t)th->th_context << TTARGET_CTX_SHIFT)|(va >> TTARGET_VA_SHIFT));
+
+	s = hash_bucket_lock(entry->the_fields);
+	lentry = tte_hash_lookup_last_entry(entry);
+
+	if (lentry->of.count == HASH_ENTRIES) {
+		hash_bucket_unlock(entry->the_fields, s);
+		newentry = tte_hash_allocate_fragment_entry(th); 
+		s = hash_bucket_lock(entry->the_fields);
+		lentry->of.flags = MAGIC_VALUE;
+		lentry->of.next = newentry;
+		lentry = newentry;
+	} 
+	tte_hash_set_field(&lentry->the_fields[lentry->of.count++], 
+			   tte_tag, tte_data);
+	hash_bucket_unlock(entry->the_fields, s);
+
+#ifdef DEBUG
+	if (tte_hash_lookup(th, va) == 0) 
+		panic("insert for va=0x%lx failed", va);
+#endif
+	th->th_entries++;
+}
+
+/* 
+ * If leave_locked is true the tte's data field will be returned to
+ * the caller with the hash bucket left locked
+ */
+tte_t 
+tte_hash_lookup(tte_hash_t th, vm_offset_t va)
+{
+	uint64_t s;
+	tte_hash_entry_t entry;
+	tte_t tte_data, tte_tag;
+
+	/* XXX - only handle 8K pages for now */
+	entry = find_entry(th, va, PAGE_SHIFT);
+
+	tte_tag = (((uint64_t)th->th_context << TTARGET_CTX_SHIFT)|(va >> TTARGET_VA_SHIFT));
+
+	s = hash_bucket_lock(entry->the_fields);
+	tte_data = tte_hash_lookup_inline(entry, tte_tag, FALSE);
+	hash_bucket_unlock(entry->the_fields, s);
+	
+	return (tte_data);
+}
+
+uint64_t
+tte_hash_set_scratchpad_kernel(tte_hash_t th)
+{
+	
+	uint64_t hash_scratch;
+	/* This breaks if a hash table grows above 32MB
+	 */
+	hash_scratch = ((vm_offset_t)th->th_hashtable) | ((vm_offset_t)th->th_size);
+	set_hash_kernel_scratchpad(hash_scratch);
+	
+	return (hash_scratch);
+}
+
+uint64_t
+tte_hash_set_scratchpad_user(tte_hash_t th, uint64_t context)
+{
+
+	uint64_t hash_scratch;
+	/* This breaks if a hash table grows above 32MB
+	 */
+	th->th_context = (uint16_t)context;
+	hash_scratch = ((vm_offset_t)th->th_hashtable) | ((vm_offset_t)th->th_size);
+	set_hash_user_scratchpad(hash_scratch);
+	
+	return (hash_scratch);
+}
+
+tte_t
+tte_hash_update(tte_hash_t th, vm_offset_t va, tte_t tte_data)
+{
+
+	uint64_t s;
+	tte_hash_entry_t entry;
+	tte_t otte_data, tte_tag;
+
+	entry = find_entry(th, va, PAGE_SHIFT);
+
+	tte_tag = (((uint64_t)th->th_context << TTARGET_CTX_SHIFT)|(va >> TTARGET_VA_SHIFT));
+
+	s = hash_bucket_lock(entry->the_fields);
+	otte_data = tte_hash_lookup_inline(entry, tte_tag, TRUE);
+
+	if (otte_data == 0) {
+		hash_bucket_unlock(entry->the_fields, s);
+		tte_hash_insert(th, va, tte_data);
+	} else {
+		tte_hash_set_field((tte_hash_field_t)PCPU_GET(lookup_field), 
+				   tte_tag, tte_data);
+		hash_bucket_unlock(entry->the_fields, s);
+	}
+
+	return (otte_data);
+}
+
diff --git a/sys/sun4v/sun4v/uio_machdep.c b/sys/sun4v/sun4v/uio_machdep.c
new file mode 100644
index 000000000000..f92a64757494
--- /dev/null
+++ b/sys/sun4v/sun4v/uio_machdep.c
@@ -0,0 +1,133 @@
+/*-
+ * Copyright (c) 2004 Alan L. Cox <alc@cs.rice.edu>
+ * Copyright (c) 1982, 1986, 1991, 1993
+ *	The Regents of the University of California.  All rights reserved.
+ * (c) UNIX System Laboratories, Inc.
+ * All or some portions of this file are derived from material licensed
+ * to the University of California by American Telephone and Telegraph
+ * Co. or Unix System Laboratories, Inc. and are reproduced herein with
+ * the permission of UNIX System Laboratories, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 4. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	@(#)kern_subr.c	8.3 (Berkeley) 1/21/94
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/kernel.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
+#include <sys/proc.h>
+#include <sys/sf_buf.h>
+#include <sys/systm.h>
+#include <sys/uio.h>
+
+#include <vm/vm.h>
+#include <vm/vm_page.h>
+#include <vm/vm_param.h>
+
+#include <machine/tlb.h>
+
+/*
+ * Implement uiomove(9) from physical memory using a combination
+ * of the direct mapping and sf_bufs to reduce the creation and
+ * destruction of ephemeral mappings.  
+ */
+int
+uiomove_fromphys(vm_page_t ma[], vm_offset_t offset, int n, struct uio *uio)
+{
+	struct sf_buf *sf;
+	struct thread *td = curthread;
+	struct iovec *iov;
+	void *cp;
+	vm_offset_t page_offset;
+	vm_paddr_t pa;
+	vm_page_t m;
+	size_t cnt;
+	int error = 0;
+	int save = 0;
+
+	KASSERT(uio->uio_rw == UIO_READ || uio->uio_rw == UIO_WRITE,
+	    ("uiomove_fromphys: mode"));
+	KASSERT(uio->uio_segflg != UIO_USERSPACE || uio->uio_td == curthread,
+	    ("uiomove_fromphys proc"));
+	save = td->td_pflags & TDP_DEADLKTREAT;
+	td->td_pflags |= TDP_DEADLKTREAT;
+	while (n > 0 && uio->uio_resid) {
+		iov = uio->uio_iov;
+		cnt = iov->iov_len;
+		if (cnt == 0) {
+			uio->uio_iov++;
+			uio->uio_iovcnt--;
+			continue;
+		}
+		if (cnt > n)
+			cnt = n;
+		page_offset = offset & PAGE_MASK;
+		cnt = ulmin(cnt, PAGE_SIZE - page_offset);
+		m = ma[offset >> PAGE_SHIFT];
+		pa = VM_PAGE_TO_PHYS(m);
+		sf = NULL;
+		cp = (char *)TLB_PHYS_TO_DIRECT(pa) + page_offset;
+		switch (uio->uio_segflg) {
+		case UIO_USERSPACE:
+			if (ticks - PCPU_GET(switchticks) >= hogticks)
+				uio_yield();
+			if (uio->uio_rw == UIO_READ)
+				error = copyout(cp, iov->iov_base, cnt);
+			else
+				error = copyin(iov->iov_base, cp, cnt);
+			if (error) {
+				if (sf != NULL)
+					sf_buf_free(sf);
+				goto out;
+			}
+			break;
+		case UIO_SYSSPACE:
+			if (uio->uio_rw == UIO_READ)
+				bcopy(cp, iov->iov_base, cnt);
+			else
+				bcopy(iov->iov_base, cp, cnt);
+			break;
+		case UIO_NOCOPY:
+			break;
+		}
+		if (sf != NULL)
+			sf_buf_free(sf);
+		iov->iov_base = (char *)iov->iov_base + cnt;
+		iov->iov_len -= cnt;
+		uio->uio_resid -= cnt;
+		uio->uio_offset += cnt;
+		offset += cnt;
+		n -= cnt;
+	}
+out:
+	if (save == 0)
+		td->td_pflags &= ~TDP_DEADLKTREAT;
+	return (error);
+}
diff --git a/sys/sun4v/sun4v/vm_machdep.c b/sys/sun4v/sun4v/vm_machdep.c
new file mode 100644
index 000000000000..46c0996fad90
--- /dev/null
+++ b/sys/sun4v/sun4v/vm_machdep.c
@@ -0,0 +1,433 @@
+/*-
+ * Copyright (c) 1982, 1986 The Regents of the University of California.
+ * Copyright (c) 1989, 1990 William Jolitz
+ * Copyright (c) 1994 John Dyson
+ * Copyright (c) 2001 Jake Burkholder.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to Berkeley by
+ * the Systems Programming Group of the University of Utah Computer
+ * Science Department, and William Jolitz.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ *    must display the following acknowledgement:
+ *	This product includes software developed by the University of
+ *	California, Berkeley and its contributors.
+ * 4. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	from: @(#)vm_machdep.c	7.3 (Berkeley) 5/13/91
+ *	Utah $Hdr: vm_machdep.c 1.16.1.1 89/06/23$
+ * 	from: FreeBSD: src/sys/i386/i386/vm_machdep.c,v 1.167 2001/07/12
+ * $FreeBSD$
+ */
+
+#include "opt_pmap.h"
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/malloc.h>
+#include <sys/proc.h>
+#include <sys/bio.h>
+#include <sys/buf.h>
+#include <sys/kernel.h>
+#include <sys/linker_set.h>
+#include <sys/mbuf.h>
+#include <sys/mutex.h>
+#include <sys/sf_buf.h>
+#include <sys/sysctl.h>
+#include <sys/unistd.h>
+#include <sys/vmmeter.h>
+
+#include <dev/ofw/openfirm.h>
+
+#include <vm/vm.h>
+#include <vm/vm_extern.h>
+#include <vm/pmap.h>
+#include <vm/vm_kern.h>
+#include <vm/vm_map.h>
+#include <vm/vm_page.h>
+#include <vm/vm_pageout.h>
+#include <vm/vm_param.h>
+#include <vm/uma.h>
+#include <vm/uma_int.h>
+
+#include <machine/cache.h>
+#include <machine/bus.h>
+#include <machine/cpu.h>
+#include <machine/fp.h>
+#include <machine/fsr.h>
+#include <machine/frame.h>
+#include <machine/md_var.h>
+#include <machine/ofw_machdep.h>
+#include <machine/ofw_mem.h>
+#include <machine/pcb.h>
+#include <machine/tlb.h>
+#include <machine/tstate.h>
+#include <machine/wstate.h>
+#include <machine/asm.h>
+
+void
+cpu_exit(struct thread *td)
+{
+	struct proc *p;
+
+	p = td->td_proc;
+	p->p_md.md_sigtramp = NULL;
+	if (p->p_md.md_utrap != NULL) {
+		utrap_free(p->p_md.md_utrap);
+		p->p_md.md_utrap = NULL;
+	}
+}
+
+void
+cpu_thread_exit(struct thread *td)
+{
+}
+
+void
+cpu_thread_clean(struct thread *td)
+{
+}
+
+void
+cpu_thread_setup(struct thread *td)
+{
+	struct pcb *pcb;
+
+	pcb = (struct pcb *)((td->td_kstack + td->td_kstack_pages * PAGE_SIZE -
+	    sizeof(struct pcb)) & ~0x3fUL);
+	pcb->pcb_kstack = (uint64_t)(((char *)pcb) - (CCFSZ + SPOFF));
+	pcb->pcb_nsaved = 0;
+	td->td_frame = (struct trapframe *)pcb - 1;
+
+	pcb = (struct pcb *)TLB_PHYS_TO_DIRECT(vtophys((vm_offset_t)pcb));
+	td->td_pcb = pcb;
+	
+}
+
+void
+cpu_thread_swapin(struct thread *td)
+{
+}
+
+void
+cpu_thread_swapout(struct thread *td)
+{
+}
+
+void
+cpu_set_upcall(struct thread *td, struct thread *td0)
+{
+	struct trapframe *tf;
+	struct frame *fr;
+	struct pcb *pcb;
+
+	bcopy(td0->td_frame, td->td_frame, sizeof(struct trapframe));
+
+	pcb = td->td_pcb;
+
+	tf = td->td_frame;
+	fr = (struct frame *)tf - 1;
+	fr->fr_local[0] = (u_long)fork_return;
+	fr->fr_local[1] = (u_long)td;
+	fr->fr_local[2] = (u_long)tf;
+	pcb->pcb_pc = (u_long)fork_trampoline - 8;
+	pcb->pcb_sp = (u_long)fr - SPOFF;
+
+	/* Setup to release sched_lock in fork_exit(). */
+	td->td_md.md_spinlock_count = 1;
+	td->td_md.md_saved_pil = 0;
+}
+
+void
+cpu_set_upcall_kse(struct thread *td, void (*entry)(void *), void *arg,
+	stack_t *stack)
+{
+	struct trapframe *tf;
+	uint64_t sp;
+
+	if (td == curthread)
+		flushw();
+	tf = td->td_frame;
+	sp = (uint64_t)stack->ss_sp + stack->ss_size;
+	tf->tf_out[0] = (uint64_t)arg;
+	tf->tf_out[6] = sp - SPOFF - sizeof(struct frame);
+	tf->tf_tpc = (uint64_t)entry;
+	tf->tf_tnpc = tf->tf_tpc + 4;
+
+	td->td_retval[0] = tf->tf_out[0];
+	td->td_retval[1] = tf->tf_out[1];
+}
+
+int
+cpu_set_user_tls(struct thread *td, void *tls_base)
+{
+
+	if (td == curthread)
+		flushw();
+	td->td_frame->tf_global[7] = (uint64_t) tls_base;
+	return (0);
+}
+
+/*
+ * Finish a fork operation, with process p2 nearly set up.
+ * Copy and update the pcb, set up the stack so that the child
+ * ready to run and return to user mode.
+ */
+void
+cpu_fork(struct thread *td1, struct proc *p2, struct thread *td2, int flags)
+{
+	struct trapframe *tf;
+	struct frame *fp;
+	struct pcb *pcb1;
+	struct pcb *pcb2, *pcb2orig;
+	vm_offset_t sp;
+	int error;
+	int i;
+
+	KASSERT(td1 == curthread || td1 == &thread0,
+	    ("cpu_fork: p1 not curproc and not proc0"));
+
+	if ((flags & RFPROC) == 0)
+		return;
+
+	p2->p_md.md_sigtramp = td1->td_proc->p_md.md_sigtramp;
+	p2->p_md.md_utrap = utrap_hold(td1->td_proc->p_md.md_utrap);
+
+	/* The pcb must be aligned on a 64-byte boundary. */
+	pcb1 = td1->td_pcb;
+
+	pcb2orig = (struct pcb *)((td2->td_kstack + td2->td_kstack_pages *
+	    PAGE_SIZE - sizeof(struct pcb)) & ~0x3fUL);
+	pcb2 = (struct pcb *)TLB_PHYS_TO_DIRECT(vtophys((vm_offset_t)pcb2orig));
+
+	td2->td_pcb = pcb2;
+
+	/*
+	 * Ensure that p1's pcb is up to date.
+	 */
+	critical_enter();
+	if ((td1->td_frame->tf_fprs & FPRS_FEF) != 0)
+		savefpctx(pcb1->pcb_ufp);
+	critical_exit();
+	/* Make sure the copied windows are spilled. */
+	flushw();
+	/* Copy the pcb (this will copy the windows saved in the pcb, too). */
+	bcopy(pcb1, pcb2, sizeof(*pcb1));
+
+	/*
+	 * If we're creating a new user process and we're sharing the address
+	 * space, the parent's top most frame must be saved in the pcb.  The
+	 * child will pop the frame when it returns to user mode, and may
+	 * overwrite it with its own data causing much suffering for the
+	 * parent.  We check if its already in the pcb, and if not copy it
+	 * in.  Its unlikely that the copyin will fail, but if so there's not
+	 * much we can do.  The parent will likely crash soon anyway in that
+	 * case.
+	 */
+	if ((flags & RFMEM) != 0 && td1 != &thread0) {
+		sp = td1->td_frame->tf_sp;
+		for (i = 0; i < pcb1->pcb_nsaved; i++) {
+			if (pcb1->pcb_rwsp[i] == sp)
+				break;
+		}
+		if (i == pcb1->pcb_nsaved) {
+			error = copyin((caddr_t)sp + SPOFF, &pcb1->pcb_rw[i],
+			    sizeof(struct rwindow));
+			if (error == 0) {
+				pcb1->pcb_rwsp[i] = sp;
+				pcb1->pcb_nsaved++;
+			}
+		}
+	}
+
+	/*
+	 * Create a new fresh stack for the new process.
+	 * Copy the trap frame for the return to user mode as if from a
+	 * syscall.  This copies most of the user mode register values.
+	 */
+	tf = (struct trapframe *)pcb2orig - 1;
+	bcopy(td1->td_frame, tf, sizeof(*tf));
+
+	tf->tf_out[0] = 0;			/* Child returns zero */
+	tf->tf_out[1] = 0;
+	tf->tf_tstate &= ~TSTATE_XCC_C;		/* success */
+	tf->tf_fprs = 0;
+	tf->tf_wstate = WSTATE_U64;
+
+
+	td2->td_frame = tf;
+	fp = (struct frame *)tf - 1;
+	fp->fr_local[0] = (u_long)fork_return;
+	fp->fr_local[1] = (u_long)td2;
+	fp->fr_local[2] = (u_long)tf;
+	/* Terminate stack traces at this frame. */
+	fp->fr_pc = fp->fr_fp = 0;
+	pcb2->pcb_sp = (u_long)fp - SPOFF;
+	pcb2->pcb_pc = (u_long)fork_trampoline - 8;
+	pcb2->pcb_kstack = (uint64_t)(((char *)pcb2orig) - (CCFSZ + SPOFF));
+
+	/* Setup to release sched_lock in fork_exit(). */
+	td2->td_md.md_spinlock_count = 1;
+	td2->td_md.md_saved_pil = 0;
+
+	/*
+	 * Now, cpu_switch() can schedule the new process.
+	 */
+}
+
+void
+cpu_reset(void)
+{
+	static char bspec[64] = "";
+	phandle_t chosen;
+#ifdef notyet
+	static struct {
+		cell_t	name;
+		cell_t	nargs;
+		cell_t	nreturns;
+		cell_t	bootspec;
+	} args = {
+		(cell_t)"boot",
+		1,
+		0,
+		(cell_t)bspec
+	};
+#endif
+	if ((chosen = OF_finddevice("/chosen")) != 0) {
+		if (OF_getprop(chosen, "bootpath", bspec, sizeof(bspec)) == -1)
+			bspec[0] = '\0';
+		bspec[sizeof(bspec) - 1] = '\0';
+	}
+#ifdef notyet
+	/* XXX SUN4V_FIXME */
+	openfirmware_exit(&args);
+#endif
+}
+
+/*
+ * Intercept the return address from a freshly forked process that has NOT
+ * been scheduled yet.
+ *
+ * This is needed to make kernel threads stay in kernel mode.
+ */
+void
+cpu_set_fork_handler(struct thread *td, void (*func)(void *), void *arg)
+{
+	struct frame *fp;
+	struct pcb *pcb;
+
+	pcb = td->td_pcb;
+	fp = (struct frame *)(pcb->pcb_sp + SPOFF);
+	fp->fr_local[0] = (u_long)func;
+	fp->fr_local[1] = (u_long)arg;
+}
+
+int
+is_physical_memory(vm_paddr_t addr)
+{
+	struct ofw_mem_region *mr;
+
+	for (mr = sparc64_memreg; mr < sparc64_memreg + sparc64_nmemreg; mr++)
+		if (addr >= mr->mr_start && addr < mr->mr_start + mr->mr_size)
+			return (1);
+	return (0);
+}
+
+/*
+ * Get an sf_buf from the freelist. Will block if none are available.
+ */
+struct sf_buf *
+sf_buf_alloc(struct vm_page *m, int flags)
+{
+	return ((struct sf_buf *)m);
+}
+
+/*
+ * Release resources back to the system.
+ */
+void
+sf_buf_free(struct sf_buf *sf)
+{
+}
+
+void
+swi_vm(void *v)
+{
+
+	/*
+	 * Nothing to do here yet - busdma bounce buffers are not yet
+	 * implemented.
+	 */
+}
+
+void *
+uma_small_alloc(uma_zone_t zone, int bytes, u_int8_t *flags, int wait)
+{
+	static vm_pindex_t color;
+	vm_paddr_t pa;
+	vm_page_t m;
+	int pflags;
+	void *va;
+
+
+	*flags = UMA_SLAB_PRIV;
+
+	if ((wait & (M_NOWAIT|M_USE_RESERVE)) == M_NOWAIT)
+		pflags = VM_ALLOC_INTERRUPT;
+	else
+		pflags = VM_ALLOC_SYSTEM;
+
+	if (wait & M_ZERO)
+		pflags |= VM_ALLOC_ZERO;
+
+	for (;;) {
+		m = vm_page_alloc(NULL, color++, pflags | VM_ALLOC_NOOBJ);
+		if (m == NULL) {
+			if (wait & M_NOWAIT)
+				return (NULL);
+			else
+				VM_WAIT;
+		} else
+			break;
+	}
+
+	pa = VM_PAGE_TO_PHYS(m);
+	va = (void *)TLB_PHYS_TO_DIRECT(pa);
+	if ((wait & M_ZERO) && ((m->flags & PG_ZERO) == 0))
+		hwblkclr((void *)TLB_PHYS_TO_DIRECT(pa), PAGE_SIZE);
+	return (va);
+}
+
+void
+uma_small_free(void *mem, int size, u_int8_t flags)
+{
+	vm_page_t m;
+
+	m = PHYS_TO_VM_PAGE(TLB_DIRECT_TO_PHYS((vm_offset_t)mem));
+	vm_page_lock_queues();
+	vm_page_free(m);
+	vm_page_unlock_queues();
+}
diff --git a/sys/sun4v/sun4v/vnex.c b/sys/sun4v/sun4v/vnex.c
new file mode 100644
index 000000000000..11aa18240e78
--- /dev/null
+++ b/sys/sun4v/sun4v/vnex.c
@@ -0,0 +1,367 @@
+/*-
+ * Copyright (c) 2006 by Marius Strobl <marius@FreeBSD.org>.
+ * Copyright (c) 2006 Kip Macy <kmacy@FreeBSD.org>.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+__FBSDID("$FreeBSD$");
+
+#include <sys/cdefs.h>
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/bus.h>
+#include <sys/pcpu.h>
+#include <sys/kernel.h>
+#include <sys/malloc.h>
+#include <sys/module.h>
+
+#include <dev/ofw/ofw_bus.h>
+#include <dev/ofw/ofw_bus_subr.h>
+#include <dev/ofw/openfirm.h>
+
+#include <machine/bus.h>
+#include <machine/bus_common.h>
+#include <machine/intr_machdep.h>
+#include <machine/nexusvar.h>
+#include <machine/resource.h>
+#include <machine/ver.h>
+
+#include <machine/mdesc_bus.h>
+#include <machine/cddl/mdesc.h>
+#include <machine/cddl/mdesc_impl.h>
+
+#include <sys/rman.h>
+
+
+#define	SUN4V_REG_SPEC2CFG_HDL(x)	((x >> 32) & ~(0xfull << 28))
+
+static device_probe_t vnex_probe;
+static device_attach_t vnex_attach;
+static bus_print_child_t vnex_print_child;
+static bus_add_child_t vnex_add_child;
+static bus_probe_nomatch_t vnex_probe_nomatch;
+static bus_setup_intr_t vnex_setup_intr;
+static bus_teardown_intr_t vnex_teardown_intr;
+static bus_get_resource_list_t vnex_get_resource_list;
+static mdesc_bus_get_devinfo_t vnex_get_devinfo;
+
+static struct vnex_devinfo * vnex_setup_dinfo(device_t, mde_cookie_t node);
+static void vnex_destroy_dinfo(struct vnex_devinfo *);
+static int vnex_print_res(struct vnex_devinfo *);
+
+struct vnex_devinfo {
+	struct mdesc_bus_devinfo	vndi_mbdinfo;
+	struct resource_list	vndi_rl;
+
+	/* Some common properties. */
+	struct		nexus_regs *vndi_reg;
+	int		vndi_nreg;
+};
+
+struct vnex_softc {
+	struct rman	sc_intr_rman;
+	struct rman	sc_mem_rman;
+};
+
+static device_method_t vnex_methods[] = {
+	/* Device interface */
+	DEVMETHOD(device_probe,		vnex_probe),
+	DEVMETHOD(device_attach,	vnex_attach),
+	DEVMETHOD(device_detach,	bus_generic_detach),
+	DEVMETHOD(device_shutdown,	bus_generic_shutdown),
+	DEVMETHOD(device_suspend,	bus_generic_suspend),
+	DEVMETHOD(device_resume,	bus_generic_resume),
+
+	/* Bus interface */
+	DEVMETHOD(bus_print_child,	vnex_print_child),
+	DEVMETHOD(bus_probe_nomatch,	vnex_probe_nomatch),
+	DEVMETHOD(bus_read_ivar,	bus_generic_read_ivar),
+	DEVMETHOD(bus_write_ivar,	bus_generic_write_ivar),
+	DEVMETHOD(bus_add_child,	vnex_add_child),
+	DEVMETHOD(bus_alloc_resource,	bus_generic_alloc_resource),
+	DEVMETHOD(bus_activate_resource,	bus_generic_activate_resource),
+	DEVMETHOD(bus_deactivate_resource,	bus_generic_deactivate_resource),
+	DEVMETHOD(bus_release_resource,	bus_generic_release_resource),
+	DEVMETHOD(bus_setup_intr,	vnex_setup_intr),
+	DEVMETHOD(bus_teardown_intr,	vnex_teardown_intr),
+	DEVMETHOD(bus_get_resource,	bus_generic_rl_get_resource),
+	DEVMETHOD(bus_get_resource_list, vnex_get_resource_list),
+
+	/* ofw_bus interface */
+	/* mdesc_bus interface */
+	DEVMETHOD(mdesc_bus_get_devinfo, vnex_get_devinfo),
+	DEVMETHOD(mdesc_bus_get_compat,	mdesc_bus_gen_get_compat),
+	DEVMETHOD(mdesc_bus_get_name,	mdesc_bus_gen_get_name),
+	DEVMETHOD(mdesc_bus_get_type,	mdesc_bus_gen_get_type),
+
+	{ 0, 0 }
+};
+
+
+static driver_t vnex_driver = {
+	"vnex",
+	vnex_methods,
+	sizeof(struct vnex_softc),
+};
+
+
+static devclass_t vnex_devclass;
+DRIVER_MODULE(vnex, nexus, vnex_driver, vnex_devclass, 0, 0);
+
+
+static int
+vnex_probe(device_t dev)
+{
+	if (strcmp(ofw_bus_get_name(dev), "virtual-devices"))
+		return (ENXIO);
+
+	device_set_desc(dev, "virtual nexus device");
+	return (0);
+}
+
+
+static int
+vnex_attach(device_t dev)
+{
+	struct vnex_devinfo  *vndi;
+	struct vnex_softc    *sc;
+	device_t              cdev;
+	phandle_t             node;
+	mde_cookie_t          rootnode, *listp = NULL;
+	int                   i, listsz, num_nodes, num_devices;
+	md_t                 *mdp;
+
+
+	node = ofw_bus_get_node(dev);
+	if (node == -1)
+		panic("%s: ofw_bus_get_node failed.", __func__);
+
+	sc = device_get_softc(dev);
+	sc->sc_intr_rman.rm_type = RMAN_ARRAY;
+	sc->sc_intr_rman.rm_descr = "Interrupts";
+	sc->sc_mem_rman.rm_type = RMAN_ARRAY;
+	sc->sc_mem_rman.rm_descr = "Device Memory";
+	if (rman_init(&sc->sc_intr_rman) != 0 ||
+	    rman_init(&sc->sc_mem_rman) != 0 ||
+	    rman_manage_region(&sc->sc_intr_rman, 0, IV_MAX - 1) != 0 ||
+	    rman_manage_region(&sc->sc_mem_rman, 0ULL, ~0ULL) != 0)
+		panic("%s: failed to set up rmans.", __func__);
+
+	if ((mdp = md_get()) == NULL) 
+		return (ENXIO);
+
+	num_nodes = md_node_count(mdp);
+	listsz = num_nodes * sizeof(mde_cookie_t);
+	listp = (mde_cookie_t *)malloc(listsz, M_DEVBUF, M_WAITOK);
+	rootnode = md_root_node(mdp);
+	
+	/*
+	 * scan the machine description for virtual devices
+	 */
+	num_devices = md_scan_dag(mdp, rootnode, 
+				  md_find_name(mdp, "virtual-device"),
+				  md_find_name(mdp, "fwd"), listp);
+
+	for (i = 0; i < num_devices; i++) {
+		if ((vndi = vnex_setup_dinfo(dev, listp[i])) == NULL)
+			continue;
+
+		cdev = device_add_child(dev, NULL, -1);
+		if (cdev == NULL) {
+			device_printf(dev, "<%s>: device_add_child failed\n",
+			    vndi->vndi_mbdinfo.mbd_name);
+			vnex_destroy_dinfo(vndi);
+			continue;
+		}
+		device_set_ivars(cdev, vndi);
+	}
+
+	bus_generic_attach(dev);
+	return (0);
+}
+
+static device_t
+vnex_add_child(device_t dev, int order, const char *name, int unit)
+{
+	device_t cdev;
+	struct vnex_devinfo *vndi;
+
+	cdev = device_add_child_ordered(dev, order, name, unit);
+	if (cdev == NULL)
+		return (NULL);
+
+	vndi = malloc(sizeof(*vndi), M_DEVBUF, M_WAITOK | M_ZERO);
+	vndi->vndi_mbdinfo.mbd_name = strdup(name, M_OFWPROP);
+	resource_list_init(&vndi->vndi_rl);
+	device_set_ivars(cdev, vndi);
+
+	return (cdev);
+}
+
+static int
+vnex_print_child(device_t dev, device_t child)
+{
+	int rv;
+
+	rv = bus_print_child_header(dev, child);
+	rv += vnex_print_res(device_get_ivars(child));
+	rv += bus_print_child_footer(dev, child);
+	return (rv);
+}
+
+static void
+vnex_probe_nomatch(device_t dev, device_t child)
+{
+	const char *type;
+
+	device_printf(dev, "<%s>", mdesc_bus_get_name(child));
+	vnex_print_res(device_get_ivars(child));
+	type = mdesc_bus_get_type(child);
+	printf(" type %s (no driver attached)\n",
+	    type != NULL ? type : "unknown");
+}
+
+
+static int
+vnex_setup_intr(device_t dev, device_t child, struct resource *res, int flags,
+    driver_intr_t *intr, void *arg, void **cookiep)
+{
+
+	uint64_t reg, nreg;
+	uint64_t ihdl, cfg;
+	uint64_t ino, nino;
+	int error, cpuid;
+	
+	if (res == NULL)
+		panic("%s: NULL interrupt resource!", __func__);
+
+	if ((error = bus_get_resource(dev, SYS_RES_MEMORY, 0, &reg, &nreg)))
+		goto fail;
+
+	if ((error = bus_get_resource(child, SYS_RES_IRQ, 0, &ino, &nino)))
+		goto fail;
+       
+	cfg = SUN4V_REG_SPEC2CFG_HDL(reg);
+
+	if (hvio_intr_devino_to_sysino(cfg, (uint32_t)ino, &ihdl) != H_EOK) {
+		error = ENXIO;
+		goto fail;
+	}
+
+	cpuid = 0;
+
+	if (hvio_intr_settarget(ihdl, cpuid) != H_EOK) {
+		error = ENXIO;
+		goto fail;
+	}
+
+	if (hvio_intr_setstate(ihdl, HV_INTR_IDLE_STATE) != H_EOK) {
+		error = ENXIO;
+		goto fail;
+	}
+
+	if (hvio_intr_setvalid(ihdl, HV_INTR_VALID) != H_EOK) {
+		error = ENXIO;
+		goto fail;
+	}
+	
+	if ((rman_get_flags(res) & RF_SHAREABLE) == 0)
+		flags |= INTR_EXCL;
+
+	/* We depend here on rman_activate_resource() being idempotent. */
+	if ((error = rman_activate_resource(res)))
+		goto fail;
+
+	error = inthand_add(device_get_nameunit(child), ihdl,
+			    intr, arg, flags, cookiep);
+
+	printf("inthandler added\n");
+fail:
+
+	return (error);
+}
+
+static int
+vnex_teardown_intr(device_t dev, device_t child, struct resource *r, void *ih)
+{
+
+	inthand_remove(rman_get_start(r), ih);
+	return (0);
+}
+
+static struct resource_list *
+vnex_get_resource_list(device_t dev, device_t child)
+{
+	struct vnex_devinfo *vndi;
+
+	vndi = device_get_ivars(child);
+	return (&vndi->vndi_rl);
+}
+
+static const struct mdesc_bus_devinfo *
+vnex_get_devinfo(device_t dev, device_t child)
+{
+	struct vnex_devinfo *vndi;
+
+	vndi = device_get_ivars(child);
+	return (&vndi->vndi_mbdinfo);
+}
+
+static struct vnex_devinfo *
+vnex_setup_dinfo(device_t dev, mde_cookie_t node)
+{
+	struct vnex_devinfo *vndi;
+
+	vndi = malloc(sizeof(*vndi), M_DEVBUF, M_WAITOK | M_ZERO);
+	if (mdesc_bus_gen_setup_devinfo(&vndi->vndi_mbdinfo, node) != 0) {
+		free(vndi, M_DEVBUF);
+		return (NULL);
+	}
+
+	return (vndi);
+}
+
+static void
+vnex_destroy_dinfo(struct vnex_devinfo *vndi)
+{
+
+	resource_list_free(&vndi->vndi_rl);
+	mdesc_bus_gen_destroy_devinfo(&vndi->vndi_mbdinfo);
+	free(vndi, M_DEVBUF);
+}
+
+
+static int
+vnex_print_res(struct vnex_devinfo *vndi)
+{
+	int rv;
+
+	rv = 0;
+	rv += resource_list_print_type(&vndi->vndi_rl, "mem", SYS_RES_MEMORY,
+	    "%#lx");
+	rv += resource_list_print_type(&vndi->vndi_rl, "irq", SYS_RES_IRQ,
+	    "%ld");
+	return (rv);
+}
diff --git a/sys/sun4v/sun4v/wbuf.S b/sys/sun4v/sun4v/wbuf.S
new file mode 100644
index 000000000000..1e4c1bb98ea0
--- /dev/null
+++ b/sys/sun4v/sun4v/wbuf.S
@@ -0,0 +1,198 @@
+
+
+#include <machine/asm.h>
+__FBSDID("$FreeBSD$")
+	
+#include <machine/asi.h>
+#include <machine/asmacros.h>
+#include <machine/pstate.h>
+
+#include "assym.s"
+
+ENTRY(fault_32bit_sn0)
+	MAGIC_TRAP_ON	
+	MAGIC_EXIT	
+END(fault_32bit_sn0)
+
+ENTRY(fault_32bit_sn1)
+	MAGIC_TRAP_ON	
+	MAGIC_EXIT	
+END(fault_32bit_sn1)
+
+ENTRY(fault_32bit_so0)
+	MAGIC_TRAP_ON	
+	MAGIC_EXIT	
+END(fault_32bit_so0)
+
+ENTRY(fault_32bit_so1)
+	MAGIC_TRAP_ON	
+	MAGIC_EXIT	
+END(fault_32bit_so1)
+
+ENTRY(fault_64bit_sn0)
+	GET_PCB(%g4)
+        stx	%sp, [%g4 + PCB_RWSP]
+        add     %g4, PCB_RW, %g3
+        SAVE_WINDOW(%g3)
+	mov	1, %g3
+	stx	%g3, [%g4 + PCB_NSAVED]
+	set	trap, %g1
+	mov	%g5, %g2
+	mov	%g6, %g3
+	sub	%g0, 1, %g4
+
+	
+	rdpr	%tstate, %g5
+	and	%g5, TSTATE_CWP_MASK, %g5
+	ba,pt	%xcc, tl0_utrap
+	  wrpr	%g0, %g5, %cwp	
+END(fault_64bit_sn0)
+
+ENTRY(fault_64bit_sn1)
+	/* XXX need to use physical addresses here */
+	GET_PCB_PHYS(%g5, %g6)
+        wr      %g0, ASI_REAL, %asi
+        stxa    %sp, [%g6 + PCB_RWSP]%asi
+        add     %g6, PCB_RW, %g5
+        SAVE_WINDOW_ASI(%g5)
+        mov     1, %g5
+        stxa    %g5, [%g6 + PCB_NSAVED]%asi
+	saved
+	set	tl0_trap, %g5
+	wrpr	%g5, %tnpc
+	done
+END(fault_64bit_sn1)
+
+ENTRY(fault_32bit_sk)
+	MAGIC_TRAP_ON
+	MAGIC_EXIT	
+END(fault_32bit_sk)
+
+ENTRY(fault_64bit_sk)
+	GET_PCPU_PHYS_SCRATCH(%g5)
+	wr	%g0, ASI_REAL, %asi
+	stxa	%sp, [PCPU_REG + PC_KWBUF_SP]%asi
+	add	PCPU_REG, PC_KWBUF, %g6
+	SAVE_WINDOW_ASI(%g6)
+	mov	1, %g6
+	sta	%g6, [PCPU_REG + PC_KWBUF_FULL]%asi
+	saved
+	retry
+END(fault_64bit_sk)
+
+	
+ENTRY(fault_64bit_so0)
+	GET_PCB(%g6)
+	ldx	[%g6 + PCB_NSAVED], %g2
+	add	%g2, 1, %g3
+	stx	%g3, [%g6 + PCB_NSAVED]
+
+	sll	%g2, PTR_SHIFT, %g4
+	add	%g6, PCB_RWSP, %g3
+	stx	%sp, [%g3 + %g4]
+	sll	%g2, RW_SHIFT, %g4
+	add	%g4, %g6, %g4
+	add	%g4, PCB_RW, %g3
+	SAVE_WINDOW(%g3)
+	saved
+	retry
+END(fault_64bit_so0)
+
+ENTRY(fault_64bit_so1)
+	GET_PCB_PHYS(%g5, %g6)
+	wr	%g0, ASI_REAL, %asi
+	ldxa	[%g6 + PCB_NSAVED]%asi, %g5
+	add	%g5, 1, %g7
+	stxa	%g7, [%g6 + PCB_NSAVED]%asi
+
+	sll	%g5, PTR_SHIFT, %g7
+	add	%g6, %g7, %g7
+	stxa	%sp, [%g7 + PCB_RWSP]%asi
+
+	sll	%g5, RW_SHIFT, %g7		! offset
+	add	%g6, %g7, %g7			! pcb + offset
+	add	%g7, PCB_RW, %g7		! offset into wbuf area
+
+	SAVE_WINDOW_ASI(%g7)
+	saved
+	set	tl0_trap, %g5
+	wrpr	%g5, %tnpc
+	done
+END(fault_64bit_so1)
+
+ENTRY(fault_32bit_fn0)
+	MAGIC_TRAP_ON	
+	MAGIC_EXIT	
+fault_fn0_common:
+
+END(fault_32bit_fn0)
+	MAGIC_TRAP_ON	
+	MAGIC_EXIT	
+ENTRY(fault_32bit_fn1)
+	MAGIC_TRAP_ON	
+	MAGIC_EXIT
+fault_fn1_common:
+	rdpr	%tstate, %g1
+	and	%g1, TSTATE_CWP_MASK, %g1
+	wrpr	%g0, %g1, %cwp
+	!
+	! fake tl1 traps regs so that after pagefault runs, we
+	! re-execute at user_rtt.
+	!
+	wrpr	%g0, 1, %tl
+	set	TSTATE_KERNEL | TSTATE_IE, %g1
+	wrpr	%g0, %g1, %tstate
+	set	user_rtt, %g1
+	wrpr	%g0, %g1, %tpc
+	add	%g1, 4, %g1
+	wrpr	%g0, %g1, %tnpc
+	
+	set	trap, %g1
+	mov	1, %g2
+	sllx	%g2, CTX_OTHER_SHIFT, %g2
+	or	%g5, %g2, %g2	
+	mov	%g6, %g3
+
+	sub	%g0, 1, %g4
+	rdpr	%wstate, %l1
+	sllx	%l1, WSTATE_SHIFT, %l1
+	wrpr	%l1, WSTATE_K64, %wstate
+	mov	KCONTEXT, %g5
+	mov	MMU_CID_P, %g6
+	SET_MMU_CONTEXT(%g6, %g5)
+	membar	#Sync
+	b	tl0_ktrap
+	  nop
+END(fault_32bit_fn1)
+	
+ENTRY(fault_64bit_fn0)
+	MAGIC_TRAP_ON	
+	MAGIC_EXIT	
+END(fault_64bit_fn0)
+
+ENTRY(fault_64bit_fn1)
+	wrpr	%g0, 1, %gl
+	b	fault_fn1_common
+	  nop
+END(fault_64bit_fn1)
+
+ENTRY(fault_rtt_fn1)
+	b	fault_fn1_common
+	  nop
+END(fault_rtt_fn1)
+
+
+ENTRY(fault_32bit_not)
+ENTRY(fault_64bit_not)
+
+	ba,pt	%xcc, ptl1_panic
+	  mov	PTL1_BAD_WTRAP, %g1
+	
+END(fault_32bit_not)
+END(fault_64bit_not)
+
+ENTRY(ptl1_panic)
+
+/* XXX IMPLEMENT ME */
+	
+END(ptl1_panic)