code:
/*	$OpenBSD: ifb.c,v 1.22 2013/10/20 20:07:27 miod Exp $	*/

/*
 * Copyright (c) 2007, 2008, 2009 Miodrag Vallat.
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

/*
 * Least-effort driver for the Sun Expert3D cards (based on the
 * ``Wildcat'' chips).
 *
 * There is no public documentation for these chips available.
 * Since they are no longer supported by 3DLabs (which got bought by
 * Creative), and Sun does not want to publish even minimal information
 * or source code, the best we can do is experiment.
 *
 * Quoting Alan Coopersmith in
 * http://mail.opensolaris.org/pipermail/opensolaris-discuss/2005-December/011885.html
 * ``Unfortunately, the lawyers have asked we not give details about why
 *   specific components are not being released.''
 */

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/device.h>
#include <sys/errno.h>
#include <sys/ioctl.h>
#include <sys/malloc.h>
#include <sys/pciio.h>

#include <uvm/uvm_extern.h>

#include <machine/autoconf.h>
#include <machine/bus.h>
#include <machine/intr.h>
#include <machine/openfirm.h>

#include <dev/pci/pcireg.h>
#include <dev/pci/pcivar.h>

#include <dev/wscons/wsconsio.h>
#include <dev/wscons/wsdisplayvar.h>

#include <dev/rasops/rasops.h>

#include <machine/fbvar.h>

#ifdef APERTURE
extern int allowaperture;
#endif

/*
 * Parts of the following hardware knowledge come from David S. Miller's
 * XVR-500 Linux driver (drivers/video/sunxvr500.c).
 */

/*
 * The Expert3D and Expert3d-Lite cards are built around the Wildcat
 * 5110, 6210 and 7210 chips.
 *
 * The card exposes the following resources:
 * - a 32MB (ifb), 64MB (xvr600) or 128MB (jfb) aperture window in which
 *   views to the different frame buffer areas can be mapped, in the first BAR.
 * - a 64KB or 128KB PROM and registers area, in the second BAR.
 * - a 8MB ``direct burst'' memory mapping, in the third BAR.
 *
 * The location of this BAR range is pointed to by a board-specific PCI
 * configuration register.
 *
 * In the state the PROM leaves us in, the 8MB frame buffer windows map
 * the video memory as interleaved stripes, of which the non-visible parts
 * can still be addressed (probably for fast screen switching).
 *
 * Unfortunately, since we do not know how to reconfigure the stripes
 * to provide at least a linear frame buffer, we have to write to both
 * windows and have them provide the complete image.
 *
 * Moreover, high pixel values in the overlay planes (such as 0xff or 0xfe)
 * seem to enable other planes with random contents, so we'll limit ourselves
 * to 7bpp operation.
 */

/*
 * The Fcode driver sets up a communication structure, allowing third-party
 * code to reprogram the video mode while still allowing the Fcode routines
 * to access the overlay planes.
 *
 * We'll use this information as well, although so far it's unlikely
 * any code will do so, as long as the only documentation for this
 * hardware amounts to zilch.
 */

/* probably some form of signature */
#define	IFB_SHARED_SIGNATURE		0x00
#define	SIG_IFB					0x09209911
#define	SIG_JFB					0x05140213
#define	IFB_SHARED_MONITOR_MODE		0x10
#define	IFB_SHARED_WIDTH		0x14
#define	IFB_SHARED_HEIGHT		0x18
#define	IFB_SHARED_V_FREQ		0x1c
#define	IFB_SHARED_TIMING_H_FP		0x20
#define	IFB_SHARED_TIMING_H_SYNC	0x24
#define	IFB_SHARED_TIMING_H_BP		0x28
#define	IFB_SHARED_TIMING_V_FP		0x2c
#define	IFB_SHARED_TIMING_V_SYNC	0x30
#define	IFB_SHARED_TIMING_V_BP		0x34
#define	IFB_SHARED_TIMING_FLAGS		0x38
#define	IFB_SHARED_CMAP_DIRTY		0x3c
#define	IFB_SHARED_TERM8_GSR		0x4c
#define	IFB_SHARED_TERM8_SPR		0x50
#define	IFB_SHARED_TERM8_SPLR		0x54

/*
 * The Expert3D has an extra BAR that is not present on the -Lite
 * version.  This register contains bits that tell us how many BARs to
 * skip before we get to the BARs that interest us.
 */
#define IFB_PCI_CFG			0x5c
#define IFB_PCI_CFG_BAR_OFFSET(x)	((x & 0x000000e0) >> 3)

/*
 * 6000 (jfb) / 8000 (ifb) engine command
 * This register is used to issue (some) commands sequences to the
 * acceleration hardware.
 */
#define JFB_REG_ENGINE			0x6000
#define IFB_REG_ENGINE			0x8000

/*
 * 8040 component configuration
 * This register controls which parts of the board will be addressed by
 * writes to other configuration registers.
 * Apparently the low two bytes control the frame buffer windows for the
 * given head (starting at 1).
 * The high two bytes are texture related.
 */
#define	IFB_REG_COMPONENT_SELECT	0x8040

/*
 * 8044 status
 * This register has a bit that signals completion of commands issued
 * to the acceleration hardware.
 */
#define IFB_REG_STATUS			0x8044
#define IFB_REG_STATUS_DONE			0x00000004

/*
 * 8058 magnifying configuration
 * This register apparently controls magnifying.
 * bits 5-6 select the window width divider (00: by 2, 01: by 4, 10: by 8,
 *   11: by 16)
 * bits 7-8 select the zoom factor (00: disabled, 01: x2, 10: x4, 11: x8)
 */
#define	IFB_REG_MAGNIFY			0x8058
#define	IFB_REG_MAGNIFY_DISABLE			0x00000000
#define	IFB_REG_MAGNIFY_X2			0x00000040
#define	IFB_REG_MAGNIFY_X4			0x00000080
#define	IFB_REG_MAGNIFY_X8			0x000000c0
#define	IFB_REG_MAGNIFY_WINDIV2			0x00000000
#define	IFB_REG_MAGNIFY_WINDIV4			0x00000010
#define	IFB_REG_MAGNIFY_WINDIV8			0x00000020
#define	IFB_REG_MAGNIFY_WINDIV16		0x00000030

/*
 * 8070 display resolution
 * Contains the size of the display, as ((height - 1) << 16) | (width - 1)
 */
#define	IFB_REG_RESOLUTION		0x8070
/*
 * 8074 configuration register
 * Contains 0x1a000088 | ((Log2 stride) << 16)
 */
#define	IFB_REG_CONFIG			0x8074
/*
 * 8078 32bit frame buffer window #0 (8 to 9 MB)
 * Contains the offset (relative to BAR0) of the 32 bit frame buffer window.
 */
#define	IFB_REG_FB32_0			0x8078
/*
 * 807c 32bit frame buffer window #1 (8 to 9 MB)
 * Contains the offset (relative to BAR0) of the 32 bit frame buffer window.
 */
#define	IFB_REG_FB32_1			0x807c
/*
 * 8080 8bit frame buffer window #0 (2 to 2.2 MB)
 * Contains the offset (relative to BAR0) of the 8 bit frame buffer window.
 */
#define	IFB_REG_FB8_0			0x8080
/*
 * 8084 8bit frame buffer window #1 (2 to 2.2 MB)
 * Contains the offset (relative to BAR0) of the 8 bit frame buffer window.
 */
#define	IFB_REG_FB8_1			0x8084
/*
 * 8088 unknown window (as large as a 32 bit frame buffer)
 */
#define	IFB_REG_FB_UNK0			0x8088
/*
 * 808c unknown window (as large as a 8 bit frame buffer)
 */
#define	IFB_REG_FB_UNK1			0x808c
/*
 * 8090 unknown window (as large as a 8 bit frame buffer)
 */
#define	IFB_REG_FB_UNK2			0x8090

/*
 * 80bc RAMDAC palette index register
 */
#define	IFB_REG_CMAP_INDEX		0x80bc
/*
 * 80c0 RAMDAC palette data register
 */
#define	IFB_REG_CMAP_DATA		0x80c0

/*
 * 80e4 DPMS state register
 * States ``off'' and ``suspend'' need chip reprogramming before video can
 * be enabled again.
 */
#define	IFB_REG_DPMS_STATE		0x80e4
#define	IFB_REG_DPMS_OFF			0x00000000
#define	IFB_REG_DPMS_SUSPEND			0x00000001
#define	IFB_REG_DPMS_STANDBY			0x00000002
#define	IFB_REG_DPMS_ON				0x00000003

/*
 * (some) ROP codes
 */

#define	IFB_ROP_CLEAR	0x00000000	/* clear bits in rop mask */
#define	IFB_ROP_SRC	0x00330000	/* copy src bits matching rop mask */
#define	IFB_ROP_XOR	0x00cc0000	/* xor src bits with rop mask */
#define	IFB_ROP_SET	0x00ff0000	/* set bits in rop mask */

#define IFB_COORDS(x, y)	((x) | (y) << 16)

/* blitter directions */
#define IFB_BLT_DIR_BACKWARDS_Y		(0x08 | 0x02)
#define IFB_BLT_DIR_BACKWARDS_X		(0x04 | 0x01)

#define	IFB_PIXELMASK	0x7f	/* 7bpp */

struct ifb_softc {
	struct sunfb sc_sunfb;

	bus_space_tag_t sc_mem_t;
	pcitag_t sc_pcitag;

	/* overlays mappings */
	bus_space_handle_t sc_mem_h;
	bus_addr_t sc_membase, sc_fb8bank0_base, sc_fb8bank1_base;
	bus_size_t sc_memlen;
	vaddr_t	sc_memvaddr, sc_fb8bank0_vaddr, sc_fb8bank1_vaddr;

	/* registers mapping */
	bus_space_handle_t sc_reg_h;
	bus_addr_t sc_regbase;
	bus_size_t sc_reglen;

	/* communication area */
	volatile uint32_t *sc_comm;

	/* acceleration information */
	u_int	sc_acceltype;
#define	IFB_ACCEL_NONE			0
#define	IFB_ACCEL_IFB			1	/* Expert3D style */
#define	IFB_ACCEL_JFB			2	/* XVR-500 style */
	void (*sc_rop)(void *, int, int, int, int, int, int, uint32_t, int32_t);

	/* wsdisplay related goo */
	u_int	sc_mode;
	struct wsdisplay_emulops sc_old_ops;
	u_int8_t sc_cmap_red[256];
	u_int8_t sc_cmap_green[256];
	u_int8_t sc_cmap_blue[256];
};

int	ifb_ioctl(void *, u_long, caddr_t, int, struct proc *);
paddr_t	ifb_mmap(void *, off_t, int);
void	ifb_burner(void *, u_int, u_int);

struct wsdisplay_accessops ifb_accessops = {
	.ioctl = ifb_ioctl,
	.mmap = ifb_mmap,
	.burn_screen = ifb_burner
};

int	ifbmatch(struct device *, void *, void *);
void	ifbattach(struct device *, struct device *, void *);

struct cfattach ifb_ca = {
	sizeof (struct ifb_softc), ifbmatch, ifbattach
};

struct cfdriver ifb_cd = {
	NULL, "ifb", DV_DULL
};

int	ifb_accel_identify(const char *);
static inline
u_int	ifb_dac_value(u_int, u_int, u_int);
int	ifb_getcmap(struct ifb_softc *, struct wsdisplay_cmap *);
static inline
int	ifb_is_console(int);
int	ifb_mapregs(struct ifb_softc *, struct pci_attach_args *);
int	ifb_putcmap(struct ifb_softc *, struct wsdisplay_cmap *);
void	ifb_setcolor(void *, u_int, u_int8_t, u_int8_t, u_int8_t);
void	ifb_setcolormap(struct sunfb *,
	    void (*)(void *, u_int, u_int8_t, u_int8_t, u_int8_t));

void	ifb_copyrect(struct ifb_softc *, int, int, int, int, int, int);
void	ifb_fillrect(struct ifb_softc *, int, int, int, int, int);
static inline
void	ifb_rop(struct ifb_softc *, int, int, int, int, int, int, uint32_t,
	    int32_t);
void	ifb_rop_common(struct ifb_softc *, bus_addr_t, int, int, int, int,
	    int, int, uint32_t, int32_t);
void	ifb_rop_ifb(void *, int, int, int, int, int, int, uint32_t, int32_t);
void	ifb_rop_jfb(void *, int, int, int, int, int, int, uint32_t, int32_t);
int	ifb_rop_wait(struct ifb_softc *);

int	ifb_putchar_dumb(void *, int, int, u_int, long);
int	ifb_copycols_dumb(void *, int, int, int, int);
int	ifb_erasecols_dumb(void *, int, int, int, long);
int	ifb_copyrows_dumb(void *, int, int, int);
int	ifb_eraserows_dumb(void *, int, int, long);
int	ifb_do_cursor_dumb(struct rasops_info *);

int	ifb_putchar(void *, int, int, u_int, long);
int	ifb_copycols(void *, int, int, int, int);
int	ifb_erasecols(void *, int, int, int, long);
int	ifb_copyrows(void *, int, int, int);
int	ifb_eraserows(void *, int, int, long);
int	ifb_do_cursor(struct rasops_info *);

int
ifbmatch(struct device *parent, void *cf, void *aux)
{
	return ifb_ident(aux);
}

void
ifbattach(struct device *parent, struct device *self, void *aux)
{
	struct ifb_softc *sc = (struct ifb_softc *)self;
	struct pci_attach_args *paa = aux;
	struct rasops_info *ri;
	uint32_t dev_comm;
	int node, console;
	char *name, *text;
	char namebuf[32];

	sc->sc_mem_t = paa->pa_memt;
	sc->sc_pcitag = paa->pa_tag;

	node = PCITAG_NODE(paa->pa_tag);
	console = ifb_is_console(node);

	printf("\n");

	/*
	 * Multiple heads appear as PCI subfunctions.
	 * However, the ofw node for it lacks most properties,
	 * and its BAR only give access to registers, not
	 * frame buffer memory.
	 */
	if (!node_has_property(node, "device_type")) {
		printf("%s: secondary output not supported yet\n",
		    self->dv_xname);
		return;
	}

	/*
	 * Describe the beast.
	 */

	name = text = getpropstringA(node, "name", namebuf);
	if (strncmp(text, "SUNW,", 5) == 0)
		text += 5;
	printf("%s: %s", self->dv_xname, text);
	text = getpropstring(node, "model");
	if (*text != '\0')
		printf(" (%s)", text);

	if (ifb_mapregs(sc, paa))
		return;

	sc->sc_fb8bank0_base = bus_space_read_4(sc->sc_mem_t, sc->sc_reg_h,
	      IFB_REG_FB8_0);
	sc->sc_fb8bank1_base = bus_space_read_4(sc->sc_mem_t, sc->sc_reg_h,
	      IFB_REG_FB8_1);

	sc->sc_memvaddr = (vaddr_t)bus_space_vaddr(sc->sc_mem_t, sc->sc_mem_h);
	sc->sc_fb8bank0_vaddr = sc->sc_memvaddr +
	    sc->sc_fb8bank0_base - sc->sc_membase;
	sc->sc_fb8bank1_vaddr = sc->sc_memvaddr +
	    sc->sc_fb8bank1_base - sc->sc_membase;

	/*
	 * The values stored into the node properties might have been
	 * modified since the Fcode was last run. Pick the geometry
	 * information from the configuration registers instead.
	 * This replaces
	fb_setsize(&sc->sc_sunfb, 8, 1152, 900, node, 0);
	 */

	sc->sc_sunfb.sf_width = (bus_space_read_4(sc->sc_mem_t, sc->sc_reg_h,
	    IFB_REG_RESOLUTION) & 0xffff) + 1;
	sc->sc_sunfb.sf_height = (bus_space_read_4(sc->sc_mem_t, sc->sc_reg_h,
	    IFB_REG_RESOLUTION) >> 16) + 1;
	sc->sc_sunfb.sf_depth = 8;
	sc->sc_sunfb.sf_linebytes = 1 << (bus_space_read_4(sc->sc_mem_t,
	    sc->sc_reg_h, IFB_REG_CONFIG) >> 16);
	sc->sc_sunfb.sf_fbsize =
	    sc->sc_sunfb.sf_height * sc->sc_sunfb.sf_linebytes;

	printf(", %dx%d\n", sc->sc_sunfb.sf_width, sc->sc_sunfb.sf_height);

	ri = &sc->sc_sunfb.sf_ro;
	ri->ri_bits = NULL;
	ri->ri_hw = sc;

	fbwscons_init(&sc->sc_sunfb, RI_BSWAP, console);

	/*
	 * Find out what flavour of ifb we are...
	 */

	sc->sc_acceltype = ifb_accel_identify(name);

	switch (sc->sc_acceltype) {
	case IFB_ACCEL_IFB:
		sc->sc_rop = ifb_rop_ifb;
		break;
	case IFB_ACCEL_JFB:
		/*
		 * Remember the address of the communication area
		 */
		if (OF_getprop(node, "dev-comm", &dev_comm,
		    sizeof dev_comm) != -1) {
			sc->sc_comm = (volatile uint32_t *)(vaddr_t)dev_comm;
		}
		sc->sc_rop = ifb_rop_jfb;
		break;
	}

	/*
	 * Clear the unwanted pixel planes: all if non console (thus
	 * white background), and all planes above 7bpp otherwise.
	 * This also allows to check whether the accelerated code works,
	 * or not.
	 */

	if (sc->sc_acceltype != IFB_ACCEL_NONE) {
		ifb_rop(sc, 0, 0, 0, 0, sc->sc_sunfb.sf_width,
		    sc->sc_sunfb.sf_height, IFB_ROP_CLEAR,
		    console ? ~IFB_PIXELMASK : ~0);
		if (ifb_rop_wait(sc) == 0) {
			/* fall back to dumb software operation */
			sc->sc_acceltype = IFB_ACCEL_NONE;
		}
	}

	if (sc->sc_acceltype == IFB_ACCEL_NONE) {
		/* due to the way we will handle updates */
		ri->ri_flg &= ~RI_FULLCLEAR;

		if (!console) {
			bzero((void *)sc->sc_fb8bank0_vaddr,
			    sc->sc_sunfb.sf_fbsize);
			bzero((void *)sc->sc_fb8bank1_vaddr,
			    sc->sc_sunfb.sf_fbsize);
		}
	}

	/* pick centering delta */
	sc->sc_fb8bank0_vaddr += ri->ri_bits - ri->ri_origbits;
	sc->sc_fb8bank1_vaddr += ri->ri_bits - ri->ri_origbits;

	sc->sc_old_ops = ri->ri_ops;	/* structure copy */

	if (sc->sc_acceltype != IFB_ACCEL_NONE) {
		ri->ri_ops.copyrows = ifb_copyrows;
		ri->ri_ops.copycols = ifb_copycols;
		ri->ri_ops.eraserows = ifb_eraserows;
		ri->ri_ops.erasecols = ifb_erasecols;
		ri->ri_ops.putchar = ifb_putchar_dumb;
		ri->ri_do_cursor = ifb_do_cursor;
	} else {
		ri->ri_ops.copyrows = ifb_copyrows_dumb;
		ri->ri_ops.copycols = ifb_copycols_dumb;
		ri->ri_ops.eraserows = ifb_eraserows_dumb;
		ri->ri_ops.erasecols = ifb_erasecols_dumb;
		ri->ri_ops.putchar = ifb_putchar_dumb;
		ri->ri_do_cursor = ifb_do_cursor_dumb;
	}

	ifb_setcolormap(&sc->sc_sunfb, ifb_setcolor);
	sc->sc_mode = WSDISPLAYIO_MODE_EMUL;

	if (console)
		fbwscons_console_init(&sc->sc_sunfb, -1);
	fbwscons_attach(&sc->sc_sunfb, &ifb_accessops, console);
}

/*
 * Attempt to identify what kind of ifb we are talking to, so as to setup
 * proper acceleration information.
 */
int
ifb_accel_identify(const char *name)
{
	if (strcmp(name, "SUNW,Expert3D") == 0 ||
	    strcmp(name, "SUNW,Expert3D-Lite") == 0)
		return IFB_ACCEL_IFB;	/* ifblite */

	if (strcmp(name, "SUNW,XVR-1200") == 0)
		return IFB_ACCEL_JFB;	/* jfb */

	if (strcmp(name, "SUNW,XVR-600") == 0)
		return IFB_ACCEL_JFB;	/* xvr600 */

	/* XVR-500 is bobcat */

	return IFB_ACCEL_NONE;
}

int
ifb_ioctl(void *v, u_long cmd, caddr_t data, int flags, struct proc *p)
{
	struct ifb_softc *sc = v;
	struct wsdisplay_fbinfo *wdf;
	struct pcisel *sel;
	int mode;

	switch (cmd) {
	case WSDISPLAYIO_GTYPE:
		*(u_int *)data = WSDISPLAY_TYPE_IFB;
		break;

	case WSDISPLAYIO_SMODE:
		mode = *(u_int *)data;
		if (mode == WSDISPLAYIO_MODE_EMUL)
			ifb_setcolormap(&sc->sc_sunfb, ifb_setcolor);
		sc->sc_mode = mode;
		break;
	case WSDISPLAYIO_GINFO:
		wdf = (void *)data;
		wdf->height = sc->sc_sunfb.sf_height;
		wdf->width  = sc->sc_sunfb.sf_width;
		wdf->depth  = sc->sc_sunfb.sf_depth;
		wdf->cmsize = 256;
		break;
	case WSDISPLAYIO_LINEBYTES:
		*(u_int *)data = sc->sc_sunfb.sf_linebytes;
		break;

	case WSDISPLAYIO_GETCMAP:
		return ifb_getcmap(sc, (struct wsdisplay_cmap *)data);
	case WSDISPLAYIO_PUTCMAP:
		return ifb_putcmap(sc, (struct wsdisplay_cmap *)data);

	case WSDISPLAYIO_GPCIID:
		sel = (struct pcisel *)data;
		sel->pc_bus = PCITAG_BUS(sc->sc_pcitag);
		sel->pc_dev = PCITAG_DEV(sc->sc_pcitag);
		sel->pc_func = PCITAG_FUN(sc->sc_pcitag);
		break;

	case WSDISPLAYIO_SVIDEO:
	case WSDISPLAYIO_GVIDEO:
		break;

	case WSDISPLAYIO_GCURPOS:
	case WSDISPLAYIO_SCURPOS:
	case WSDISPLAYIO_GCURMAX:
	case WSDISPLAYIO_GCURSOR:
	case WSDISPLAYIO_SCURSOR:
	default:
		return -1; /* not supported yet */
        }

	return 0;
}

static inline
u_int
ifb_dac_value(u_int r, u_int g, u_int b)
{
	/*
	 * Convert 8 bit values to 10 bit scale, by shifting and inserting
	 * the former high bits in the low two bits.
	 * Simply shifting is sligthly too dull.
	 */
	r = (r << 2) | (r >> 6);
	g = (g << 2) | (g >> 6);
	b = (b << 2) | (b >> 6);

	return (b << 20) | (g << 10) | r;
}

int
ifb_getcmap(struct ifb_softc *sc, struct wsdisplay_cmap *cm)
{
	u_int index = cm->index;
	u_int count = cm->count;
	int error;

	if (index >= 256 || count > 256 - index)
		return EINVAL;

	error = copyout(&sc->sc_cmap_red[index], cm->red, count);
	if (error)
		return error;
	error = copyout(&sc->sc_cmap_green[index], cm->green, count);
	if (error)
		return error;
	error = copyout(&sc->sc_cmap_blue[index], cm->blue, count);
	if (error)
		return error;
	return 0;
}

int
ifb_putcmap(struct ifb_softc *sc, struct wsdisplay_cmap *cm)
{
	u_int index = cm->index;
	u_int count = cm->count;
	u_int i;
	int error;
	u_char *r, *g, *b;

	if (index >= 256 || count > 256 - index)
		return EINVAL;

	if ((error = copyin(cm->red, &sc->sc_cmap_red[index], count)) != 0)
		return error;
	if ((error = copyin(cm->green, &sc->sc_cmap_green[index], count)) != 0)
		return error;
	if ((error = copyin(cm->blue, &sc->sc_cmap_blue[index], count)) != 0)
		return error;

	r = &sc->sc_cmap_red[index];
	g = &sc->sc_cmap_green[index];
	b = &sc->sc_cmap_blue[index];

	for (i = 0; i < count; i++) {
		bus_space_write_4(sc->sc_mem_t, sc->sc_reg_h,
		    IFB_REG_CMAP_INDEX, index);
		bus_space_write_4(sc->sc_mem_t, sc->sc_reg_h, IFB_REG_CMAP_DATA,
		    ifb_dac_value(*r, *g, *b));
		r++, g++, b++, index++;
	}
	return 0;
}

void
ifb_setcolor(void *v, u_int index, u_int8_t r, u_int8_t g, u_int8_t b)
{
	struct ifb_softc *sc = v;

	sc->sc_cmap_red[index] = r;
	sc->sc_cmap_green[index] = g;
	sc->sc_cmap_blue[index] = b;

	bus_space_write_4(sc->sc_mem_t, sc->sc_reg_h, IFB_REG_CMAP_INDEX,
	    index);
	bus_space_write_4(sc->sc_mem_t, sc->sc_reg_h, IFB_REG_CMAP_DATA,
	    ifb_dac_value(r, g, b));
}

/* similar in spirit to fbwscons_setcolormap() */
void
ifb_setcolormap(struct sunfb *sf,
    void (*setcolor)(void *, u_int, u_int8_t, u_int8_t, u_int8_t))
{
	struct rasops_info *ri = &sf->sf_ro;
	int i;
	const u_char *color;

	/*
	 * Compensate for overlay plane limitations. Since we'll operate
	 * in 7bpp mode, our basic colors will use positions 00 to 0f,
	 * and the inverted colors will use positions 7f to 70.
	 */

	for (i = 0x00; i < 0x10; i++) {
		color = &rasops_cmap[i * 3];
		setcolor(sf, i, color[0], color[1], color[2]);
	}
	for (i = 0x70; i < 0x80; i++) {
		color = &rasops_cmap[(0xf0 | i) * 3];
		setcolor(sf, i, color[0], color[1], color[2]);
	}

	/*
	 * Proper operation apparently needs black to be 01, always.
	 * Replace black, red and white with white, black and red.
	 * Kind of ugly, but it works.
	 */
	ri->ri_devcmap[WSCOL_WHITE] = 0x00000000;
	ri->ri_devcmap[WSCOL_BLACK] = 0x01010101;
	ri->ri_devcmap[WSCOL_RED] = 0x07070707;

	color = &rasops_cmap[(WSCOL_WHITE + 8) * 3];	/* real white */
	setcolor(sf, 0, color[0], color[1], color[2]);
	setcolor(sf, IFB_PIXELMASK ^ 0, ~color[0], ~color[1], ~color[2]);
	color = &rasops_cmap[WSCOL_BLACK * 3];
	setcolor(sf, 1, color[0], color[1], color[2]);
	setcolor(sf, IFB_PIXELMASK ^ 1, ~color[0], ~color[1], ~color[2]);
	color = &rasops_cmap[WSCOL_RED * 3];
	setcolor(sf, 7, color[0], color[1], color[2]);
	setcolor(sf, IFB_PIXELMASK ^ 7, ~color[0], ~color[1], ~color[2]);
}

paddr_t
ifb_mmap(void *v, off_t off, int prot)
{
	struct ifb_softc *sc = (struct ifb_softc *)v;

	switch (sc->sc_mode) {
	case WSDISPLAYIO_MODE_MAPPED:
		/*
		 * In mapped mode, provide access to the two overlays,
		 * followed by the control registers, at the following
		 * addresses:
		 * 00000000	overlay 0, size up to 2MB (visible fb size)
		 * 01000000	overlay 1, size up to 2MB (visible fb size)
		 * 02000000	control registers
		 */
		off -= 0x00000000;
		if (off >= 0 && off < round_page(sc->sc_sunfb.sf_fbsize)) {
			return bus_space_mmap(sc->sc_mem_t,
			    sc->sc_fb8bank0_base,
			    off, prot, BUS_SPACE_MAP_LINEAR);
		}
		off -= 0x01000000;
		if (off >= 0 && off < round_page(sc->sc_sunfb.sf_fbsize)) {
			return bus_space_mmap(sc->sc_mem_t,
			    sc->sc_fb8bank1_base,
			    off, prot, BUS_SPACE_MAP_LINEAR);
		}
#ifdef APERTURE
		off -= 0x01000000;
		if (allowaperture != 0 && sc->sc_acceltype != IFB_ACCEL_NONE) {
			if (off >= 0 && off < round_page(sc->sc_reglen)) {
				return bus_space_mmap(sc->sc_mem_t,
				    sc->sc_regbase,
				    off, prot, BUS_SPACE_MAP_LINEAR);
			}
		}
#endif
		break;
	}

	return -1;
}

void
ifb_burner(void *v, u_int on, u_int flags)
{
	struct ifb_softc *sc = (struct ifb_softc *)v;
	int s;
	uint32_t dpms;

	s = splhigh();
	if (on)
		dpms = IFB_REG_DPMS_ON;
	else {
#ifdef notyet
		if (flags & WSDISPLAY_BURN_VBLANK)
			dpms = IFB_REG_DPMS_SUSPEND;
		else
#endif
			dpms = IFB_REG_DPMS_STANDBY;
	}
	bus_space_write_4(sc->sc_mem_t, sc->sc_reg_h, IFB_REG_DPMS_STATE, dpms);
	splx(s);
}

static inline int
ifb_is_console(int node)
{
	extern int fbnode;

	return fbnode == node;
}

int
ifb_mapregs(struct ifb_softc *sc, struct pci_attach_args *pa)
{
	u_int32_t cf;
	int bar, rc;

	cf = pci_conf_read(pa->pa_pc, pa->pa_tag, IFB_PCI_CFG);
	bar = PCI_MAPREG_START + IFB_PCI_CFG_BAR_OFFSET(cf);

	cf = pci_conf_read(pa->pa_pc, pa->pa_tag, bar);
	if (PCI_MAPREG_TYPE(cf) == PCI_MAPREG_TYPE_IO)
		rc = EINVAL;
	else {
		rc = pci_mapreg_map(pa, bar, cf,
		    BUS_SPACE_MAP_LINEAR, NULL, &sc->sc_mem_h,
		    &sc->sc_membase, &sc->sc_memlen, 0);
	}
	if (rc != 0) {
		printf("\n%s: can't map video memory\n",
		    sc->sc_sunfb.sf_dev.dv_xname);
		return rc;
	}

	cf = pci_conf_read(pa->pa_pc, pa->pa_tag, bar + 4);
	if (PCI_MAPREG_TYPE(cf) == PCI_MAPREG_TYPE_IO)
		rc = EINVAL;
	else {
		rc = pci_mapreg_map(pa, bar + 4, cf,
		    0, NULL, &sc->sc_reg_h,
		     &sc->sc_regbase, &sc->sc_reglen, 0x9000);
	}
	if (rc != 0) {
		printf("\n%s: can't map register space\n",
		    sc->sc_sunfb.sf_dev.dv_xname);
		return rc;
	}

	return 0;
}

/*
 * Non accelerated routines.
 */

int
ifb_putchar_dumb(void *cookie, int row, int col, u_int uc, long attr)
{
	struct rasops_info *ri = cookie;
	struct ifb_softc *sc = ri->ri_hw;

	ri->ri_bits = (void *)sc->sc_fb8bank0_vaddr;
	sc->sc_old_ops.putchar(cookie, row, col, uc, attr);
	ri->ri_bits = (void *)sc->sc_fb8bank1_vaddr;
	sc->sc_old_ops.putchar(cookie, row, col, uc, attr);

	return 0;
}

int
ifb_copycols_dumb(void *cookie, int row, int src, int dst, int num)
{
	struct rasops_info *ri = cookie;
	struct ifb_softc *sc = ri->ri_hw;

	ri->ri_bits = (void *)sc->sc_fb8bank0_vaddr;
	sc->sc_old_ops.copycols(cookie, row, src, dst, num);
	ri->ri_bits = (void *)sc->sc_fb8bank1_vaddr;
	sc->sc_old_ops.copycols(cookie, row, src, dst, num);

	return 0;
}

int
ifb_erasecols_dumb(void *cookie, int row, int col, int num, long attr)
{
	struct rasops_info *ri = cookie;
	struct ifb_softc *sc = ri->ri_hw;

	ri->ri_bits = (void *)sc->sc_fb8bank0_vaddr;
	sc->sc_old_ops.erasecols(cookie, row, col, num, attr);
	ri->ri_bits = (void *)sc->sc_fb8bank1_vaddr;
	sc->sc_old_ops.erasecols(cookie, row, col, num, attr);

	return 0;
}

int
ifb_copyrows_dumb(void *cookie, int src, int dst, int num)
{
	struct rasops_info *ri = cookie;
	struct ifb_softc *sc = ri->ri_hw;

	ri->ri_bits = (void *)sc->sc_fb8bank0_vaddr;
	sc->sc_old_ops.copyrows(cookie, src, dst, num);
	ri->ri_bits = (void *)sc->sc_fb8bank1_vaddr;
	sc->sc_old_ops.copyrows(cookie, src, dst, num);

	return 0;
}

int
ifb_eraserows_dumb(void *cookie, int row, int num, long attr)
{
	struct rasops_info *ri = cookie;
	struct ifb_softc *sc = ri->ri_hw;

	ri->ri_bits = (void *)sc->sc_fb8bank0_vaddr;
	sc->sc_old_ops.eraserows(cookie, row, num, attr);
	ri->ri_bits = (void *)sc->sc_fb8bank1_vaddr;
	sc->sc_old_ops.eraserows(cookie, row, num, attr);

	return 0;
}

/* Similar to rasops_do_cursor(), but using a 7bit pixel mask. */

#define	CURSOR_MASK	0x7f7f7f7f

int
ifb_do_cursor_dumb(struct rasops_info *ri)
{
	struct ifb_softc *sc = ri->ri_hw;
	int full1, height, cnt, slop1, slop2, row, col;
	int ovl_offset = sc->sc_fb8bank1_vaddr - sc->sc_fb8bank0_vaddr;
	u_char *dp0, *dp1, *rp;

	row = ri->ri_crow;
	col = ri->ri_ccol;

	ri->ri_bits = (void *)sc->sc_fb8bank0_vaddr;
	rp = ri->ri_bits + row * ri->ri_yscale + col * ri->ri_xscale;
	height = ri->ri_font->fontheight;
	slop1 = (4 - ((long)rp & 3)) & 3;

	if (slop1 > ri->ri_xscale)
		slop1 = ri->ri_xscale;

	slop2 = (ri->ri_xscale - slop1) & 3;
	full1 = (ri->ri_xscale - slop1 - slop2) >> 2;

	if ((slop1 | slop2) == 0) {
		/* A common case */
		while (height--) {
			dp0 = rp;
			dp1 = dp0 + ovl_offset;
			rp += ri->ri_stride;

			for (cnt = full1; cnt; cnt--) {
				*(int32_t *)dp0 ^= CURSOR_MASK;
				*(int32_t *)dp1 ^= CURSOR_MASK;
				dp0 += 4;
				dp1 += 4;
			}
		}
	} else {
		/* XXX this is stupid.. use masks instead */
		while (height--) {
			dp0 = rp;
			dp1 = dp0 + ovl_offset;
			rp += ri->ri_stride;

			if (slop1 & 1) {
				*dp0++ ^= (u_char)CURSOR_MASK;
				*dp1++ ^= (u_char)CURSOR_MASK;
			}

			if (slop1 & 2) {
				*(int16_t *)dp0 ^= (int16_t)CURSOR_MASK;
				*(int16_t *)dp1 ^= (int16_t)CURSOR_MASK;
				dp0 += 2;
				dp1 += 2;
			}

			for (cnt = full1; cnt; cnt--) {
				*(int32_t *)dp0 ^= CURSOR_MASK;
				*(int32_t *)dp1 ^= CURSOR_MASK;
				dp0 += 4;
				dp1 += 4;
			}

			if (slop2 & 1) {
				*dp0++ ^= (u_char)CURSOR_MASK;
				*dp1++ ^= (u_char)CURSOR_MASK;
			}

			if (slop2 & 2) {
				*(int16_t *)dp0 ^= (int16_t)CURSOR_MASK;
				*(int16_t *)dp1 ^= (int16_t)CURSOR_MASK;
			}
		}
	}

	return 0;
}

/*
 * Accelerated routines.
 */

int
ifb_copycols(void *cookie, int row, int src, int dst, int num)
{
	struct rasops_info *ri = cookie;
	struct ifb_softc *sc = ri->ri_hw;

	num *= ri->ri_font->fontwidth;
	src *= ri->ri_font->fontwidth;
	dst *= ri->ri_font->fontwidth;
	row *= ri->ri_font->fontheight;

	ifb_copyrect(sc, ri->ri_xorigin + src, ri->ri_yorigin + row,
	    ri->ri_xorigin + dst, ri->ri_yorigin + row,
	    num, ri->ri_font->fontheight);

	return 0;
}

int
ifb_erasecols(void *cookie, int row, int col, int num, long attr)
{
	struct rasops_info *ri = cookie;
	struct ifb_softc *sc = ri->ri_hw;
	int bg, fg;

	ri->ri_ops.unpack_attr(cookie, attr, &fg, &bg, NULL);

	row *= ri->ri_font->fontheight;
	col *= ri->ri_font->fontwidth;
	num *= ri->ri_font->fontwidth;

	ifb_fillrect(sc, ri->ri_xorigin + col, ri->ri_yorigin + row,
	    num, ri->ri_font->fontheight, ri->ri_devcmap[bg]);

	return 0;
}

int
ifb_copyrows(void *cookie, int src, int dst, int num)
{
	struct rasops_info *ri = cookie;
	struct ifb_softc *sc = ri->ri_hw;

	num *= ri->ri_font->fontheight;
	src *= ri->ri_font->fontheight;
	dst *= ri->ri_font->fontheight;

	ifb_copyrect(sc, ri->ri_xorigin, ri->ri_yorigin + src,
	    ri->ri_xorigin, ri->ri_yorigin + dst, ri->ri_emuwidth, num);

	return 0;
}

int
ifb_eraserows(void *cookie, int row, int num, long attr)
{
	struct rasops_info *ri = cookie;
	struct ifb_softc *sc = ri->ri_hw;
	int bg, fg;
	int x, y, w;

	ri->ri_ops.unpack_attr(cookie, attr, &fg, &bg, NULL);

	if ((num == ri->ri_rows) && ISSET(ri->ri_flg, RI_FULLCLEAR)) {
		num = ri->ri_height;
		x = y = 0;
		w = ri->ri_width;
	} else {
		num *= ri->ri_font->fontheight;
		x = ri->ri_xorigin;
		y = ri->ri_yorigin + row * ri->ri_font->fontheight;
		w = ri->ri_emuwidth;
	}
	ifb_fillrect(sc, x, y, w, num, ri->ri_devcmap[bg]);

	return 0;
}

void
ifb_copyrect(struct ifb_softc *sc, int sx, int sy, int dx, int dy, int w, int h)
{
	ifb_rop(sc, sx, sy, dx, dy, w, h, IFB_ROP_SRC, IFB_PIXELMASK);
	ifb_rop_wait(sc);
}

void
ifb_fillrect(struct ifb_softc *sc, int x, int y, int w, int h, int bg)
{
	int32_t mask;

	/* pixels to set... */
	mask = IFB_PIXELMASK & bg;
	if (mask != 0) {
		ifb_rop(sc, x, y, x, y, w, h, IFB_ROP_SET, mask);
		ifb_rop_wait(sc);
	}

	/* pixels to clear... */
	mask = IFB_PIXELMASK & ~bg;
	if (mask != 0) {
		ifb_rop(sc, x, y, x, y, w, h, IFB_ROP_CLEAR, mask);
		ifb_rop_wait(sc);
	}
}

/*
 * Perform a raster operation on both overlay planes.
 * Puzzled by all the magic numbers in there? So are we. Isn't a dire
 * lack of documentation wonderful?
 */

static inline void
ifb_rop(struct ifb_softc *sc, int sx, int sy, int dx, int dy, int w, int h,
    uint32_t rop, int32_t planemask)
{
	(*sc->sc_rop)(sc, sx, sy, dx, dy, w, h, rop, planemask);
}

void
ifb_rop_common(struct ifb_softc *sc, bus_addr_t reg, int sx, int sy,
    int dx, int dy, int w, int h, uint32_t rop, int32_t planemask)
{
	int dir = 0;

	/*
	 * Compute rop direction. This only really matters for
	 * screen-to-screen copies.
	 */
	if (sy < dy /* && sy + h > dy */) {
		sy += h - 1;
		dy += h;
		dir |= IFB_BLT_DIR_BACKWARDS_Y;
	}
	if (sx < dx /* && sx + w > dx */) {
		sx += w - 1;
		dx += w;
		dir |= IFB_BLT_DIR_BACKWARDS_X;
	}

	/* Which one of those below is your magic number for today? */
	bus_space_write_4(sc->sc_mem_t, sc->sc_reg_h, reg, 0x61000001);
	bus_space_write_4(sc->sc_mem_t, sc->sc_reg_h, reg, 0);
	bus_space_write_4(sc->sc_mem_t, sc->sc_reg_h, reg, 0x6301c080);
	bus_space_write_4(sc->sc_mem_t, sc->sc_reg_h, reg, 0x80000000);
	bus_space_write_4(sc->sc_mem_t, sc->sc_reg_h, reg, rop);
	bus_space_write_4(sc->sc_mem_t, sc->sc_reg_h, reg, planemask);
	bus_space_write_4(sc->sc_mem_t, sc->sc_reg_h, reg, 0);
	bus_space_write_4(sc->sc_mem_t, sc->sc_reg_h, reg, 0x64000303);
	/*
	 * This value is a pixel offset within the destination area. It is
	 * probably used to define complex polygon shapes, with the
	 * last pixel in the list being back to (0,0).
	 */
	bus_space_write_4(sc->sc_mem_t, sc->sc_reg_h, reg, IFB_COORDS(0, 0));
	bus_space_write_4(sc->sc_mem_t, sc->sc_reg_h, reg, 0);
	bus_space_write_4(sc->sc_mem_t, sc->sc_reg_h, reg, 0x00030000);
	bus_space_write_4(sc->sc_mem_t, sc->sc_reg_h, reg, 0x2200010d);

	bus_space_write_4(sc->sc_mem_t, sc->sc_reg_h, reg, 0x33f01000 | dir);
	bus_space_write_4(sc->sc_mem_t, sc->sc_reg_h, reg, IFB_COORDS(dx, dy));
	bus_space_write_4(sc->sc_mem_t, sc->sc_reg_h, reg, IFB_COORDS(w, h));
	bus_space_write_4(sc->sc_mem_t, sc->sc_reg_h, reg, IFB_COORDS(sx, sy));
}

void
ifb_rop_ifb(void *v, int sx, int sy, int dx, int dy, int w, int h,
    uint32_t rop, int32_t planemask)
{
	struct ifb_softc *sc = (struct ifb_softc *)v;
	bus_addr_t reg = IFB_REG_ENGINE;

	bus_space_write_4(sc->sc_mem_t, sc->sc_reg_h, reg, 2);
	bus_space_write_4(sc->sc_mem_t, sc->sc_reg_h, reg, 1);
	/* the ``0101'' part is probably a component selection */
	bus_space_write_4(sc->sc_mem_t, sc->sc_reg_h, reg, 0x540101ff);

	ifb_rop_common(sc, reg, sx, sy, dx, dy, w, h, rop, planemask);
}

void
ifb_rop_jfb(void *v, int sx, int sy, int dx, int dy, int w, int h,
    uint32_t rop, int32_t planemask)
{
	struct ifb_softc *sc = (struct ifb_softc *)v;
	bus_addr_t reg = JFB_REG_ENGINE;
	uint32_t spr, splr;

	/*
	 * Pick the current spr and splr values from the communication
	 * area if possible.
	 */
	if (sc->sc_comm != NULL) {
		spr = sc->sc_comm[IFB_SHARED_TERM8_SPR >> 2];
		splr = sc->sc_comm[IFB_SHARED_TERM8_SPLR >> 2];
	} else {
		/* supposedly sane defaults */
		spr = 0x54ff0303;
		splr = 0x5c0000ff;
	}

	bus_space_write_4(sc->sc_mem_t, sc->sc_reg_h, reg, 0x00400016);
	bus_space_write_4(sc->sc_mem_t, sc->sc_reg_h, reg, 0x5b000002);
	bus_space_write_4(sc->sc_mem_t, sc->sc_reg_h, reg, 0x5a000000);
	bus_space_write_4(sc->sc_mem_t, sc->sc_reg_h, reg, spr);
	bus_space_write_4(sc->sc_mem_t, sc->sc_reg_h, reg, splr);

	ifb_rop_common(sc, reg, sx, sy, dx, dy, w, h, rop, planemask);

	bus_space_write_4(sc->sc_mem_t, sc->sc_reg_h, reg, 0x5a000001);
	bus_space_write_4(sc->sc_mem_t, sc->sc_reg_h, reg, 0x5b000001);
}

int
ifb_rop_wait(struct ifb_softc *sc)
{
	int i;

	for (i = 1000000; i != 0; i--) {
		if (bus_space_read_4(sc->sc_mem_t, sc->sc_reg_h,
		    IFB_REG_STATUS) & IFB_REG_STATUS_DONE)
			break;
		DELAY(1);
	}

	return i;
}

int
ifb_do_cursor(struct rasops_info *ri)
{
	struct ifb_softc *sc = ri->ri_hw;
	int y, x;

	y = ri->ri_yorigin + ri->ri_crow * ri->ri_font->fontheight;
	x = ri->ri_xorigin + ri->ri_ccol * ri->ri_font->fontwidth;

	ifb_rop(sc, x, y, x, y, ri->ri_font->fontwidth, ri->ri_font->fontheight,
	    IFB_ROP_XOR, IFB_PIXELMASK);
	ifb_rop_wait(sc);

	return 0;
}


code:
/*	$OpenBSD: ifb_ident.c,v 1.2 2009/06/27 22:43:41 miod Exp $	*/

/*
 * Copyright (c) 2007, 2008 Miodrag Vallat.
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

/*
 * Identify an Expert3D card.
 * Used by both vgafb and ifb.
 */

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/device.h>

#include <machine/autoconf.h>
#include <machine/openfirm.h>

#include <dev/pci/pcireg.h>
#include <dev/pci/pcivar.h>

#include <dev/wscons/wsdisplayvar.h>
#include <dev/rasops/rasops.h>

#include <machine/fbvar.h>

int
ifb_ident(void *aux)
{
	struct pci_attach_args *paa = aux;
	int node;
	char *name;

	node = PCITAG_NODE(paa->pa_tag);
	name = getpropstring(node, "name");

	if (strcmp(name, "SUNW,Expert3D") == 0 ||
	    strcmp(name, "SUNW,Expert3D-Lite") == 0 ||
	    strcmp(name, "SUNW,XVR-500") == 0 ||
	    strcmp(name, "SUNW,XVR-600") == 0 ||
	    strcmp(name, "SUNW,XVR-1200") == 0)
		return 1;

	return 0;
}

license: unknown


code:
/*	$OpenBSD: iommu.c,v 1.72 2015/01/09 14:23:25 kettenis Exp $	*/
/*	$NetBSD: iommu.c,v 1.47 2002/02/08 20:03:45 eeh Exp $	*/

/*
 * Copyright (c) 2003 Henric Jungheim
 * Copyright (c) 2001, 2002 Eduardo Horvath
 * Copyright (c) 1999, 2000 Matthew R. Green
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. The name of the author may not be used to endorse or promote products
 *    derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */

/*
 * UltraSPARC IOMMU support; used by both the sbus and pci code.
 */
#include <sys/param.h>
#include <sys/extent.h>
#include <sys/malloc.h>
#include <sys/systm.h>
#include <sys/proc.h>
#include <sys/device.h>
#include <sys/mbuf.h>

#include <uvm/uvm_extern.h>

#include <machine/bus.h>
#include <sparc64/sparc64/cache.h>
#include <sparc64/dev/iommureg.h>
#include <sparc64/dev/iommuvar.h>

#include <machine/autoconf.h>
#include <machine/cpu.h>

#ifdef DDB
#include <machine/db_machdep.h>
#include <ddb/db_sym.h>
#include <ddb/db_extern.h>
#endif

#ifdef DEBUG
#define IDB_BUSDMA	0x1
#define IDB_IOMMU	0x2
#define IDB_INFO	0x4
#define IDB_SYNC	0x8
#define IDB_XXX		0x10
#define IDB_PRINT_MAP	0x20
#define IDB_BREAK	0x40
int iommudebug = IDB_INFO;
#define DPRINTF(l, s)   do { if (iommudebug & l) printf s; } while (0)
#else
#define DPRINTF(l, s)
#endif

void iommu_enter(struct iommu_state *, struct strbuf_ctl *, bus_addr_t,
    paddr_t, int);
void iommu_remove(struct iommu_state *, struct strbuf_ctl *, bus_addr_t);
int iommu_dvmamap_sync_range(struct strbuf_ctl*, bus_addr_t, bus_size_t);
int iommu_strbuf_flush_done(struct iommu_map_state *);
int iommu_dvmamap_load_seg(bus_dma_tag_t, struct iommu_state *,
    bus_dmamap_t, bus_dma_segment_t *, int, int, bus_size_t, bus_size_t);
int iommu_dvmamap_load_mlist(bus_dma_tag_t, struct iommu_state *,
    bus_dmamap_t, struct pglist *, int, bus_size_t, bus_size_t);
int iommu_dvmamap_validate_map(bus_dma_tag_t, struct iommu_state *,
    bus_dmamap_t);
void iommu_dvmamap_print_map(bus_dma_tag_t, struct iommu_state *,
    bus_dmamap_t);
int iommu_dvmamap_append_range(bus_dma_tag_t, bus_dmamap_t, paddr_t,
    bus_size_t, int, bus_size_t);
int64_t iommu_tsb_entry(struct iommu_state *, bus_addr_t);
void strbuf_reset(struct strbuf_ctl *);
int iommu_iomap_insert_page(struct iommu_map_state *, paddr_t);
bus_addr_t iommu_iomap_translate(struct iommu_map_state *, paddr_t);
void iommu_iomap_load_map(struct iommu_state *, struct iommu_map_state *,
    bus_addr_t, int);
void iommu_iomap_unload_map(struct iommu_state *, struct iommu_map_state *);
struct iommu_map_state *iommu_iomap_create(int);
void iommu_iomap_destroy(struct iommu_map_state *);
void iommu_iomap_clear_pages(struct iommu_map_state *);
void _iommu_dvmamap_sync(bus_dma_tag_t, bus_dma_tag_t, bus_dmamap_t,
    bus_addr_t, bus_size_t, int);

/*
 * Initiate an STC entry flush.
 */
static inline void
iommu_strbuf_flush(struct strbuf_ctl *sb, bus_addr_t va)
{
#ifdef DEBUG
	if (sb->sb_flush == NULL) {
		printf("iommu_strbuf_flush: attempting to flush w/o STC\n");
		return;
	}
#endif

	bus_space_write_8(sb->sb_bustag, sb->sb_sb,
	    STRBUFREG(strbuf_pgflush), va);
}

/*
 * initialise the UltraSPARC IOMMU (SBus or PCI):
 *	- allocate and setup the iotsb.
 *	- enable the IOMMU
 *	- initialise the streaming buffers (if they exist)
 *	- create a private DVMA map.
 */
void
iommu_init(char *name, struct iommu_state *is, int tsbsize, u_int32_t iovabase)
{
	psize_t size;
	vaddr_t va;
	paddr_t pa;
	struct vm_page *m;
	struct pglist mlist;

	/*
	 * Setup the iommu.
	 *
	 * The sun4u iommu is part of the SBus or PCI controller so we will
	 * deal with it here..
	 *
	 * For sysio and psycho/psycho+ the IOMMU address space always ends at
	 * 0xffffe000, but the starting address depends on the size of the
	 * map.  The map size is 1024 * 2 ^ is->is_tsbsize entries, where each
	 * entry is 8 bytes.  The start of the map can be calculated by
	 * (0xffffe000 << (8 + is->is_tsbsize)).
	 *
	 * But sabre and hummingbird use a different scheme that seems to
	 * be hard-wired, so we read the start and size from the PROM and
	 * just use those values.
	 */
	if (strncmp(name, "pyro", 4) == 0) {
		is->is_cr = IOMMUREG_READ(is, iommu_cr);
		is->is_cr &= ~IOMMUCR_FIRE_BE;
		is->is_cr |= (IOMMUCR_FIRE_SE | IOMMUCR_FIRE_CM_EN |
		    IOMMUCR_FIRE_TE);
	} else
		is->is_cr = IOMMUCR_EN;
	is->is_tsbsize = tsbsize;
	if (iovabase == (u_int32_t)-1) {
		is->is_dvmabase = IOTSB_VSTART(is->is_tsbsize);
		is->is_dvmaend = IOTSB_VEND;
	} else {
		is->is_dvmabase = iovabase;
		is->is_dvmaend = iovabase + IOTSB_VSIZE(tsbsize) - 1;
	}

	/*
	 * Allocate memory for I/O pagetables.  They need to be physically
	 * contiguous.
	 */

	size = PAGE_SIZE << is->is_tsbsize;
	TAILQ_INIT(&mlist);
	if (uvm_pglistalloc((psize_t)size, (paddr_t)0, (paddr_t)-1,
	    (paddr_t)PAGE_SIZE, (paddr_t)0, &mlist, 1, UVM_PLA_NOWAIT) != 0)
		panic("iommu_init: no memory");

	va = (vaddr_t)km_alloc(size, &kv_any, &kp_none, &kd_nowait);
	if (va == 0)
		panic("iommu_init: no memory");
	is->is_tsb = (int64_t *)va;

	m = TAILQ_FIRST(&mlist);
	is->is_ptsb = VM_PAGE_TO_PHYS(m);

	/* Map the pages */
	for (; m != NULL; m = TAILQ_NEXT(m,pageq)) {
		pa = VM_PAGE_TO_PHYS(m);
		pmap_enter(pmap_kernel(), va, pa | PMAP_NVC,
		    PROT_READ | PROT_WRITE,
		    PROT_READ | PROT_WRITE | PMAP_WIRED);
		va += PAGE_SIZE;
	}
	pmap_update(pmap_kernel());
	memset(is->is_tsb, 0, size);

#ifdef DEBUG
	if (iommudebug & IDB_INFO) {
		/* Probe the iommu */
		/* The address or contents of the regs...? */
		printf("iommu regs at: cr=%lx tsb=%lx flush=%lx\n",
		    (u_long)bus_space_vaddr(is->is_bustag, is->is_iommu) +
			IOMMUREG(iommu_cr),
		    (u_long)bus_space_vaddr(is->is_bustag, is->is_iommu) +
			IOMMUREG(iommu_tsb),
		    (u_long)bus_space_vaddr(is->is_bustag, is->is_iommu) +
			IOMMUREG(iommu_flush));
		printf("iommu cr=%llx tsb=%llx\n",
		    IOMMUREG_READ(is, iommu_cr),
		    IOMMUREG_READ(is, iommu_tsb));
		printf("TSB base %p phys %llx\n",
		    (void *)is->is_tsb, (unsigned long long)is->is_ptsb);
		delay(1000000); /* 1 s */
	}
#endif

	/*
	 * Now all the hardware's working we need to allocate a dvma map.
	 */
	printf("dvma map %x-%x", is->is_dvmabase, is->is_dvmaend);
#ifdef DEBUG
	printf(", iotdb %llx-%llx",
	    (unsigned long long)is->is_ptsb,
	    (unsigned long long)(is->is_ptsb + size));
#endif
	is->is_dvmamap = extent_create(name,
	    is->is_dvmabase, (u_long)is->is_dvmaend + 1,
	    M_DEVBUF, NULL, 0, EX_NOCOALESCE);
	mtx_init(&is->is_mtx, IPL_HIGH);

	/*
	 * Set the TSB size.  The relevant bits were moved to the TSB
	 * base register in the PCIe host bridges.
	 */
	if (strncmp(name, "pyro", 4) == 0)
		is->is_ptsb |= is->is_tsbsize;
	else
		is->is_cr |= (is->is_tsbsize << 16);

	/*
	 * Now actually start up the IOMMU.
	 */
	iommu_reset(is);
	printf("\n");
}

/*
 * Streaming buffers don't exist on the UltraSPARC IIi/e; we should have
 * detected that already and disabled them.  If not, we will notice that
 * they aren't there when the STRBUF_EN bit does not remain.
 */
void
iommu_reset(struct iommu_state *is)
{
	int i;

	IOMMUREG_WRITE(is, iommu_tsb, is->is_ptsb);

	/* Enable IOMMU */
	IOMMUREG_WRITE(is, iommu_cr, is->is_cr);

	for (i = 0; i < 2; ++i) {
		struct strbuf_ctl *sb = is->is_sb[i];

		if (sb == NULL)
			continue;

		sb->sb_iommu = is;
		strbuf_reset(sb);

		if (sb->sb_flush)
			printf(", STC%d enabled", i);
	}

	if (is->is_flags & IOMMU_FLUSH_CACHE)
		IOMMUREG_WRITE(is, iommu_cache_invalidate, -1ULL);
}

/*
 * Initialize one STC.
 */
void
strbuf_reset(struct strbuf_ctl *sb)
{
	if(sb->sb_flush == NULL)
		return;

	bus_space_write_8(sb->sb_bustag, sb->sb_sb,
	    STRBUFREG(strbuf_ctl), STRBUF_EN);

	membar(Lookaside);

	/* No streaming buffers? Disable them */
	if (bus_space_read_8(sb->sb_bustag, sb->sb_sb,
	    STRBUFREG(strbuf_ctl)) == 0) {
		sb->sb_flush = NULL;
	} else {
		/*
		 * locate the pa of the flush buffer
		 */
		if (pmap_extract(pmap_kernel(),
		    (vaddr_t)sb->sb_flush, &sb->sb_flushpa) == FALSE)
			sb->sb_flush = NULL;
		mtx_init(&sb->sb_mtx, IPL_HIGH);
	}
}

/*
 * Add an entry to the IOMMU table.
 *
 * The entry is marked streaming if an STC was detected and
 * the BUS_DMA_STREAMING flag is set.
 */
void
iommu_enter(struct iommu_state *is, struct strbuf_ctl *sb, bus_addr_t va,
    paddr_t pa, int flags)
{
	int64_t tte;
	volatile int64_t *tte_ptr = &is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)];

#ifdef DIAGNOSTIC
	if (va < is->is_dvmabase || (va + PAGE_MASK) > is->is_dvmaend)
		panic("iommu_enter: va %#lx not in DVMA space", va);

	tte = *tte_ptr;

	if (tte & IOTTE_V) {
		printf("Overwriting valid tte entry (dva %lx pa %lx "
		    "&tte %p tte %llx)\n", va, pa, tte_ptr, tte);
		extent_print(is->is_dvmamap);
		panic("IOMMU overwrite");
	}
#endif

	tte = MAKEIOTTE(pa, !(flags & BUS_DMA_NOWRITE),
	    !(flags & BUS_DMA_NOCACHE), (flags & BUS_DMA_STREAMING));

	DPRINTF(IDB_IOMMU, ("Clearing TSB slot %d for va %p\n",
	    (int)IOTSBSLOT(va,is->is_tsbsize), (void *)(u_long)va));

	*tte_ptr = tte;

	/*
	 * Why bother to flush this va?  It should only be relevant for
	 * V ==> V or V ==> non-V transitions.  The former is illegal and
	 * the latter is never done here.  It is true that this provides
	 * some protection against a misbehaving master using an address
	 * after it should.  The IOMMU documentations specifically warns
	 * that the consequences of a simultaneous IOMMU flush and DVMA
	 * access to the same address are undefined.  (By that argument,
	 * the STC should probably be flushed as well.)   Note that if
	 * a bus master keeps using a memory region after it has been
	 * unmapped, the specific behavior of the IOMMU is likely to
	 * be the least of our worries.
	 */
	IOMMUREG_WRITE(is, iommu_flush, va);

	DPRINTF(IDB_IOMMU, ("iommu_enter: va %lx pa %lx TSB[%lx]@%p=%lx\n",
	    va, (long)pa, (u_long)IOTSBSLOT(va,is->is_tsbsize),
	    (void *)(u_long)&is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)],
	    (u_long)tte));
}

/*
 * Remove an entry from the IOMMU table.
 *
 * The entry is flushed from the STC if an STC is detected and the TSB
 * entry has the IOTTE_STREAM flags set.  It should be impossible for
 * the TSB entry to have this flag set without the BUS_DMA_STREAMING
 * flag, but better to be safe.  (The IOMMU will be ignored as long
 * as an STC entry exists.)
 */
void
iommu_remove(struct iommu_state *is, struct strbuf_ctl *sb, bus_addr_t va)
{
	int64_t *tte_ptr = &is->is_tsb[IOTSBSLOT(va, is->is_tsbsize)];
	int64_t tte;

#ifdef DIAGNOSTIC
	if (va < is->is_dvmabase || (va + PAGE_MASK) > is->is_dvmaend)
		panic("iommu_remove: va 0x%lx not in DVMA space", (u_long)va);
	if (va != trunc_page(va)) {
		printf("iommu_remove: unaligned va: %lx\n", va);
		va = trunc_page(va);
	}
#endif
	tte = *tte_ptr;

	DPRINTF(IDB_IOMMU, ("iommu_remove: va %lx TSB[%llx]@%p\n",
	    va, tte, tte_ptr));

#ifdef DIAGNOSTIC
	if ((tte & IOTTE_V) == 0) {
		printf("Removing invalid tte entry (dva %lx &tte %p "
		    "tte %llx)\n", va, tte_ptr, tte);
		extent_print(is->is_dvmamap);
		panic("IOMMU remove overwrite");
	}
#endif

	*tte_ptr = tte & ~IOTTE_V;

	/*
	 * IO operations are strongly ordered WRT each other.  It is
	 * unclear how they relate to normal memory accesses.
	 */
	membar(StoreStore);

	IOMMUREG_WRITE(is, iommu_flush, va);

	if (sb && (tte & IOTTE_STREAM))
		iommu_strbuf_flush(sb, va);

	/* Should we sync the iommu and stc here? */
}

/*
 * Find the physical address of a DVMA address (debug routine).
 */
paddr_t
iommu_extract(struct iommu_state *is, bus_addr_t dva)
{
	int64_t tte = 0;

	if (dva >= is->is_dvmabase && dva <= is->is_dvmaend)
		tte = is->is_tsb[IOTSBSLOT(dva, is->is_tsbsize)];

	return (tte & IOTTE_PAMASK);
}

/*
 * Lookup a TSB entry for a given DVMA (debug routine).
 */
int64_t
iommu_lookup_tte(struct iommu_state *is, bus_addr_t dva)
{
	int64_t tte = 0;

	if (dva >= is->is_dvmabase && dva <= is->is_dvmaend)
		tte = is->is_tsb[IOTSBSLOT(dva, is->is_tsbsize)];

	return (tte);
}

/*
 * Lookup a TSB entry at a given physical address (debug routine).
 */
int64_t
iommu_fetch_tte(struct iommu_state *is, paddr_t pa)
{
	int64_t tte = 0;

	if (pa >= is->is_ptsb && pa < is->is_ptsb +
	    (PAGE_SIZE << is->is_tsbsize))
		tte = ldxa(pa, ASI_PHYS_CACHED);

	return (tte);
}

/*
 * Fetch a TSB entry with some sanity checking.
 */
int64_t
iommu_tsb_entry(struct iommu_state *is, bus_addr_t dva)
{
	int64_t tte;

	if (dva < is->is_dvmabase || dva > is->is_dvmaend)
		panic("invalid dva: %llx", (long long)dva);

	tte = is->is_tsb[IOTSBSLOT(dva,is->is_tsbsize)];

	if ((tte & IOTTE_V) == 0)
		panic("iommu_tsb_entry: invalid entry %lx", dva);

	return (tte);
}

/*
 * Initiate and then block until an STC flush synchronization has completed.
 */
int
iommu_strbuf_flush_done(struct iommu_map_state *ims)
{
	struct strbuf_ctl *sb = ims->ims_sb;
	struct strbuf_flush *sf = &ims->ims_flush;
	struct timeval cur, flushtimeout;
	struct timeval to = { 0, 500000 };
	u_int64_t flush;
	int timeout_started = 0;

#ifdef DIAGNOSTIC
	if (sb == NULL) {
		panic("iommu_strbuf_flush_done: invalid flush buffer");
	}
#endif

	mtx_enter(&sb->sb_mtx);

	/*
	 * Streaming buffer flushes:
	 *
	 *   1 Tell strbuf to flush by storing va to strbuf_pgflush.
	 *   2 Store 0 in flag
	 *   3 Store pointer to flag in flushsync
	 *   4 wait till flushsync becomes 0x1
	 *
	 * If it takes more than .5 sec, something went very, very wrong.
	 */

	/*
	 * If we're reading from ASI_PHYS_CACHED, then we'll write to
	 * it too.  No need to tempt fate or learn about Si bugs or such.
	 * FreeBSD just uses normal "volatile" reads/writes...
	 */

	stxa(sf->sbf_flushpa, ASI_PHYS_CACHED, 0);

	/*
	 * Insure any previous strbuf operations are complete and that
	 * memory is initialized before the IOMMU uses it.
	 * Is this Needed?  How are IO and memory operations ordered?
	 */
	membar(StoreStore);

	bus_space_write_8(sb->sb_bustag, sb->sb_sb,
		    STRBUFREG(strbuf_flushsync), sf->sbf_flushpa);

	DPRINTF(IDB_IOMMU,
	    ("iommu_strbuf_flush_done: flush = %llx pa = %lx\n",
		ldxa(sf->sbf_flushpa, ASI_PHYS_CACHED), sf->sbf_flushpa));

	membar(StoreLoad | Lookaside);

	for(;;) {
		int i;

		/*
		 * Try to shave a few instruction cycles off the average
		 * latency by only checking the elapsed time every few
		 * fetches.
		 */
		for (i = 0; i < 1000; ++i) {
			membar(LoadLoad);
			/* Bypass non-coherent D$ */
			/* non-coherent...?   Huh? */
			flush = ldxa(sf->sbf_flushpa, ASI_PHYS_CACHED);

			if (flush) {
				DPRINTF(IDB_IOMMU,
				    ("iommu_strbuf_flush_done: flushed\n"));
				mtx_leave(&sb->sb_mtx);
				return (0);
			}
		}

		microtime(&cur);

		if (timeout_started) {
			if (timercmp(&cur, &flushtimeout, >))
				panic("STC timeout at %lx (%lld)",
				    sf->sbf_flushpa, flush);
		} else {
			timeradd(&cur, &to, &flushtimeout);

			timeout_started = 1;

			DPRINTF(IDB_IOMMU,
			    ("iommu_strbuf_flush_done: flush = %llx pa = %lx "
				"now=%lx:%lx until = %lx:%lx\n",
				ldxa(sf->sbf_flushpa, ASI_PHYS_CACHED),
				sf->sbf_flushpa, cur.tv_sec, cur.tv_usec,
				flushtimeout.tv_sec, flushtimeout.tv_usec));
		}
	}
}

/*
 * IOMMU DVMA operations, common to SBus and PCI.
 */

#define BUS_DMA_FIND_PARENT(t, fn)                                      \
        if (t->_parent == NULL)                                         \
                panic("null bus_dma parent (" #fn ")");                 \
        for (t = t->_parent; t->fn == NULL; t = t->_parent)             \
                if (t->_parent == NULL)                                 \
                        panic("no bus_dma " #fn " located");

int
iommu_dvmamap_create(bus_dma_tag_t t, bus_dma_tag_t t0, struct strbuf_ctl *sb,
    bus_size_t size, int nsegments, bus_size_t maxsegsz, bus_size_t boundary,
    int flags, bus_dmamap_t *dmamap)
{
	int ret;
	bus_dmamap_t map;
	struct iommu_map_state *ims;

	BUS_DMA_FIND_PARENT(t, _dmamap_create);
	ret = (*t->_dmamap_create)(t, t0, size, nsegments, maxsegsz, boundary,
	    flags, &map);

	if (ret)
		return (ret);

	ims = iommu_iomap_create(atop(round_page(size)));

	if (ims == NULL) {
		bus_dmamap_destroy(t0, map);
		return (ENOMEM);
	}

	ims->ims_sb = sb;
	map->_dm_cookie = ims;

#ifdef DIAGNOSTIC
	if (ims->ims_sb == NULL)
		panic("iommu_dvmamap_create: null sb");
	if (ims->ims_sb->sb_iommu == NULL)
		panic("iommu_dvmamap_create: null iommu");
#endif
	*dmamap = map;

	return (0);
}

void
iommu_dvmamap_destroy(bus_dma_tag_t t, bus_dma_tag_t t0, bus_dmamap_t map)
{
	/*
	 * The specification (man page) requires a loaded
	 * map to be unloaded before it is destroyed.
	 */
	if (map->dm_nsegs)
		bus_dmamap_unload(t0, map);

        if (map->_dm_cookie)
                iommu_iomap_destroy(map->_dm_cookie);
	map->_dm_cookie = NULL;

	BUS_DMA_FIND_PARENT(t, _dmamap_destroy);
	(*t->_dmamap_destroy)(t, t0, map);
}

/*
 * Load a contiguous kva buffer into a dmamap.  The physical pages are
 * not assumed to be contiguous.  Two passes are made through the buffer
 * and both call pmap_extract() for the same va->pa translations.  It
 * is possible to run out of pa->dvma mappings; the code should be smart
 * enough to resize the iomap (when the "flags" permit allocation).  It
 * is trivial to compute the number of entries required (round the length
 * up to the page size and then divide by the page size)...
 */
int
iommu_dvmamap_load(bus_dma_tag_t t, bus_dma_tag_t t0, bus_dmamap_t map,
    void *buf, bus_size_t buflen, struct proc *p, int flags)
{
	int err = 0;
	bus_size_t sgsize;
	u_long dvmaddr, sgstart, sgend;
	bus_size_t align, boundary;
	struct iommu_state *is;
	struct iommu_map_state *ims = map->_dm_cookie;
	pmap_t pmap;

#ifdef DIAGNOSTIC
	if (ims == NULL)
		panic("iommu_dvmamap_load: null map state");
#endif
#ifdef DEBUG
	if (ims->ims_sb == NULL)
		panic("iommu_dvmamap_load: null sb");
	if (ims->ims_sb->sb_iommu == NULL)
		panic("iommu_dvmamap_load: null iommu");
#endif /* DEBUG */
	is = ims->ims_sb->sb_iommu;

	if (map->dm_nsegs) {
		/*
		 * Is it still in use? _bus_dmamap_load should have taken care
		 * of this.
		 */
#ifdef DIAGNOSTIC
		panic("iommu_dvmamap_load: map still in use");
#endif
		bus_dmamap_unload(t0, map);
	}

	/*
	 * Make sure that on error condition we return "no valid mappings".
	 */
	map->dm_nsegs = 0;

	if (buflen < 1 || buflen > map->_dm_size) {
		DPRINTF(IDB_BUSDMA,
		    ("iommu_dvmamap_load(): error %d > %d -- "
		     "map size exceeded!\n", (int)buflen, (int)map->_dm_size));
		return (EINVAL);
	}

	/*
	 * A boundary presented to bus_dmamem_alloc() takes precedence
	 * over boundary in the map.
	 */
	if ((boundary = (map->dm_segs[0]._ds_boundary)) == 0)
		boundary = map->_dm_boundary;
	align = MAX(map->dm_segs[0]._ds_align, PAGE_SIZE);

	pmap = p ? p->p_vmspace->vm_map.pmap : pmap_kernel();

	/* Count up the total number of pages we need */
	iommu_iomap_clear_pages(ims);
	{ /* Scope */
		bus_addr_t a, aend;
		bus_addr_t addr = (bus_addr_t)buf;
		int seg_len = buflen;

		aend = round_page(addr + seg_len);
		for (a = trunc_page(addr); a < aend; a += PAGE_SIZE) {
			paddr_t pa;

			if (pmap_extract(pmap, a, &pa) == FALSE)
				panic("iomap pmap error addr 0x%lx\n", a);

			err = iommu_iomap_insert_page(ims, pa);
			if (err) {
				printf("iomap insert error: %d for "
				    "va 0x%lx pa 0x%lx "
				    "(buf %p len %ld/%lx)\n",
				    err, a, pa, buf, buflen, buflen);
				iommu_dvmamap_print_map(t, is, map);
				iommu_iomap_clear_pages(ims);
				return (EFBIG);
			}
		}
	}
	sgsize = ims->ims_map.ipm_pagecnt * PAGE_SIZE;

	mtx_enter(&is->is_mtx);
	if (flags & BUS_DMA_24BIT) {
		sgstart = MAX(is->is_dvmamap->ex_start, 0xff000000);
		sgend = MIN(is->is_dvmamap->ex_end, 0xffffffff);
	} else {
		sgstart = is->is_dvmamap->ex_start;
		sgend = is->is_dvmamap->ex_end;
	}

	/*
	 * If our segment size is larger than the boundary we need to
	 * split the transfer up into little pieces ourselves.
	 */
	err = extent_alloc_subregion_with_descr(is->is_dvmamap, sgstart, sgend,
	    sgsize, align, 0, (sgsize > boundary) ? 0 : boundary,
	    EX_NOWAIT | EX_BOUNDZERO, &ims->ims_er, (u_long *)&dvmaddr);
	mtx_leave(&is->is_mtx);

#ifdef DEBUG
	if (err || (dvmaddr == (bus_addr_t)-1))	{
		printf("iommu_dvmamap_load(): extent_alloc(%d, %x) failed!\n",
		    (int)sgsize, flags);
#ifdef DDB
		if (iommudebug & IDB_BREAK)
			Debugger();
#endif
	}
#endif
	if (err != 0) {
		iommu_iomap_clear_pages(ims);
		return (err);
	}

	/* Set the active DVMA map */
	map->_dm_dvmastart = dvmaddr;
	map->_dm_dvmasize = sgsize;

	map->dm_mapsize = buflen;

#ifdef DEBUG
	iommu_dvmamap_validate_map(t, is, map);
#endif

	iommu_iomap_load_map(is, ims, dvmaddr, flags);

	{ /* Scope */
		bus_addr_t a, aend;
		bus_addr_t addr = (bus_addr_t)buf;
		int seg_len = buflen;

		aend = round_page(addr + seg_len);
		for (a = trunc_page(addr); a < aend; a += PAGE_SIZE) {
			bus_addr_t pgstart;
			bus_addr_t pgend;
			paddr_t pa;
			int pglen;

			/* Yuck... Redoing the same pmap_extract... */
			if (pmap_extract(pmap, a, &pa) == FALSE)
				panic("iomap pmap error addr 0x%lx\n", a);

			pgstart = pa | (MAX(a, addr) & PAGE_MASK);
			pgend = pa | (MIN(a + PAGE_SIZE - 1,
			    addr + seg_len - 1) & PAGE_MASK);
			pglen = pgend - pgstart + 1;

			if (pglen < 1)
				continue;

			err = iommu_dvmamap_append_range(t, map, pgstart,
			    pglen, flags, boundary);
			if (err == EFBIG)
				break;
			else if (err) {
				printf("iomap load seg page: %d for "
				    "va 0x%lx pa %lx (%lx - %lx) "
				    "for %d/0x%x\n",
				    err, a, pa, pgstart, pgend, pglen, pglen);
				break;
			}
		}
	}
#ifdef DEBUG
	iommu_dvmamap_validate_map(t, is, map);

	if (err)
		printf("**** iommu_dvmamap_load failed with error %d\n",
		    err);

	if (err || (iommudebug & IDB_PRINT_MAP)) {
		iommu_dvmamap_print_map(t, is, map);
#ifdef DDB
		if (iommudebug & IDB_BREAK)
			Debugger();
#endif
	}
#endif
	if (err)
		iommu_dvmamap_unload(t, t0, map);

	return (err);
}

/*
 * Load a dvmamap from an array of segs or an mlist (if the first
 * "segs" entry's mlist is non-null).  It calls iommu_dvmamap_load_segs()
 * or iommu_dvmamap_load_mlist() for part of the 2nd pass through the
 * mapping.  This is ugly.  A better solution would probably be to have
 * function pointers for implementing the traversal.  That way, there
 * could be one core load routine for each of the three required algorithms
 * (buffer, seg, and mlist).  That would also mean that the traversal
 * algorithm would then only need one implementation for each algorithm
 * instead of two (one for populating the iomap and one for populating
 * the dvma map).
 */
int
iommu_dvmamap_load_raw(bus_dma_tag_t t, bus_dma_tag_t t0, bus_dmamap_t map,
    bus_dma_segment_t *segs, int nsegs, bus_size_t size, int flags)
{
	int i;
	int left;
	int err = 0;
	bus_size_t sgsize;
	bus_size_t boundary, align;
	u_long dvmaddr, sgstart, sgend;
	struct iommu_state *is;
	struct iommu_map_state *ims = map->_dm_cookie;

#ifdef DIAGNOSTIC
	if (ims == NULL)
		panic("iommu_dvmamap_load_raw: null map state");
#endif
#ifdef DEBUG
	if (ims->ims_sb == NULL)
		panic("iommu_dvmamap_load_raw: null sb");
	if (ims->ims_sb->sb_iommu == NULL)
		panic("iommu_dvmamap_load_raw: null iommu");
#endif /* DEBUG */
	is = ims->ims_sb->sb_iommu;

	if (map->dm_nsegs) {
		/* Already in use?? */
#ifdef DIAGNOSTIC
		panic("iommu_dvmamap_load_raw: map still in use");
#endif
		bus_dmamap_unload(t0, map);
	}

	/*
	 * A boundary presented to bus_dmamem_alloc() takes precedence
	 * over boundary in the map.
	 */
	if ((boundary = segs[0]._ds_boundary) == 0)
		boundary = map->_dm_boundary;

	align = MAX(segs[0]._ds_align, PAGE_SIZE);

	/*
	 * Make sure that on error condition we return "no valid mappings".
	 */
	map->dm_nsegs = 0;

	iommu_iomap_clear_pages(ims);
	if (segs[0]._ds_mlist) {
		struct pglist *mlist = segs[0]._ds_mlist;
		struct vm_page *m;
		for (m = TAILQ_FIRST(mlist); m != NULL;
		    m = TAILQ_NEXT(m,pageq)) {
			err = iommu_iomap_insert_page(ims, VM_PAGE_TO_PHYS(m));

			if(err) {
				printf("iomap insert error: %d for "
				    "pa 0x%lx\n", err, VM_PAGE_TO_PHYS(m));
				iommu_dvmamap_print_map(t, is, map);
				iommu_iomap_clear_pages(ims);
				return (EFBIG);
			}
		}
	} else {
		/* Count up the total number of pages we need */
		for (i = 0, left = size; left > 0 && i < nsegs; i++) {
			bus_addr_t a, aend;
			bus_size_t len = segs[i].ds_len;
			bus_addr_t addr = segs[i].ds_addr;
			int seg_len = MIN(left, len);

			if (len < 1)
				continue;

			aend = round_page(addr + seg_len);
			for (a = trunc_page(addr); a < aend; a += PAGE_SIZE) {

				err = iommu_iomap_insert_page(ims, a);
				if (err) {
					printf("iomap insert error: %d for "
					    "pa 0x%lx\n", err, a);
					iommu_dvmamap_print_map(t, is, map);
					iommu_iomap_clear_pages(ims);
					return (EFBIG);
				}
			}

			left -= seg_len;
		}
	}
	sgsize = ims->ims_map.ipm_pagecnt * PAGE_SIZE;

	mtx_enter(&is->is_mtx);
	if (flags & BUS_DMA_24BIT) {
		sgstart = MAX(is->is_dvmamap->ex_start, 0xff000000);
		sgend = MIN(is->is_dvmamap->ex_end, 0xffffffff);
	} else {
		sgstart = is->is_dvmamap->ex_start;
		sgend = is->is_dvmamap->ex_end;
	}

	/*
	 * If our segment size is larger than the boundary we need to
	 * split the transfer up into little pieces ourselves.
	 */
	err = extent_alloc_subregion_with_descr(is->is_dvmamap, sgstart, sgend,
	    sgsize, align, 0, (sgsize > boundary) ? 0 : boundary,
	    EX_NOWAIT | EX_BOUNDZERO, &ims->ims_er, (u_long *)&dvmaddr);
	mtx_leave(&is->is_mtx);

	if (err != 0) {
		iommu_iomap_clear_pages(ims);
		return (err);
	}

#ifdef DEBUG
	if (dvmaddr == (bus_addr_t)-1)	{
		printf("iommu_dvmamap_load_raw(): extent_alloc(%d, %x) "
		    "failed!\n", (int)sgsize, flags);
#ifdef DDB
		if (iommudebug & IDB_BREAK)
			Debugger();
#else
		panic("");
#endif
	}
#endif

	/* Set the active DVMA map */
	map->_dm_dvmastart = dvmaddr;
	map->_dm_dvmasize = sgsize;

	map->dm_mapsize = size;

#ifdef DEBUG
	iommu_dvmamap_validate_map(t, is, map);
#endif

	iommu_iomap_load_map(is, ims, dvmaddr, flags);

	if (segs[0]._ds_mlist)
		err = iommu_dvmamap_load_mlist(t, is, map, segs[0]._ds_mlist,
		    flags, size, boundary);
	else
		err = iommu_dvmamap_load_seg(t, is, map, segs, nsegs,
		    flags, size, boundary);

#ifdef DEBUG
	/* The map should be valid even if the load failed */
	if (iommu_dvmamap_validate_map(t, is, map)) {
		printf("load size %ld/0x%lx\n", size, size);
		if (segs[0]._ds_mlist)
			printf("mlist %p\n", segs[0]._ds_mlist);
		else  {
			long tot_len = 0;
			long clip_len = 0;
			printf("segs %p nsegs %d\n", segs, nsegs);

			left = size;
			for(i = 0; i < nsegs; i++) {
				bus_size_t len = segs[i].ds_len;
				bus_addr_t addr = segs[i].ds_addr;
				int seg_len = MIN(left, len);

				printf("addr %lx len %ld/0x%lx seg_len "
				    "%ld/0x%lx left %ld/0xl%x\n", addr,
				    len, len, seg_len, seg_len, left, left);

				left -= seg_len;

				clip_len += seg_len;
				tot_len += segs[i].ds_len;
			}
			printf("total length %ld/0x%lx total seg. "
			    "length %ld/0x%lx\n", tot_len, tot_len, clip_len,
			    clip_len);
		}

		if (err == 0)
			err = 1;
	}

	if (err)
		printf("**** iommu_dvmamap_load_raw failed with error %d\n",
		    err);

	if (err || (iommudebug & IDB_PRINT_MAP)) {
		iommu_dvmamap_print_map(t, is, map);
#ifdef DDB
		if (iommudebug & IDB_BREAK)
			Debugger();
#endif
	}
#endif
	if (err)
		iommu_dvmamap_unload(t, t0, map);

	return (err);
}

/*
 * Insert a range of addresses into a loaded map respecting the specified
 * boundary and alignment restrictions.  The range is specified by its
 * physical address and length.  The range cannot cross a page boundary.
 * This code (along with most of the rest of the function in this file)
 * assumes that the IOMMU page size is equal to PAGE_SIZE.
 */
int
iommu_dvmamap_append_range(bus_dma_tag_t t, bus_dmamap_t map, paddr_t pa,
    bus_size_t length, int flags, bus_size_t boundary)
{
	struct iommu_map_state *ims = map->_dm_cookie;
	bus_addr_t sgstart, sgend, bd_mask;
	bus_dma_segment_t *seg = NULL;
	int i = map->dm_nsegs;

#ifdef DEBUG
	if (ims == NULL)
		panic("iommu_dvmamap_append_range: null map state");
#endif

	sgstart = iommu_iomap_translate(ims, pa);
	sgend = sgstart + length - 1;

#ifdef DIAGNOSTIC
	if (sgstart == 0 || sgstart > sgend) {
		printf("append range invalid mapping for %lx "
		    "(0x%lx - 0x%lx)\n", pa, sgstart, sgend);
		map->dm_nsegs = 0;
		return (EINVAL);
	}
#endif

#ifdef DEBUG
	if (trunc_page(sgstart) != trunc_page(sgend)) {
		printf("append range crossing page boundary! "
		    "pa %lx length %ld/0x%lx sgstart %lx sgend %lx\n",
		    pa, length, length, sgstart, sgend);
	}
#endif

	/*
	 * We will attempt to merge this range with the previous entry
	 * (if there is one).
	 */
	if (i > 0) {
		seg = &map->dm_segs[i - 1];
		if (sgstart == seg->ds_addr + seg->ds_len) {
			length += seg->ds_len;
			sgstart = seg->ds_addr;
			sgend = sgstart + length - 1;
		} else
			seg = NULL;
	}

	if (seg == NULL) {
		seg = &map->dm_segs[i];
		if (++i > map->_dm_segcnt) {
			map->dm_nsegs = 0;
			return (EFBIG);
		}
	}

	/*
	 * At this point, "i" is the index of the *next* bus_dma_segment_t
	 * (the segment count, aka map->dm_nsegs) and "seg" points to the
	 * *current* entry.  "length", "sgstart", and "sgend" reflect what
	 * we intend to put in "*seg".  No assumptions should be made about
	 * the contents of "*seg".  Only "boundary" issue can change this
	 * and "boundary" is often zero, so explicitly test for that case
	 * (the test is strictly an optimization).
	 */
	if (boundary != 0) {
		bd_mask = ~(boundary - 1);

		while ((sgstart & bd_mask) != (sgend & bd_mask)) {
			/*
			 * We are crossing a boundary so fill in the current
			 * segment with as much as possible, then grab a new
			 * one.
			 */

			seg->ds_addr = sgstart;
			seg->ds_len = boundary - (sgstart & ~bd_mask);

			sgstart += seg->ds_len; /* sgend stays the same */
			length -= seg->ds_len;

			seg = &map->dm_segs[i];
			if (++i > map->_dm_segcnt) {
				map->dm_nsegs = 0;
				return (EFBIG);
			}
		}
	}

	seg->ds_addr = sgstart;
	seg->ds_len = length;
	map->dm_nsegs = i;

	return (0);
}

/*
 * Populate the iomap from a bus_dma_segment_t array.  See note for
 * iommu_dvmamap_load() * regarding page entry exhaustion of the iomap.
 * This is less of a problem for load_seg, as the number of pages
 * is usually similar to the number of segments (nsegs).
 */
int
iommu_dvmamap_load_seg(bus_dma_tag_t t, struct iommu_state *is,
    bus_dmamap_t map, bus_dma_segment_t *segs, int nsegs, int flags,
    bus_size_t size, bus_size_t boundary)
{
	int i;
	int left;
	int seg;

	/*
	 * This segs is made up of individual physical
	 * segments, probably by _bus_dmamap_load_uio() or
	 * _bus_dmamap_load_mbuf().  Ignore the mlist and
	 * load each one individually.
	 */

	/*
	 * Keep in mind that each segment could span
	 * multiple pages and that these are not always
	 * adjacent. The code is no longer adding dvma
	 * aliases to the IOMMU.  The STC will not cross
	 * page boundaries anyway and a IOMMU table walk
	 * vs. what may be a streamed PCI DMA to a ring
	 * descriptor is probably a wash.  It eases TLB
	 * pressure and in the worst possible case, it is
	 * only as bad a non-IOMMUed architecture.  More
	 * importantly, the code is not quite as hairy.
	 * (It's bad enough as it is.)
	 */
	left = size;
	seg = 0;
	for (i = 0; left > 0 && i < nsegs; i++) {
		bus_addr_t a, aend;
		bus_size_t len = segs[i].ds_len;
		bus_addr_t addr = segs[i].ds_addr;
		int seg_len = MIN(left, len);

		if (len < 1)
			continue;

		aend = round_page(addr + seg_len);
		for (a = trunc_page(addr); a < aend; a += PAGE_SIZE) {
			bus_addr_t pgstart;
			bus_addr_t pgend;
			int pglen;
			int err;

			pgstart = MAX(a, addr);
			pgend = MIN(a + PAGE_SIZE - 1, addr + seg_len - 1);
			pglen = pgend - pgstart + 1;

			if (pglen < 1)
				continue;

			err = iommu_dvmamap_append_range(t, map, pgstart,
			    pglen, flags, boundary);
			if (err == EFBIG)
				return (err);
			if (err) {
				printf("iomap load seg page: %d for "
				    "pa 0x%lx (%lx - %lx for %d/%x\n",
				    err, a, pgstart, pgend, pglen, pglen);
				return (err);
			}

		}

		left -= seg_len;
	}
	return (0);
}

/*
 * Populate the iomap from an mlist.  See note for iommu_dvmamap_load()
 * regarding page entry exhaustion of the iomap.
 */
int
iommu_dvmamap_load_mlist(bus_dma_tag_t t, struct iommu_state *is,
    bus_dmamap_t map, struct pglist *mlist, int flags,
    bus_size_t size, bus_size_t boundary)
{
	struct vm_page *m;
	paddr_t pa;
	int err;

	/*
	 * This was allocated with bus_dmamem_alloc.
	 * The pages are on an `mlist'.
	 */
	for (m = TAILQ_FIRST(mlist); m != NULL; m = TAILQ_NEXT(m,pageq)) {
		pa = VM_PAGE_TO_PHYS(m);

		err = iommu_dvmamap_append_range(t, map, pa,
		    MIN(PAGE_SIZE, size), flags, boundary);
		if (err == EFBIG)
			return (err);
		if (err) {
			printf("iomap load seg page: %d for pa 0x%lx "
			    "(%lx - %lx for %d/%x\n", err, pa, pa,
			    pa + PAGE_SIZE, PAGE_SIZE, PAGE_SIZE);
			return (err);
		}
		if (size < PAGE_SIZE)
			break;
		size -= PAGE_SIZE;
	}

	return (0);
}

/*
 * Unload a dvmamap.
 */
void
iommu_dvmamap_unload(bus_dma_tag_t t, bus_dma_tag_t t0, bus_dmamap_t map)
{
	struct iommu_state *is;
	struct iommu_map_state *ims = map->_dm_cookie;
	bus_addr_t dvmaddr = map->_dm_dvmastart;
	bus_size_t sgsize = map->_dm_dvmasize;
	int error;

#ifdef DEBUG
	if (ims == NULL)
		panic("iommu_dvmamap_unload: null map state");
	if (ims->ims_sb == NULL)
		panic("iommu_dvmamap_unload: null sb");
	if (ims->ims_sb->sb_iommu == NULL)
		panic("iommu_dvmamap_unload: null iommu");
#endif /* DEBUG */

	is = ims->ims_sb->sb_iommu;

	/* Flush the iommu */
#ifdef DEBUG
	if (dvmaddr == 0) {
		printf("iommu_dvmamap_unload: No dvmastart\n");
#ifdef DDB
		if (iommudebug & IDB_BREAK)
			Debugger();
#endif
		return;
	}

	iommu_dvmamap_validate_map(t, is, map);

	if (iommudebug & IDB_PRINT_MAP)
		iommu_dvmamap_print_map(t, is, map);
#endif /* DEBUG */

	/* Remove the IOMMU entries */
	iommu_iomap_unload_map(is, ims);

	/* Clear the iomap */
	iommu_iomap_clear_pages(ims);

	bus_dmamap_unload(t->_parent, map);

	/* Mark the mappings as invalid. */
	map->dm_mapsize = 0;
	map->dm_nsegs = 0;

	mtx_enter(&is->is_mtx);
	error = extent_free(is->is_dvmamap, dvmaddr, sgsize, EX_NOWAIT);
	map->_dm_dvmastart = 0;
	map->_dm_dvmasize = 0;
	mtx_leave(&is->is_mtx);
	if (error != 0)
		printf("warning: %ld of DVMA space lost\n", sgsize);
}

#ifdef DEBUG
/*
 * Perform internal consistency checking on a dvmamap.
 */
int
iommu_dvmamap_validate_map(bus_dma_tag_t t, struct iommu_state *is,
    bus_dmamap_t map)
{
	int err = 0;
	int seg;

	if (trunc_page(map->_dm_dvmastart) != map->_dm_dvmastart) {
		printf("**** dvmastart address not page aligned: %llx",
			map->_dm_dvmastart);
		err = 1;
	}
	if (trunc_page(map->_dm_dvmasize) != map->_dm_dvmasize) {
		printf("**** dvmasize not a multiple of page size: %llx",
			map->_dm_dvmasize);
		err = 1;
	}
	if (map->_dm_dvmastart < is->is_dvmabase ||
	    (round_page(map->_dm_dvmastart + map->_dm_dvmasize) - 1) >
	    is->is_dvmaend) {
		printf("dvmaddr %llx len %llx out of range %x - %x\n",
			    map->_dm_dvmastart, map->_dm_dvmasize,
			    is->is_dvmabase, is->is_dvmaend);
		err = 1;
	}
	for (seg = 0; seg < map->dm_nsegs; seg++) {
		if (map->dm_segs[seg].ds_addr == 0 ||
		    map->dm_segs[seg].ds_len == 0) {
			printf("seg %d null segment dvmaddr %llx len %llx for "
			    "range %llx len %llx\n",
			    seg,
			    map->dm_segs[seg].ds_addr,
			    map->dm_segs[seg].ds_len,
			    map->_dm_dvmastart, map->_dm_dvmasize);
			err = 1;
		} else if (map->dm_segs[seg].ds_addr < map->_dm_dvmastart ||
		    round_page(map->dm_segs[seg].ds_addr +
			map->dm_segs[seg].ds_len) >
		    map->_dm_dvmastart + map->_dm_dvmasize) {
			printf("seg %d dvmaddr %llx len %llx out of "
			    "range %llx len %llx\n",
			    seg,
			    map->dm_segs[seg].ds_addr,
			    map->dm_segs[seg].ds_len,
			    map->_dm_dvmastart, map->_dm_dvmasize);
			err = 1;
		}
	}

	if (err) {
		iommu_dvmamap_print_map(t, is, map);
#if defined(DDB) && defined(DEBUG)
		if (iommudebug & IDB_BREAK)
			Debugger();
#endif
	}

	return (err);
}
#endif /* DEBUG */

void
iommu_dvmamap_print_map(bus_dma_tag_t t, struct iommu_state *is,
    bus_dmamap_t map)
{
	int seg, i;
	long full_len, source_len;
	struct mbuf *m;

	printf("DVMA %x for %x, mapping %p: dvstart %lx dvsize %lx "
	    "size %ld/%lx maxsegsz %lx boundary %lx segcnt %d "
	    "flags %x type %d source %p "
	    "cookie %p mapsize %lx nsegs %d\n",
	    is ? is->is_dvmabase : 0, is ? is->is_dvmaend : 0, map,
	    map->_dm_dvmastart, map->_dm_dvmasize,
	    map->_dm_size, map->_dm_size, map->_dm_maxsegsz, map->_dm_boundary,
	    map->_dm_segcnt, map->_dm_flags, map->_dm_type,
	    map->_dm_source, map->_dm_cookie, map->dm_mapsize,
	    map->dm_nsegs);

	full_len = 0;
	for (seg = 0; seg < map->dm_nsegs; seg++) {
		printf("seg %d dvmaddr %lx pa %lx len %lx (tte %llx)\n",
		    seg, map->dm_segs[seg].ds_addr,
		    is ? iommu_extract(is, map->dm_segs[seg].ds_addr) : 0,
		    map->dm_segs[seg].ds_len,
		    is ? iommu_lookup_tte(is, map->dm_segs[seg].ds_addr) : 0);
		full_len += map->dm_segs[seg].ds_len;
	}
	printf("total length = %ld/0x%lx\n", full_len, full_len);

	if (map->_dm_source) switch (map->_dm_type) {
	case _DM_TYPE_MBUF:
		m = map->_dm_source;
		if (m->m_flags & M_PKTHDR)
			printf("source PKTHDR mbuf (%p) hdr len = %d/0x%x:\n",
			    m, m->m_pkthdr.len, m->m_pkthdr.len);
		else
			printf("source mbuf (%p):\n", m);

		source_len = 0;
		for ( ; m; m = m->m_next) {
			vaddr_t vaddr = mtod(m, vaddr_t);
			long len = m->m_len;
			paddr_t pa;

			if (pmap_extract(pmap_kernel(), vaddr, &pa))
				printf("kva %lx pa %lx len %ld/0x%lx\n",
				    vaddr, pa, len, len);
			else
				printf("kva %lx pa <invalid> len %ld/0x%lx\n",
				    vaddr, len, len);

			source_len += len;
		}

		if (full_len != source_len)
			printf("mbuf length %ld/0x%lx is %s than mapping "
			    "length %ld/0x%lx\n", source_len, source_len,
			    (source_len > full_len) ? "greater" : "less",
			    full_len, full_len);
		else
			printf("mbuf length %ld/0x%lx\n", source_len,
			    source_len);
		break;
	case _DM_TYPE_LOAD:
	case _DM_TYPE_SEGS:
	case _DM_TYPE_UIO:
	default:
		break;
	}

	if (map->_dm_cookie) {
		struct iommu_map_state *ims = map->_dm_cookie;
		struct iommu_page_map *ipm = &ims->ims_map;

		printf("page map (%p) of size %d with %d entries\n",
		    ipm, ipm->ipm_maxpage, ipm->ipm_pagecnt);
		for (i = 0; i < ipm->ipm_pagecnt; ++i) {
			struct iommu_page_entry *e = &ipm->ipm_map[i];
			printf("%d: vmaddr 0x%lx pa 0x%lx\n", i,
			    e->ipe_va, e->ipe_pa);
		}
	} else
		printf("iommu map state (cookie) is NULL\n");
}

void
_iommu_dvmamap_sync(bus_dma_tag_t t, bus_dma_tag_t t0, bus_dmamap_t map,
	bus_addr_t offset, bus_size_t len, int ops)
{
	struct iommu_state *is;
	struct iommu_map_state *ims = map->_dm_cookie;
	struct strbuf_ctl *sb;
	bus_size_t count;
	int i, needsflush = 0;

	sb = ims->ims_sb;
	is = sb->sb_iommu;

	for (i = 0; i < map->dm_nsegs; i++) {
		if (offset < map->dm_segs[i].ds_len)
			break;
		offset -= map->dm_segs[i].ds_len;
	}

	if (i == map->dm_nsegs)
		panic("iommu_dvmamap_sync: too short %lu", offset);

	for (; len > 0 && i < map->dm_nsegs; i++) {
		count = MIN(map->dm_segs[i].ds_len - offset, len);
		if (count > 0 && iommu_dvmamap_sync_range(sb,
		    map->dm_segs[i].ds_addr + offset, count))
			needsflush = 1;
		len -= count;
	}

#ifdef DIAGNOSTIC
	if (i == map->dm_nsegs && len > 0)
		panic("iommu_dvmamap_sync: leftover %lu", len);
#endif

	if (needsflush)
		iommu_strbuf_flush_done(ims);
}

void
iommu_dvmamap_sync(bus_dma_tag_t t, bus_dma_tag_t t0, bus_dmamap_t map,
    bus_addr_t offset, bus_size_t len, int ops)
{
	struct iommu_map_state *ims = map->_dm_cookie;

#ifdef DIAGNOSTIC
	if (ims == NULL)
		panic("iommu_dvmamap_sync: null map state");
	if (ims->ims_sb == NULL)
		panic("iommu_dvmamap_sync: null sb");
	if (ims->ims_sb->sb_iommu == NULL)
		panic("iommu_dvmamap_sync: null iommu");
#endif
	if (len == 0)
		return;

	if (ops & BUS_DMASYNC_PREWRITE)
		membar(MemIssue);

	if ((ims->ims_flags & IOMMU_MAP_STREAM) &&
	    (ops & (BUS_DMASYNC_POSTREAD | BUS_DMASYNC_PREWRITE)))
		_iommu_dvmamap_sync(t, t0, map, offset, len, ops);

	if (ops & BUS_DMASYNC_POSTREAD)
		membar(MemIssue);
}

/*
 * Flush an individual dma segment, returns non-zero if the streaming buffers
 * need flushing afterwards.
 */
int
iommu_dvmamap_sync_range(struct strbuf_ctl *sb, bus_addr_t va, bus_size_t len)
{
	bus_addr_t vaend;
#ifdef DIAGNOSTIC
	struct iommu_state *is = sb->sb_iommu;

	if (va < is->is_dvmabase || va > is->is_dvmaend)
		panic("invalid va: %llx", (long long)va);

	if ((is->is_tsb[IOTSBSLOT(va, is->is_tsbsize)] & IOTTE_STREAM) == 0) {
		printf("iommu_dvmamap_sync_range: attempting to flush "
		    "non-streaming entry\n");
		return (0);
	}
#endif

	vaend = (va + len + PAGE_MASK) & ~PAGE_MASK;
	va &= ~PAGE_MASK;

#ifdef DIAGNOSTIC
	if (va < is->is_dvmabase || (vaend - 1) > is->is_dvmaend)
		panic("invalid va range: %llx to %llx (%x to %x)",
		    (long long)va, (long long)vaend,
		    is->is_dvmabase,
		    is->is_dvmaend);
#endif

	for ( ; va <= vaend; va += PAGE_SIZE) {
		DPRINTF(IDB_BUSDMA,
		    ("iommu_dvmamap_sync_range: flushing va %p\n",
		    (void *)(u_long)va));
		iommu_strbuf_flush(sb, va);
	}

	return (1);
}

int
iommu_dvmamem_alloc(bus_dma_tag_t t, bus_dma_tag_t t0, bus_size_t size,
    bus_size_t alignment, bus_size_t boundary, bus_dma_segment_t *segs,
    int nsegs, int *rsegs, int flags)
{

	DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_alloc: sz %llx align %llx "
	    "bound %llx segp %p flags %d\n", (unsigned long long)size,
	    (unsigned long long)alignment, (unsigned long long)boundary,
	    segs, flags));
	BUS_DMA_FIND_PARENT(t, _dmamem_alloc);
	return ((*t->_dmamem_alloc)(t, t0, size, alignment, boundary,
	    segs, nsegs, rsegs, flags | BUS_DMA_DVMA));
}

void
iommu_dvmamem_free(bus_dma_tag_t t, bus_dma_tag_t t0, bus_dma_segment_t *segs,
    int nsegs)
{

	DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_free: segp %p nsegs %d\n",
	    segs, nsegs));
	BUS_DMA_FIND_PARENT(t, _dmamem_free);
	(*t->_dmamem_free)(t, t0, segs, nsegs);
}

/*
 * Create a new iomap.
 */
struct iommu_map_state *
iommu_iomap_create(int n)
{
	struct iommu_map_state *ims;
	struct strbuf_flush *sbf;
	vaddr_t va;

	/* Safety for heavily fragmented data, such as mbufs */
	n += 4;
	if (n < 16)
		n = 16;

	ims = malloc(sizeof(*ims) + (n - 1) * sizeof(ims->ims_map.ipm_map[0]),
		M_DEVBUF, M_NOWAIT | M_ZERO);
	if (ims == NULL)
		return (NULL);

	/* Initialize the map. */
	ims->ims_map.ipm_maxpage = n;
	SPLAY_INIT(&ims->ims_map.ipm_tree);

	/* Initialize the flush area. */
	sbf = &ims->ims_flush;
	va = (vaddr_t)&sbf->sbf_area[0x40];
	va &= ~0x3f;
	pmap_extract(pmap_kernel(), va, &sbf->sbf_flushpa);
	sbf->sbf_flush = (void *)va;

	return (ims);
}

/*
 * Destroy an iomap.
 */
void
iommu_iomap_destroy(struct iommu_map_state *ims)
{
#ifdef DIAGNOSTIC
	if (ims->ims_map.ipm_pagecnt > 0)
		printf("iommu_iomap_destroy: %d page entries in use\n",
		    ims->ims_map.ipm_pagecnt);
#endif

	free(ims, M_DEVBUF, 0);
}

/*
 * Utility function used by splay tree to order page entries by pa.
 */
static inline int
iomap_compare(struct iommu_page_entry *a, struct iommu_page_entry *b)
{
	return ((a->ipe_pa > b->ipe_pa) ? 1 :
		(a->ipe_pa < b->ipe_pa) ? -1 : 0);
}

SPLAY_PROTOTYPE(iommu_page_tree, iommu_page_entry, ipe_node, iomap_compare);

SPLAY_GENERATE(iommu_page_tree, iommu_page_entry, ipe_node, iomap_compare);

/*
 * Insert a pa entry in the iomap.
 */
int
iommu_iomap_insert_page(struct iommu_map_state *ims, paddr_t pa)
{
	struct iommu_page_map *ipm = &ims->ims_map;
	struct iommu_page_entry *e;

	if (ipm->ipm_pagecnt >= ipm->ipm_maxpage) {
		struct iommu_page_entry ipe;

		ipe.ipe_pa = pa;
		if (SPLAY_FIND(iommu_page_tree, &ipm->ipm_tree, &ipe))
			return (0);

		return (ENOMEM);
	}

	e = &ipm->ipm_map[ipm->ipm_pagecnt];

	e->ipe_pa = pa;
	e->ipe_va = 0;

	e = SPLAY_INSERT(iommu_page_tree, &ipm->ipm_tree, e);

	/* Duplicates are okay, but only count them once. */
	if (e)
		return (0);

	++ipm->ipm_pagecnt;

	return (0);
}

/*
 * Locate the iomap by filling in the pa->va mapping and inserting it
 * into the IOMMU tables.
 */
void
iommu_iomap_load_map(struct iommu_state *is, struct iommu_map_state *ims,
    bus_addr_t vmaddr, int flags)
{
	struct iommu_page_map *ipm = &ims->ims_map;
	struct iommu_page_entry *e;
	struct strbuf_ctl *sb = ims->ims_sb;
	int i, slot;

	if (sb->sb_flush == NULL)
		flags &= ~BUS_DMA_STREAMING;

	if (flags & BUS_DMA_STREAMING)
		ims->ims_flags |= IOMMU_MAP_STREAM;
	else
		ims->ims_flags &= ~IOMMU_MAP_STREAM;

	for (i = 0, e = ipm->ipm_map; i < ipm->ipm_pagecnt; ++i, ++e) {
		e->ipe_va = vmaddr;
		iommu_enter(is, sb, e->ipe_va, e->ipe_pa, flags);

		/* Flush cache if necessary. */
		slot = IOTSBSLOT(e->ipe_va, is->is_tsbsize);
		if (is->is_flags & IOMMU_FLUSH_CACHE &&
		    (i == (ipm->ipm_pagecnt - 1) || (slot % 8) == 7))
			IOMMUREG_WRITE(is, iommu_cache_flush,
			    is->is_ptsb + slot * 8);

		vmaddr += PAGE_SIZE;
	}
}

/*
 * Remove the iomap from the IOMMU.
 */
void
iommu_iomap_unload_map(struct iommu_state *is, struct iommu_map_state *ims)
{
	struct iommu_page_map *ipm = &ims->ims_map;
	struct iommu_page_entry *e;
	struct strbuf_ctl *sb = ims->ims_sb;
	int i, slot;

	for (i = 0, e = ipm->ipm_map; i < ipm->ipm_pagecnt; ++i, ++e) {
		iommu_remove(is, sb, e->ipe_va);

		/* Flush cache if necessary. */
		slot = IOTSBSLOT(e->ipe_va, is->is_tsbsize);
		if (is->is_flags & IOMMU_FLUSH_CACHE &&
		    (i == (ipm->ipm_pagecnt - 1) || (slot % 8) == 7))
			IOMMUREG_WRITE(is, iommu_cache_flush,
			    is->is_ptsb + slot * 8);
	}
}

/*
 * Translate a physical address (pa) into a DVMA address.
 */
bus_addr_t
iommu_iomap_translate(struct iommu_map_state *ims, paddr_t pa)
{
	struct iommu_page_map *ipm = &ims->ims_map;
	struct iommu_page_entry *e;
	struct iommu_page_entry pe;
	paddr_t offset = pa & PAGE_MASK;

	pe.ipe_pa = trunc_page(pa);

	e = SPLAY_FIND(iommu_page_tree, &ipm->ipm_tree, &pe);

	if (e == NULL)
		return (0);

	return (e->ipe_va | offset);
}

/*
 * Clear the iomap table and tree.
 */
void
iommu_iomap_clear_pages(struct iommu_map_state *ims)
{
	ims->ims_map.ipm_pagecnt = 0;
	SPLAY_INIT(&ims->ims_map.ipm_tree);
}


code:
/*	$OpenBSD: iommureg.h,v 1.17 2012/08/17 20:46:50 kettenis Exp $	*/
/*	$NetBSD: iommureg.h,v 1.6 2001/07/20 00:07:13 eeh Exp $	*/

/*
 * Copyright (c) 1992, 1993
 *	The Regents of the University of California.  All rights reserved.
 *
 * This software was developed by the Computer Systems Engineering group
 * at Lawrence Berkeley Laboratory under DARPA contract BG 91-66 and
 * contributed to Berkeley.
 *
 * All advertising materials mentioning features or use of this software
 * must display the following acknowledgement:
 *	This product includes software developed by the University of
 *	California, Lawrence Berkeley Laboratory.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 *	@(#)sbusreg.h	8.1 (Berkeley) 6/11/93
 */

#ifndef _SPARC64_DEV_IOMMUREG_H_
#define _SPARC64_DEV_IOMMUREG_H_

/*
 * UltraSPARC IOMMU registers, common to both the sbus and PCI
 * controllers.
 */

/* iommmu registers */
struct iommureg {
	volatile u_int64_t	iommu_cr;	/* IOMMU control register */
	volatile u_int64_t	iommu_tsb;	/* IOMMU TSB base register */
	volatile u_int64_t	iommu_flush;	/* IOMMU flush register */
	volatile u_int64_t	iommu_ctxflush;
	volatile u_int64_t	iommu_reserved[28];
	volatile u_int64_t	iommu_cache_flush;
	volatile u_int64_t	iommu_cache_invalidate;
	volatile u_int64_t	iommu_reserved2[30];
};

/* streaming buffer registers */
struct iommu_strbuf {
	volatile u_int64_t	strbuf_ctl;	/* streaming buffer control reg */
	volatile u_int64_t	strbuf_pgflush;	/* streaming buffer page flush */
	volatile u_int64_t	strbuf_flushsync;/* streaming buffer flush sync */
};

#define IOMMUREG(x)     (offsetof(struct iommureg, x))
#define STRBUFREG(x)    (offsetof(struct iommu_strbuf, x))

/* streaming buffer control register */
#define STRBUF_EN		0x000000000000000001LL
#define STRBUF_D		0x000000000000000002LL

/* control register bits */
#define IOMMUCR_TSB1K		0x000000000000000000LL	/* Nummber of entries in IOTSB */
#define IOMMUCR_TSB2K		0x000000000000010000LL
#define IOMMUCR_TSB4K		0x000000000000020000LL
#define IOMMUCR_TSB8K		0x000000000000030000LL
#define IOMMUCR_TSB16K		0x000000000000040000LL
#define IOMMUCR_TSB32K		0x000000000000050000LL
#define IOMMUCR_TSB64K		0x000000000000060000LL
#define IOMMUCR_TSB128K		0x000000000000070000LL
#define IOMMUCR_TSBMASK		0xfffffffffffff8ffffLL	/* Mask for above */
#define IOMMUCR_8KPG		0x000000000000000000LL	/* 8K iommu page size */
#define IOMMUCR_64KPG		0x000000000000000004LL	/* 64K iommu page size */
#define IOMMUCR_DE		0x000000000000000002LL	/* Diag enable */
#define IOMMUCR_EN		0x000000000000000001LL	/* Enable IOMMU */

#define IOMMUCR_FIRE_SE		0x000000000000000400LL	/* Snoop enable */
#define IOMMUCR_FIRE_CM_EN	0x000000000000000300LL  /* Cache mode enable */
#define IOMMUCR_FIRE_BE		0x000000000000000002LL	/* Bypass enable */
#define IOMMUCR_FIRE_TE		0x000000000000000001LL	/* Translation enabled */

/*
 * IOMMU stuff
 */
#define	IOTTE_V		0x8000000000000000LL	/* Entry valid */
#define IOTTE_64K	0x2000000000000000LL	/* 8K or 64K page? */
#define IOTTE_8K	0x0000000000000000LL
#define IOTTE_STREAM	0x1000000000000000LL	/* Is page streamable? */
#define	IOTTE_LOCAL	0x0800000000000000LL	/* Accesses to same bus segment? */
#define	IOTTE_CONTEXT	0x07ff800000000000LL	/* context number */
#define IOTTE_PAMASK	0x00007fffffffe000LL	/* Let's assume this is correct (bits 42..13) */
#define IOTTE_C		0x0000000000000010LL	/* Accesses to cacheable space */
#define IOTTE_W		0x0000000000000002LL	/* Writeable */
#define IOTTE_SOFTWARE	0x0000000000001f80LL	/* For software use (bits 12..7) */

/*
 * On sun4u each bus controller has a separate IOMMU.  The IOMMU has
 * a TSB which must be page aligned and physically contiguous.  Mappings
 * can be of 8K IOMMU pages or 64K IOMMU pages.  We use 8K for compatibility
 * with the CPU's MMU.
 *
 * On sysio, psycho, and psycho+, IOMMU TSBs using 8K pages can map the
 * following size segments:
 *
 *	VA size		VA base		TSB size	tsbsize
 *	--------	--------	---------	-------
 *	8MB		ff800000	8K		0
 *	16MB		ff000000	16K		1
 *	32MB		fe000000	32K		2
 *	64MB		fc000000	64K		3
 *	128MB		f8000000	128K		4
 *	256MB		f0000000	256K		5
 *	512MB		e0000000	512K		6
 *	1GB		c0000000	1MB		7
 *
 * Unfortunately, sabres on UltraSPARC IIi and IIe processors does not use
 * this scheme to determine the IOVA base address.  Instead, bits 31-29 are
 * used to check against the Target Address Space register in the IIi and
 * the IOMMU is used if they hit.  God knows what goes on in the IIe.
 *
 */

#define IOTSB_VEND		0xffffffffU
#define IOTSB_VSTART(sz)	(u_int)(IOTSB_VEND << ((sz)+10+PGSHIFT))
#define IOTSB_VSIZE(sz)		(u_int)(1 << ((sz)+10+PGSHIFT))

#define MAKEIOTTE(pa,w,c,s)	(((pa)&IOTTE_PAMASK)|((w)?IOTTE_W:0)|((c)?IOTTE_C:0)|((s)?IOTTE_STREAM:0)|(IOTTE_V|IOTTE_8K))
#define IOTSBSLOT(va,sz)	((u_int)(((vaddr_t)(va))-(is->is_dvmabase))>>PGSHIFT)

/*
 * interrupt map stuff.  this belongs elsewhere.
 */

#define INTMAP_V	0x080000000LL	/* Interrupt valid (enabled) */
#define INTMAP_TID	0x07c000000LL	/* UPA target ID mask */
#define INTMAP_IGN	0x0000007c0LL	/* Interrupt group no (sbus only). */
#define INTMAP_IGN_SHIFT	6
#define INTMAP_INO	0x00000003fLL	/* Interrupt number */
#define INTMAP_INR	(INTMAP_IGN|INTMAP_INO)
#define INTMAP_SBUSSLOT	0x000000018LL	/* SBus slot # */
#define INTMAP_PCIBUS	0x000000010LL	/* PCI bus number (A or B) */
#define INTMAP_PCISLOT	0x00000000cLL	/* PCI slot # */
#define INTMAP_PCIINT	0x000000003LL	/* PCI interrupt #A,#B,#C,#D */
#define INTMAP_OBIO	0x000000020LL	/* Onboard device */
#define INTMAP_LSHIFT	11		/* Encode level in vector */
#define	INTLEVENCODE(x)	(((x)&0x0f)<<INTMAP_LSHIFT)
#define INTLEV(x)	(((x)>>INTMAP_LSHIFT)&0x0f)
#define INTVEC(x)	((x)&INTMAP_INR)
#define INTSLOT(x)	(((x)>>3)&0x7)
#define	INTPRI(x)	((x)&0x7)
#define INTIGN(x)	((x)&INTMAP_IGN)
#define	INTINO(x)	((x)&INTMAP_INO)
#define INTTID_SHIFT	26
#define INTTID(x)	(((x) & INTMAP_TID) >> INTTID_SHIFT)

#define	INTPCI_MAXOBINO	0x16		/* maximum OBIO INO value for PCI */
#define	INTPCIOBINOX(x)	((x)&0x1f)	/* OBIO ino index (for PCI machines) */
#define	INTPCIINOX(x)	(((x)&0x1c)>>2)	/* PCI ino index */

#define	INTCLR_IDLE	0

#endif /* _SPARC64_DEV_IOMMUREG_H_ */


code:
/*	$OpenBSD: iommuvar.h,v 1.16 2014/01/22 10:52:35 kettenis Exp $	*/
/*	$NetBSD: iommuvar.h,v 1.9 2001/10/07 20:30:41 eeh Exp $	*/

/*
 * Copyright (c) 2003 Henric Jungheim
 * Copyright (c) 1999 Matthew R. Green
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. The name of the author may not be used to endorse or promote products
 *    derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */

#ifndef _SPARC64_DEV_IOMMUVAR_H_
#define _SPARC64_DEV_IOMMUVAR_H_

#ifndef _SYS_TREE_H_
#include <sys/tree.h>
#endif

#include <sys/extent.h>
#include <sys/mutex.h>

/*
 * per-Streaming Buffer state
 */
struct strbuf_ctl {
	bus_space_tag_t		sb_bustag;	/* streaming buffer registers */
	bus_space_handle_t	sb_sb;		/* Handle for our regs */
	struct iommu_state	*sb_iommu;	/* Associated IOMMU */
	/*
	 * Since implementing the per-map IOMMU state, these per-STC
	 * flush areas are not used other than as a boolean flag to indicate
	 * the presence of a working and enabled STC.  For inconsistency's
	 * sake, the "sb" pointers of iommu_state are sometimes used for the
	 * same purpose.  This should be consolidated.  DEFINATELY, since
	 * mutex operations must happen at this level.
	 */
	struct mutex		sb_mtx;		/* one flush at a time */

	paddr_t			sb_flushpa;	/* to flush streaming buffers */
	volatile int64_t	*sb_flush;
};

/*
 * per-map STC flush area
 */
struct strbuf_flush {
	char	sbf_area[0x80];		/* Holds 64-byte long/aligned buffer */
	void	*sbf_flush;		/* Kernel virtual address of buffer */
	paddr_t	sbf_flushpa;		/* Physical address of buffer area */
};

/*
 * per-map DVMA page table
 */
struct iommu_page_entry {
	SPLAY_ENTRY(iommu_page_entry) ipe_node;
	paddr_t		ipe_pa;
	bus_addr_t	ipe_va;
};
struct iommu_page_map {
	SPLAY_HEAD(iommu_page_tree, iommu_page_entry) ipm_tree;
	int ipm_maxpage;	/* Size of allocated page map */
	int ipm_pagecnt;	/* Number of entries in use */
	struct iommu_page_entry	ipm_map[1];
};

/*
 * per-map IOMMU state
 *
 * This is what bus_dvmamap_t'c _dm_cookie should be pointing to.
 */
struct iommu_map_state {
	struct strbuf_flush ims_flush;	/* flush should be first (alignment) */
	struct strbuf_ctl *ims_sb;	/* Link to parent */
	struct iommu_state *ims_iommu;
	int ims_flags;
	struct extent_region ims_er;
	struct iommu_page_map ims_map;	/* map must be last (array at end) */
};
#define IOMMU_MAP_STREAM	1

/*
 * per-IOMMU state
 */
struct iommu_state {
	paddr_t			is_ptsb;	/* TSB physical address */
	int64_t			*is_tsb;	/* TSB virtual address */
	int			is_tsbsize;	/* 0 = 8K, ... */
	u_int			is_dvmabase;
	u_int			is_dvmaend;
	int64_t			is_cr;		/* Control register value */
	struct mutex		is_mtx;
	struct extent		*is_dvmamap;	/* DVMA map for this instance */
	int			is_flags;
#define IOMMU_FLUSH_CACHE	0x00000001

	struct strbuf_ctl	*is_sb[2];	/* Streaming buffers if any */

	/* copies of our parents state, to allow us to be self contained */
	bus_space_tag_t		is_bustag;	/* our bus tag */
	bus_space_handle_t	is_iommu;	/* IOMMU registers */
	uint64_t		is_devhandle;
};

/* interfaces for PCI/SBus code */
void	iommu_init(char *, struct iommu_state *, int, u_int32_t);
void	iommu_reset(struct iommu_state *);
paddr_t iommu_extract(struct iommu_state *, bus_addr_t);
int64_t iommu_lookup_tte(struct iommu_state *, bus_addr_t);
int64_t iommu_fetch_tte(struct iommu_state *, paddr_t);
/* bus_dma_tag_t implementation functions */
int	iommu_dvmamap_create(bus_dma_tag_t, bus_dma_tag_t, struct strbuf_ctl *,
	    bus_size_t, int, bus_size_t, bus_size_t, int, bus_dmamap_t *);
void	iommu_dvmamap_destroy(bus_dma_tag_t, bus_dma_tag_t, bus_dmamap_t);
int	iommu_dvmamap_load(bus_dma_tag_t, bus_dma_tag_t, bus_dmamap_t, void *,
	    bus_size_t, struct proc *, int);
void	iommu_dvmamap_unload(bus_dma_tag_t, bus_dma_tag_t, bus_dmamap_t);
int	iommu_dvmamap_load_raw(bus_dma_tag_t, bus_dma_tag_t, bus_dmamap_t,
	    bus_dma_segment_t *, int, bus_size_t, int);
void	iommu_dvmamap_sync(bus_dma_tag_t, bus_dma_tag_t, bus_dmamap_t,
	    bus_addr_t, bus_size_t, int);
int	iommu_dvmamem_alloc(bus_dma_tag_t, bus_dma_tag_t, bus_size_t,
	    bus_size_t, bus_size_t, bus_dma_segment_t *, int, int *, int);
void	iommu_dvmamem_free(bus_dma_tag_t, bus_dma_tag_t, bus_dma_segment_t *,
	    int);

#define IOMMUREG_READ(is, reg)				\
	bus_space_read_8((is)->is_bustag,		\
		(is)->is_iommu,				\
		IOMMUREG(reg))

#define IOMMUREG_WRITE(is, reg, v)			\
	bus_space_write_8((is)->is_bustag,		\
		(is)->is_iommu,				\
		IOMMUREG(reg),				\
		(v))

#endif /* _SPARC64_DEV_IOMMUVAR_H_ */