code:
/* $OpenBSD: amd64_mem.c,v 1.11 2015/03/14 03:38:46 jsg Exp $ */
/*
 * Copyright (c) 1999 Michael Smith <msmith@freebsd.org>
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 * $FreeBSD: src/sys/i386/i386/i686_mem.c,v 1.8 1999/10/12 22:53:05 green Exp $
 */

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/malloc.h>
#include <sys/memrange.h>

#include <machine/cpufunc.h>
#include <machine/intr.h>
#include <machine/specialreg.h>

/*
 * This code implements a set of MSRs known as MTRR which define caching
 * modes/behavior for various memory ranges.
 */

char *mem_owner_bios = "BIOS";

#define MR_FIXMTRR	(1<<0)

#define mrwithin(mr, a) \
    (((a) >= (mr)->mr_base) && ((a) < ((mr)->mr_base + (mr)->mr_len)))
#define mroverlap(mra, mrb) \
    (mrwithin(mra, mrb->mr_base) || mrwithin(mrb, mra->mr_base))

#define mrvalid(base, len) 						\
    ((!(base & ((1 << 12) - 1))) && 	/* base is multiple of 4k */	\
     ((len) >= (1 << 12)) && 		/* length is >= 4k */		\
     powerof2((len)) && 		/* ... and power of two */	\
     !((base) & ((len) - 1)))		/* range is not discontiuous */

#define mrcopyflags(curr, new) (((curr) & ~MDF_ATTRMASK) | \
	((new) & MDF_ATTRMASK))

#define FIXTOP	((MTRR_N64K * 0x10000) + (MTRR_N16K * 0x4000) + \
	(MTRR_N4K * 0x1000))

void	mrinit(struct mem_range_softc *sc);
int	mrset(struct mem_range_softc *sc,
	    struct mem_range_desc *mrd, int *arg);
void	mrinit_cpu(struct mem_range_softc *sc);
void	mrreload_cpu(struct mem_range_softc *sc);

struct mem_range_ops mrops = {
	mrinit,
	mrset,
	mrinit_cpu,
	mrreload_cpu
};

u_int64_t	mtrrcap, mtrrdef;
u_int64_t	mtrrmask = 0x0000000ffffff000ULL;

struct mem_range_desc	*mem_range_match(struct mem_range_softc *sc,
			     struct mem_range_desc *mrd);
void			 mrfetch(struct mem_range_softc *sc);
int			 mtrrtype(u_int64_t flags);
int			 mrt2mtrr(u_int64_t flags);
int			 mtrr2mrt(int val);
int			 mtrrconflict(u_int64_t flag1, u_int64_t flag2);
void			 mrstore(struct mem_range_softc *sc);
void			 mrstoreone(struct mem_range_softc *sc);
struct mem_range_desc	*mtrrfixsearch(struct mem_range_softc *sc,
			     u_int64_t addr);
int			 mrsetlow(struct mem_range_softc *sc,
			     struct mem_range_desc *mrd, int *arg);
int			 mrsetvariable(struct mem_range_softc *sc,
			     struct mem_range_desc *mrd, int *arg);

/* MTRR type to memory range type conversion */
int mtrrtomrt[] = {
	MDF_UNCACHEABLE,
	MDF_WRITECOMBINE,
	MDF_UNKNOWN,
	MDF_UNKNOWN,
	MDF_WRITETHROUGH,
	MDF_WRITEPROTECT,
	MDF_WRITEBACK
};

int
mtrr2mrt(int val)
{
	if (val < 0 || val >= nitems(mtrrtomrt))
		return MDF_UNKNOWN;
	return mtrrtomrt[val];
}

/*
 * MTRR conflicts. Writeback and uncachable may overlap.
 */
int
mtrrconflict(u_int64_t flag1, u_int64_t flag2)
{
	flag1 &= MDF_ATTRMASK;
	flag2 &= MDF_ATTRMASK;
	if (flag1 == flag2 ||
	    (flag1 == MDF_WRITEBACK && flag2 == MDF_UNCACHEABLE) ||
	    (flag2 == MDF_WRITEBACK && flag1 == MDF_UNCACHEABLE))
		return 0;
	return 1;
}

/*
 * Look for an exactly-matching range.
 */
struct mem_range_desc *
mem_range_match(struct mem_range_softc *sc, struct mem_range_desc *mrd)
{
	struct mem_range_desc	*cand;
	int			 i;

	for (i = 0, cand = sc->mr_desc; i < sc->mr_ndesc; i++, cand++)
		if ((cand->mr_base == mrd->mr_base) &&
		    (cand->mr_len == mrd->mr_len))
			return(cand);
	return(NULL);
}

/*
 * Fetch the current mtrr settings from the current CPU (assumed to all
 * be in sync in the SMP case).  Note that if we are here, we assume
 * that MTRRs are enabled, and we may or may not have fixed MTRRs.
 */
void
mrfetch(struct mem_range_softc *sc)
{
	struct mem_range_desc	*mrd;
	u_int64_t		 msrv;
	int			 i, j, msr, mrt;

	mrd = sc->mr_desc;

	/* We should never be fetching MTRRs from an AP */
	KASSERT(CPU_IS_PRIMARY(curcpu()));

	/* Get fixed-range MTRRs, if the CPU supports them */
	if (sc->mr_cap & MR_FIXMTRR) {
		msr = MSR_MTRRfix64K_00000;
		for (i = 0; i < (MTRR_N64K / 8); i++, msr++) {
			msrv = rdmsr(msr);
			for (j = 0; j < 8; j++, mrd++) {
				mrt = mtrr2mrt(msrv & 0xff);
				if (mrt == MDF_UNKNOWN)
					mrt = MDF_UNCACHEABLE;
				mrd->mr_flags = (mrd->mr_flags & ~MDF_ATTRMASK) |
					mrt | MDF_ACTIVE;
				if (mrd->mr_owner[0] == 0)
					strlcpy(mrd->mr_owner, mem_owner_bios,
					    sizeof(mrd->mr_owner));
				msrv = msrv >> 8;
			}
		}

		msr = MSR_MTRRfix16K_80000;
		for (i = 0; i < (MTRR_N16K / 8); i++, msr++) {
			msrv = rdmsr(msr);
			for (j = 0; j < 8; j++, mrd++) {
				mrt = mtrr2mrt(msrv & 0xff);
				if (mrt == MDF_UNKNOWN)
					mrt = MDF_UNCACHEABLE;
				mrd->mr_flags = (mrd->mr_flags & ~MDF_ATTRMASK) |
					mrt | MDF_ACTIVE;
				if (mrd->mr_owner[0] == 0)
					strlcpy(mrd->mr_owner, mem_owner_bios,
					    sizeof(mrd->mr_owner));
				msrv = msrv >> 8;
			}
		}

		msr = MSR_MTRRfix4K_C0000;
		for (i = 0; i < (MTRR_N4K / 8); i++, msr++) {
			msrv = rdmsr(msr);
			for (j = 0; j < 8; j++, mrd++) {
				mrt = mtrr2mrt(msrv & 0xff);
				if (mrt == MDF_UNKNOWN)
					mrt = MDF_UNCACHEABLE;
				mrd->mr_flags = (mrd->mr_flags & ~MDF_ATTRMASK) |
					mrt | MDF_ACTIVE;
				if (mrd->mr_owner[0] == 0)
					strlcpy(mrd->mr_owner, mem_owner_bios,
					    sizeof(mrd->mr_owner));
				msrv = msrv >> 8;
			}
		}
	}

	/* Get remainder which must be variable MTRRs */
	msr = MSR_MTRRvarBase;
	for (; (mrd - sc->mr_desc) < sc->mr_ndesc; msr += 2, mrd++) {
		msrv = rdmsr(msr);
		mrt = mtrr2mrt(msrv & 0xff);
		if (mrt == MDF_UNKNOWN)
			mrt = MDF_UNCACHEABLE;
		mrd->mr_flags = (mrd->mr_flags & ~MDF_ATTRMASK) | mrt;
		mrd->mr_base = msrv & mtrrmask;
		msrv = rdmsr(msr + 1);
		mrd->mr_flags = (msrv & 0x800) ?
			(mrd->mr_flags | MDF_ACTIVE) :
			(mrd->mr_flags & ~MDF_ACTIVE);
		/* Compute the range from the mask. Ick. */
		mrd->mr_len = (~(msrv & mtrrmask) & mtrrmask) + 0x1000;
		if (!mrvalid(mrd->mr_base, mrd->mr_len))
			mrd->mr_flags |= MDF_BOGUS;
		/* If unclaimed and active, must be the BIOS */
		if ((mrd->mr_flags & MDF_ACTIVE) && (mrd->mr_owner[0] == 0))
			strlcpy(mrd->mr_owner, mem_owner_bios,
			    sizeof(mrd->mr_owner));
	}
}

/*
 * Return the MTRR memory type matching a region's flags
 */
int
mtrrtype(u_int64_t flags)
{
	int i;

	flags &= MDF_ATTRMASK;

	for (i = 0; i < nitems(mtrrtomrt); i++) {
		if (mtrrtomrt[i] == MDF_UNKNOWN)
			continue;
		if (flags == mtrrtomrt[i])
			return(i);
	}
	return MDF_UNCACHEABLE;
}

int
mrt2mtrr(u_int64_t flags)
{
	int val;

	val = mtrrtype(flags);

	return val & 0xff;
}

/*
 * Update running CPU(s) MTRRs to match the ranges in the descriptor
 * list.
 *
 * XXX Must be called with interrupts enabled.
 */
void
mrstore(struct mem_range_softc *sc)
{
	disable_intr();				/* disable interrupts */
#ifdef MULTIPROCESSOR
	x86_broadcast_ipi(X86_IPI_MTRR);
#endif
	mrstoreone(sc);
	enable_intr();
}

/*
 * Update the current CPU's MTRRs with those represented in the
 * descriptor list.  Note that we do this wholesale rather than
 * just stuffing one entry; this is simpler (but slower, of course).
 */
void
mrstoreone(struct mem_range_softc *sc)
{
	struct mem_range_desc	*mrd;
	u_int64_t		 msrv;
	int			 i, j, msr;
	u_int			 cr4save;

	mrd = sc->mr_desc;

	cr4save = rcr4();	/* save cr4 */
	if (cr4save & CR4_PGE)
		lcr4(cr4save & ~CR4_PGE);

	/* Flush caches, then disable caches, then disable MTRRs */
	wbinvd();
	lcr0((rcr0() & ~CR0_NW) | CR0_CD);
	wrmsr(MSR_MTRRdefType, rdmsr(MSR_MTRRdefType) & ~0x800);

	/* Set fixed-range MTRRs */
	if (sc->mr_cap & MR_FIXMTRR) {
		msr = MSR_MTRRfix64K_00000;
		for (i = 0; i < (MTRR_N64K / 8); i++, msr++) {
			msrv = 0;
			for (j = 7; j >= 0; j--) {
				msrv = msrv << 8;
				msrv |= mrt2mtrr((mrd + j)->mr_flags);
			}
			wrmsr(msr, msrv);
			mrd += 8;
		}

		msr = MSR_MTRRfix16K_80000;
		for (i = 0, msrv = 0; i < (MTRR_N16K / 8); i++, msr++) {
			for (j = 7; j >= 0; j--) {
				msrv = msrv << 8;
				msrv |= mrt2mtrr((mrd + j)->mr_flags);
			}
			wrmsr(msr, msrv);
			mrd += 8;
		}

		msr = MSR_MTRRfix4K_C0000;
		for (i = 0, msrv = 0; i < (MTRR_N4K / 8); i++, msr++) {
			for (j = 7; j >= 0; j--) {
				msrv = msrv << 8;
				msrv |= mrt2mtrr((mrd + j)->mr_flags);
			}
			wrmsr(msr, msrv);
			mrd += 8;
		}
	}

	/* Set remainder which must be variable MTRRs */
	msr = MSR_MTRRvarBase;
	for (; (mrd - sc->mr_desc) < sc->mr_ndesc; msr += 2, mrd++) {
		if (mrd->mr_flags & MDF_ACTIVE) {
			msrv = mrd->mr_base & mtrrmask;
			msrv |= mrt2mtrr(mrd->mr_flags);
		} else
			msrv = 0;

		wrmsr(msr, msrv);

		/* mask/active register */
		if (mrd->mr_flags & MDF_ACTIVE) {
			msrv = 0x800 | (~(mrd->mr_len - 1) & mtrrmask);
		} else
			msrv = 0;

		wrmsr(msr + 1, msrv);
	}

	/* Re-enable caches and MTRRs */
	wrmsr(MSR_MTRRdefType, mtrrdef | 0x800);
	lcr0(rcr0() & ~(CR0_CD | CR0_NW));
	lcr4(cr4save);
}

/*
 * Hunt for the fixed MTRR referencing (addr)
 */
struct mem_range_desc *
mtrrfixsearch(struct mem_range_softc *sc, u_int64_t addr)
{
	struct mem_range_desc *mrd;
	int			i;

	for (i = 0, mrd = sc->mr_desc; i < (MTRR_N64K + MTRR_N16K + MTRR_N4K); i++, mrd++)
		if ((addr >= mrd->mr_base) && (addr < (mrd->mr_base + mrd->mr_len)))
			return(mrd);
	return(NULL);
}

/*
 * Try to satisfy the given range request by manipulating the fixed MTRRs that
 * cover low memory.
 *
 * Note that we try to be generous here; we'll bloat the range out to the
 * next higher/lower boundary to avoid the consumer having to know too much
 * about the mechanisms here.
 */
int
mrsetlow(struct mem_range_softc *sc, struct mem_range_desc *mrd, int *arg)
{
	struct mem_range_desc	*first_md, *last_md, *curr_md;

	/* range check */
	if (((first_md = mtrrfixsearch(sc, mrd->mr_base)) == NULL) ||
	    ((last_md = mtrrfixsearch(sc, mrd->mr_base + mrd->mr_len - 1)) == NULL))
		return(EINVAL);

	/* check we aren't doing something risky */
	if (!(mrd->mr_flags & MDF_FORCE))
		for (curr_md = first_md; curr_md <= last_md; curr_md++) {
			if ((curr_md->mr_flags & MDF_ATTRMASK) == MDF_UNKNOWN)
				return (EACCES);
		}

	/* set flags, clear set-by-firmware flag */
	for (curr_md = first_md; curr_md <= last_md; curr_md++) {
		curr_md->mr_flags = mrcopyflags(curr_md->mr_flags & ~MDF_FIRMWARE, mrd->mr_flags);
		memcpy(curr_md->mr_owner, mrd->mr_owner, sizeof(mrd->mr_owner));
	}

	return(0);
}

/*
 * Modify/add a variable MTRR to satisfy the request.
 */
int
mrsetvariable(struct mem_range_softc *sc, struct mem_range_desc *mrd, int *arg)
{
	struct mem_range_desc	*curr_md, *free_md;
	int			 i;

	/*
	 * Scan the currently active variable descriptors, look for
	 * one we exactly match (straight takeover) and for possible
	 * accidental overlaps.
	 * Keep track of the first empty variable descriptor in case we
	 * can't perform a takeover.
	 */
	i = (sc->mr_cap & MR_FIXMTRR) ? MTRR_N64K + MTRR_N16K + MTRR_N4K : 0;
	curr_md = sc->mr_desc + i;
	free_md = NULL;
	for (; i < sc->mr_ndesc; i++, curr_md++) {
		if (curr_md->mr_flags & MDF_ACTIVE) {
			/* exact match? */
			if ((curr_md->mr_base == mrd->mr_base) &&
			    (curr_md->mr_len == mrd->mr_len)) {
				/* check we aren't doing something risky */
				if (!(mrd->mr_flags & MDF_FORCE) &&
				    ((curr_md->mr_flags & MDF_ATTRMASK)
				    == MDF_UNKNOWN))
					return (EACCES);
				/* Ok, just hijack this entry */
				free_md = curr_md;
				break;
			}
			/* non-exact overlap ? */
			if (mroverlap(curr_md, mrd)) {
				/* between conflicting region types? */
				if (mtrrconflict(curr_md->mr_flags,
						      mrd->mr_flags))
					return(EINVAL);
			}
		} else if (free_md == NULL) {
			free_md = curr_md;
		}
	}
	/* got somewhere to put it? */
	if (free_md == NULL)
		return(ENOSPC);

	/* Set up new descriptor */
	free_md->mr_base = mrd->mr_base;
	free_md->mr_len = mrd->mr_len;
	free_md->mr_flags = mrcopyflags(MDF_ACTIVE, mrd->mr_flags);
	memcpy(free_md->mr_owner, mrd->mr_owner, sizeof(mrd->mr_owner));
	return(0);
}

/*
 * Handle requests to set memory range attributes by manipulating MTRRs.
 */
int
mrset(struct mem_range_softc *sc, struct mem_range_desc *mrd, int *arg)
{
	struct mem_range_desc	*targ;
	int			 error = 0;

	switch(*arg) {
	case MEMRANGE_SET_UPDATE:
		/* make sure that what's being asked for is possible */
		if (!mrvalid(mrd->mr_base, mrd->mr_len) ||
		    mtrrtype(mrd->mr_flags) == -1)
			return(EINVAL);

		/* are the "low memory" conditions applicable? */
		if ((sc->mr_cap & MR_FIXMTRR) &&
		    ((mrd->mr_base + mrd->mr_len) <= FIXTOP)) {
			if ((error = mrsetlow(sc, mrd, arg)) != 0)
				return(error);
		} else {
			/* it's time to play with variable MTRRs */
			if ((error = mrsetvariable(sc, mrd, arg)) != 0)
				return(error);
		}
		break;

	case MEMRANGE_SET_REMOVE:
		if ((targ = mem_range_match(sc, mrd)) == NULL)
			return(ENOENT);
		if (targ->mr_flags & MDF_FIXACTIVE)
			return(EPERM);
		targ->mr_flags &= ~MDF_ACTIVE;
		targ->mr_owner[0] = 0;
		break;

	default:
		return(EOPNOTSUPP);
	}

	/* update the hardware */
	mrstore(sc);
	return(0);
}

/*
 * Work out how many ranges we support, initialise storage for them,
 * fetch the initial settings.
 */
void
mrinit(struct mem_range_softc *sc)
{
	struct mem_range_desc	*mrd;
	uint32_t		 regs[4];
	int			 nmdesc = 0;
	int			 i;

	mtrrcap = rdmsr(MSR_MTRRcap);
	mtrrdef = rdmsr(MSR_MTRRdefType);

	/* For now, bail out if MTRRs are not enabled */
	if (!(mtrrdef & 0x800)) {
		printf("mtrr: CPU supports MTRRs but not enabled by BIOS\n");
		return;
	}
	nmdesc = mtrrcap & 0xff;
	printf("mtrr: Pentium Pro MTRR support, %d var ranges", nmdesc);

	/* If fixed MTRRs supported and enabled */
	if ((mtrrcap & 0x100) && (mtrrdef & 0x400)) {
		sc->mr_cap = MR_FIXMTRR;
		nmdesc += MTRR_N64K + MTRR_N16K + MTRR_N4K;
		printf(", %d fixed ranges", MTRR_N64K + MTRR_N16K + MTRR_N4K);
	}

	printf("\n");

	sc->mr_desc = mallocarray(nmdesc, sizeof(struct mem_range_desc),
	     M_MEMDESC, M_WAITOK|M_ZERO);
	sc->mr_ndesc = nmdesc;

	mrd = sc->mr_desc;

	/* Populate the fixed MTRR entries' base/length */
	if (sc->mr_cap & MR_FIXMTRR) {
		for (i = 0; i < MTRR_N64K; i++, mrd++) {
			mrd->mr_base = i * 0x10000;
			mrd->mr_len = 0x10000;
			mrd->mr_flags = MDF_FIXBASE | MDF_FIXLEN | MDF_FIXACTIVE;
		}

		for (i = 0; i < MTRR_N16K; i++, mrd++) {
			mrd->mr_base = i * 0x4000 + 0x80000;
			mrd->mr_len = 0x4000;
			mrd->mr_flags = MDF_FIXBASE | MDF_FIXLEN | MDF_FIXACTIVE;
		}

		for (i = 0; i < MTRR_N4K; i++, mrd++) {
			mrd->mr_base = i * 0x1000 + 0xc0000;
			mrd->mr_len = 0x1000;
			mrd->mr_flags = MDF_FIXBASE | MDF_FIXLEN | MDF_FIXACTIVE;
		}
	}

	/*
	 * Fetch maximum physical address size supported by the
	 * processor as supported by CPUID leaf function 0x80000008.
	 * If CPUID does not support leaf function 0x80000008, use the
	 * default a 36-bit address size.
	 */
	CPUID(0x80000000, regs[0], regs[1], regs[2], regs[3]);
	if (regs[0] >= 0x80000008) {
		CPUID(0x80000008, regs[0], regs[1], regs[2], regs[3]);
		if (regs[0] & 0xff) {
			mtrrmask = (1ULL << (regs[0] & 0xff)) - 1;
			mtrrmask &= ~0x0000000000000fffULL;
		}
	}

	/*
	 * Get current settings, anything set now is considered to have
	 * been set by the firmware.
	 */
	mrfetch(sc);
	mrd = sc->mr_desc;
	for (i = 0; i < sc->mr_ndesc; i++, mrd++) {
		if (mrd->mr_flags & MDF_ACTIVE)
			mrd->mr_flags |= MDF_FIRMWARE;
	}
}

/*
 * Initialise MTRRs on a cpu from the software state.
 */
void
mrinit_cpu(struct mem_range_softc *sc)
{
	mrstoreone(sc); /* set MTRRs to match BSP */
}

void
mrreload_cpu(struct mem_range_softc *sc)
{
	disable_intr();
	mrstoreone(sc); /* set MTRRs to match BSP */
	enable_intr();
}


code:
/*	$OpenBSD: amd64errata.c,v 1.7 2014/07/13 12:11:01 jasper Exp $	*/
/*	$NetBSD: errata.c,v 1.6 2007/02/05 21:05:45 ad Exp $	*/

/*-
 * Copyright (c) 2007 The NetBSD Foundation, Inc.
 * All rights reserved.
 *
 * This code is derived from software contributed to The NetBSD Foundation
 * by Andrew Doran.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

/*
 * Detect, report on, and work around known errata with AMD's amd64 CPUs.
 *
 * This is generalised because there are quite a few problems that the
 * BIOS can patch via MSR, but it is not known if the OS can patch these
 * yet.  The list is expected to grow over time.
 *
 * The data here are from: Revision Guide for AMD Athlon 64 and
 * AMD Opteron Processors, Publication #25759, Revision: 3.69,
 * Issue Date: September 2006
 */

#include <sys/param.h>

#include <sys/types.h>
#include <sys/systm.h>

#include <machine/cpu.h>
#include <machine/cpufunc.h>
#include <machine/specialreg.h>

typedef struct errata {
	u_short		e_num;
	u_short		e_reported;
	u_int		e_data1;
	const uint8_t	*e_set;
	int		(*e_act)(struct cpu_info *, struct errata *);
	uint64_t	e_data2;
} errata_t;

typedef enum cpurev {
	BH_E4, CH_CG, CH_D0, DH_CG, DH_D0, DH_E3, DH_E6, JH_E1,
	JH_E6, SH_B0, SH_B3, SH_C0, SH_CG, SH_D0, SH_E4, SH_E5,
	DR_BA, DR_B2, DR_B3, RB_C2, RB_C3, BL_C2, BL_C3, DA_C2,
	DA_C3, HY_D0, HY_D1, HY_D1_G34R1,  PH_E0, LN_B0,
	OINK
} cpurev_t;

static const u_int cpurevs[] = {
	BH_E4, 0x0020fb1, CH_CG, 0x0000f82, CH_CG, 0x0000fb2,
	CH_D0, 0x0010f80, CH_D0, 0x0010fb0, DH_CG, 0x0000fc0,
	DH_CG, 0x0000fe0, DH_CG, 0x0000ff0, DH_D0, 0x0010fc0,
	DH_D0, 0x0010ff0, DH_E3, 0x0020fc0, DH_E3, 0x0020ff0,
	DH_E6, 0x0020fc2, DH_E6, 0x0020ff2, JH_E1, 0x0020f10,
	JH_E6, 0x0020f12, JH_E6, 0x0020f32, SH_B0, 0x0000f40,
	SH_B3, 0x0000f51, SH_C0, 0x0000f48, SH_C0, 0x0000f58,
	SH_CG, 0x0000f4a, SH_CG, 0x0000f5a, SH_CG, 0x0000f7a,
	SH_D0, 0x0010f40, SH_D0, 0x0010f50, SH_D0, 0x0010f70,
	SH_E4, 0x0020f51, SH_E4, 0x0020f71, SH_E5, 0x0020f42,
	DR_BA, 0x0100f2a, DR_B2, 0x0100f22, DR_B3, 0x0100f23,
	RB_C2, 0x0100f42, RB_C3, 0x0100f43, BL_C2, 0x0100f52,
	BL_C3, 0x0100f53, DA_C2, 0x0100f62, DA_C3, 0x0100f63,
	HY_D0, 0x0100f80, HY_D1, 0x0100f81, HY_D1_G34R1, 0x0100f91,
	PH_E0, 0x0100fa0, LN_B0, 0x0300f10,
	OINK
};

static const uint8_t amd64_errata_set1[] = {
	SH_B3, SH_C0, SH_CG, DH_CG, CH_CG, OINK
};

static const uint8_t amd64_errata_set2[] = {
	SH_B3, SH_C0, SH_CG, DH_CG, CH_CG, SH_D0, DH_D0, CH_D0, OINK
};

static const uint8_t amd64_errata_set3[] = {
	JH_E1, DH_E3, OINK
};

#if 0
static const uint8_t amd64_errata_set4[] = {
	SH_C0, SH_CG, DH_CG, CH_CG, SH_D0, DH_D0, CH_D0, JH_E1,
	DH_E3, SH_E4, BH_E4, SH_E5, DH_E6, JH_E6, OINK
};
#endif

static const uint8_t amd64_errata_set5[] = {
	SH_B3, OINK
};

static const uint8_t amd64_errata_set6[] = {
	SH_C0, SH_CG, DH_CG, CH_CG, OINK
};

static const uint8_t amd64_errata_set7[] = {
	SH_C0, SH_CG, DH_CG, CH_CG, SH_D0, DH_D0, CH_D0, OINK
};

static const uint8_t amd64_errata_set8[] = {
	BH_E4, CH_CG, CH_CG, CH_D0, CH_D0, DH_CG, DH_CG, DH_CG,
	DH_D0, DH_D0, DH_E3, DH_E3, DH_E6, DH_E6, JH_E1, JH_E6,
	JH_E6, SH_B0, SH_B3, SH_C0, SH_C0, SH_CG, SH_CG, SH_CG,
	SH_D0, SH_D0, SH_D0, SH_E4, SH_E4, SH_E5, OINK
};

static const uint8_t amd64_errata_set9[] = {
	DR_BA, DR_B2, DR_B3, RB_C2, RB_C3, BL_C2, BL_C3, DA_C2,
	DA_C3, HY_D0, HY_D1, HY_D1_G34R1,  PH_E0, LN_B0, OINK
};

int amd64_errata_setmsr(struct cpu_info *, errata_t *);
int amd64_errata_testmsr(struct cpu_info *, errata_t *);

static errata_t errata[] = {
	/*
	 * 81: Cache Coherency Problem with Hardware Prefetching
	 * and Streaming Stores
	 */
	{
		81, 0, MSR_DC_CFG, amd64_errata_set5,
		amd64_errata_testmsr, DC_CFG_DIS_SMC_CHK_BUF
	},
	/*
	 * 86: DRAM Data Masking Feature Can Cause ECC Failures
	 */
	{
		86, 0, MSR_NB_CFG, amd64_errata_set1,
		amd64_errata_testmsr, NB_CFG_DISDATMSK
	},
	/*
	 * 89: Potential Deadlock With Locked Transactions
	 */
	{
		89, 0, MSR_NB_CFG, amd64_errata_set8,
		amd64_errata_testmsr, NB_CFG_DISIOREQLOCK
	},
	/*
	 * 94: Sequential Prefetch Feature May Cause Incorrect
	 * Processor Operation
	 */
	{
		94, 0, MSR_IC_CFG, amd64_errata_set1,
		amd64_errata_testmsr, IC_CFG_DIS_SEQ_PREFETCH
	},
	/*
	 * 97: 128-Bit Streaming Stores May Cause Coherency
	 * Failure
	 *
	 * XXX "This workaround must not be applied to processors
	 * prior to revision C0."  We don't apply it, but if it
	 * can't be applied, it shouldn't be reported.
	 */
	{
		97, 0, MSR_DC_CFG, amd64_errata_set6,
		amd64_errata_testmsr, DC_CFG_DIS_CNV_WC_SSO
	},
	/*
	 * 104: DRAM Data Masking Feature Causes ChipKill ECC
	 * Failures When Enabled With x8/x16 DRAM Devices
	 */
	{
		104, 0, MSR_NB_CFG, amd64_errata_set7,
		amd64_errata_testmsr, NB_CFG_DISDATMSK
	},
	/*
	 * 113: Enhanced Write-Combining Feature Causes System Hang
	 */
	{
		113, 0, MSR_BU_CFG, amd64_errata_set3,
		amd64_errata_setmsr, BU_CFG_WBENHWSBDIS
	},
#ifdef MULTIPROCESSOR
	/*
	 * 69: Multiprocessor Coherency Problem with Hardware
	 * Prefetch Mechanism
	 */
	{
		69, 0, MSR_BU_CFG, amd64_errata_set5,
		amd64_errata_setmsr, BU_CFG_WBPFSMCCHKDIS
	},
	/*
	 * 101: DRAM Scrubber May Cause Data Corruption When Using
	 * Node-Interleaved Memory
	 */
	{
		101, 0, 0, amd64_errata_set2,
		NULL, 0
	},
	/*
	 * 106: Potential Deadlock with Tightly Coupled Semaphores
	 * in an MP System
	 */
	{
		106, 0, MSR_LS_CFG, amd64_errata_set2,
		amd64_errata_testmsr, LS_CFG_DIS_LS2_SQUISH
	},
	/*
	 * 107: Possible Multiprocessor Coherency Problem with
	 * Setting Page Table A/D Bits
	 */
	{
		107, 0, MSR_BU_CFG, amd64_errata_set2,
		amd64_errata_testmsr, BU_CFG_THRL2IDXCMPDIS
	},
#if 0
	/*
	 * 122: TLB Flush Filter May Cause Coherency Problem in
	 * Multiprocessor Systems
	 */
	{
		122, 0, MSR_HWCR, amd64_errata_set4,
		amd64_errata_setmsr, HWCR_FFDIS
	},
#endif
#endif	/* MULTIPROCESSOR */
	/*
	 * 721: Processor May Incorrectly Update Stack Pointer
	 */
	{
		721, 0, MSR_DE_CFG, amd64_errata_set9,
		amd64_errata_setmsr, DE_CFG_721
	},
};

int
amd64_errata_testmsr(struct cpu_info *ci, errata_t *e)
{
	uint64_t val;

	(void)ci;

	val = rdmsr_locked(e->e_data1, OPTERON_MSR_PASSCODE);
	if ((val & e->e_data2) != 0)
		return 0;		/* not found */

	e->e_reported = 1;
	return 1;			/* found */
}

int
amd64_errata_setmsr(struct cpu_info *ci, errata_t *e)
{
	uint64_t val;

	(void)ci;

	val = rdmsr_locked(e->e_data1, OPTERON_MSR_PASSCODE);
	if ((val & e->e_data2) != 0)
		return 0;		/* not found */

	wrmsr_locked(e->e_data1, OPTERON_MSR_PASSCODE, val | e->e_data2);

#ifdef ERRATA_DEBUG
	printf("ERRATA: writing a fix\n");
	val = rdmsr_locked(e->e_data1, OPTERON_MSR_PASSCODE);
	if ((val & e->e_data2) != 0)
		printf("ERRATA: fix seems to have worked!\n");
#endif

	e->e_reported = 1;
	return 2;			/* found and fixed */
}

void
amd64_errata(struct cpu_info *ci)
{
	u_int32_t code, dummy;
	errata_t *e, *ex;
	cpurev_t rev;
	int i, j;
	int rc;
	int found = 0;
	int corrected = 0;

	CPUID(0x80000001, code, dummy, dummy, dummy);

	for (i = 0; ; i += 2) {
		if ((rev = cpurevs[i]) == OINK) {
#ifdef ERRATA_DEBUG
			printf("ERRATA: this CPU ok\n");
#endif
			return;
		}
		if (cpurevs[i + 1] == code) {
#ifdef ERRATA_DEBUG
			printf("ERRATA: this CPU has errata\n");
#endif
			break;
		}
	}

	ex = errata + nitems(errata);

	/* Reset e_reporteds (for multiple CPUs) */
	for (e = errata; e < ex; e++)
		e->e_reported = 0;

	for (e = errata; e < ex; e++) {
		if (e->e_reported)
			continue;
		if (e->e_set != NULL) {
			for (j = 0; e->e_set[j] != OINK; j++)
				if (e->e_set[j] == rev)
					break;
			if (e->e_set[j] == OINK)
				continue;
		}

#ifdef ERRATA_DEBUG
		printf("%s: testing for erratum %d\n",
		    ci->ci_dev->dv_xname, e->e_num);
#endif

		/*
		 * If we have an action routine, call it, otherwise
		 * the default is that this erratum is present.
		 */
		rc = (e->e_act == NULL) ? 1 : (*e->e_act)(ci, e);

		if (rc == 0)			/* not found */
			continue;
		if (rc == 1)
			found++;
		if (rc == 2)
			corrected++;

		e->e_reported = rc;

#ifdef ERRATA_DEBUG
		printf("%s: erratum %d present%s\n",
		    ci->ci_dev->dv_xname, e->e_num,
		    (rc == 2) ? " and patched" : "");
#endif
	}

#define ERRATA_VERBOSE
#ifdef ERRATA_VERBOSE
	if (corrected) {
		int first = 1;

		/* Print out found and corrected */
		printf("%s: AMD %s", ci->ci_dev->dv_xname,
		    (corrected == 1) ? "erratum" : "errata");
		for (e = errata; e < ex; e++) {
			if (e->e_reported == 2) {
				if (! first)
					printf(",");
				printf(" %d", e->e_num);
				first = 0;
			}
		}
		printf(" detected and fixed\n");
	}
#endif

	if (found) {
		int first = 1;

		/* Print out found but not corrected */
		printf("%s: AMD %s", ci->ci_dev->dv_xname,
		    (found == 1) ? "erratum" : "errata");
		for (e = errata; e < ex; e++) {
			if (e->e_reported == 1) {
				if (! first)
					printf(",");
				printf(" %d", e->e_num);
				first = 0;
			}
		}
		printf(" present, BIOS upgrade may be required\n");
	}
}

license: unknown


code:
/*	$OpenBSD: apic.c,v 1.6 2015/03/14 03:38:46 jsg Exp $	*/
/* $NetBSD: apic.c,v 1.1 2003/02/26 21:26:11 fvdl Exp $ */

/*-
 * Copyright (c) 2000 The NetBSD Foundation, Inc.
 * All rights reserved.
 *
 * This code is derived from software contributed to The NetBSD Foundation
 * by RedBack Networks Inc.
 *
 * Author: Bill Sommerfeld
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#include <sys/types.h>
#include <sys/systm.h>

#include <machine/i82489reg.h>
#include <machine/i82489var.h>

void
apic_format_redir(char *where1, char *where2, int idx, u_int32_t redirhi,
    u_int32_t redirlo)
{
	printf("%s: %s%d 0x%x", where1, where2, idx, redirlo);

	if ((redirlo & LAPIC_DEST_MASK) == 0)
		printf(" 0x%x", redirhi);

	printf("\n");
}


code:
/*	$OpenBSD: autoconf.c,v 1.43 2015/07/17 21:53:56 mlarkin Exp $	*/
/*	$NetBSD: autoconf.c,v 1.1 2003/04/26 18:39:26 fvdl Exp $	*/

/*-
 * Copyright (c) 1990 The Regents of the University of California.
 * All rights reserved.
 *
 * This code is derived from software contributed to Berkeley by
 * William Jolitz.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 *	@(#)autoconf.c	7.1 (Berkeley) 5/9/91
 */

/*
 * Setup the system to run on the current machine.
 *
 * cpu_configure() is called at boot time and initializes the vba
 * device tables and the memory controller monitoring.  Available
 * devices are determined (from possibilities mentioned in ioconf.c),
 * and the drivers are initialized.
 */

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/buf.h>
#include <sys/disklabel.h>
#include <sys/conf.h>
#include <sys/device.h>
#include <sys/malloc.h>
#include <sys/vnode.h>
#include <sys/fcntl.h>
#include <sys/dkio.h>
#include <sys/reboot.h>
#include <sys/socket.h>
#include <sys/socketvar.h>
#include <sys/timeout.h>
#include <sys/hibernate.h>

#include <net/if.h>
#include <net/if_types.h>
#include <netinet/in.h>
#include <netinet/if_ether.h>

#include <machine/pte.h>
#include <machine/cpu.h>
#include <machine/cpufunc.h>
#include <machine/biosvar.h>

#include "ioapic.h"
#include "lapic.h"

#if NIOAPIC > 0
#include <machine/i82093var.h>
#endif

#if NLAPIC > 0
#include <machine/i82489var.h>
#endif

int	cold = 1;	/* if 1, still working on cold-start */
extern dev_t bootdev;

/* Support for VIA C3 RNG */
extern struct timeout viac3_rnd_tmo;
extern int	viac3_rnd_present;
void		viac3_rnd(void *);

extern struct timeout rdrand_tmo;
extern int	has_rdrand;
void		rdrand(void *);

#ifdef CRYPTO
void		viac3_crypto_setup(void);
extern int	amd64_has_xcrypt;

void		aesni_setup(void);
extern int	amd64_has_aesni;
#endif

/*
 * Determine i/o configuration for a machine.
 */
void
cpu_configure(void)
{
	x86_64_proc0_tss_ldt_init();

	if (config_rootfound("mainbus", NULL) == NULL)
		panic("configure: mainbus not configured");

	intr_printconfig();

#if NIOAPIC > 0
	lapic_set_lvt();
	ioapic_enable();
#endif

#ifdef MULTIPROCESSOR
	cpu_init_idle_pcbs();
#endif

	lcr8(0);
	spl0();
	cold = 0;

	/*
	 * At this point the RNG is running, and if FSXR is set we can
	 * use it.  Here we setup a periodic timeout to collect the data.
	 */
	if (viac3_rnd_present) {
		timeout_set(&viac3_rnd_tmo, viac3_rnd, &viac3_rnd_tmo);
		viac3_rnd(&viac3_rnd_tmo);
	}
	if (has_rdrand) {
		timeout_set(&rdrand_tmo, rdrand, &rdrand_tmo);
		rdrand(&rdrand_tmo);
	}
#ifdef CRYPTO
	/*
	 * Also, if the chip has crypto available, enable it.
	 */
	if (amd64_has_xcrypt)
		viac3_crypto_setup();

	if (amd64_has_aesni)
		aesni_setup();
#endif
}

void
device_register(struct device *dev, void *aux)
{
}

/*
 * Now that we are fully operational, we can checksum the
 * disks, and using some heuristics, hopefully are able to
 * always determine the correct root disk.
 */
void
diskconf(void)
{
	int majdev, unit, part = 0;
	struct device *bootdv = NULL;
	dev_t tmpdev;
	char buf[128];
	extern bios_bootmac_t *bios_bootmac;

	dkcsumattach();

	if ((bootdev & B_MAGICMASK) == (u_int)B_DEVMAGIC) {
		majdev = B_TYPE(bootdev);
		unit = B_UNIT(bootdev);
		part = B_PARTITION(bootdev);
		snprintf(buf, sizeof buf, "%s%d%c", findblkname(majdev),
		    unit, part + 'a');
		bootdv = parsedisk(buf, strlen(buf), part, &tmpdev);
	}

	if (bios_bootmac) {
		struct ifnet *ifp;

		for (ifp = TAILQ_FIRST(&ifnet); ifp != NULL;
		    ifp = TAILQ_NEXT(ifp, if_list)) {
			if (ifp->if_type == IFT_ETHER &&
			    bcmp(bios_bootmac->mac,
			    ((struct arpcom *)ifp)->ac_enaddr,
			    ETHER_ADDR_LEN) == 0)
				break;
		}
		if (ifp) {
#if defined(NFSCLIENT)
			printf("PXE boot MAC address %s, interface %s\n",
			    ether_sprintf(bios_bootmac->mac), ifp->if_xname);
			bootdv = parsedisk(ifp->if_xname, strlen(ifp->if_xname),
			    0, &tmpdev);
			part = 0;
#endif
		} else
			printf("PXE boot MAC address %s, interface %s\n",
			    ether_sprintf(bios_bootmac->mac), "unknown");
	}

	setroot(bootdv, part, RB_USERREQ);
	dumpconf();

#ifdef HIBERNATE
	hibernate_resume();
#endif /* HIBERNATE */
}

struct nam2blk nam2blk[] = {
	{ "wd",		0 },
	{ "fd",		2 },
	{ "sd",		4 },
	{ "cd",		6 },
	{ "vnd",	14 },
	{ "rd",		17 },
	{ "raid",	19 },
	{ NULL,		-1 }
};


code:
/*	$OpenBSD: bios.c,v 1.31 2015/03/14 03:38:46 jsg Exp $	*/
/*
 * Copyright (c) 2006 Gordon Willem Klok <gklok@cogeco.ca>
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/device.h>
#include <sys/errno.h>
#include <sys/malloc.h>

#include <uvm/uvm_extern.h>
#include <sys/sysctl.h>

#include <machine/conf.h>
#include <machine/biosvar.h>
#include <machine/mpbiosvar.h>
#include <machine/smbiosvar.h>

#include <dev/isa/isareg.h>
#include <amd64/include/isa_machdep.h>
#include <dev/pci/pcivar.h>

#include <dev/acpi/acpivar.h>

#include <dev/rndvar.h>

#include "acpi.h"
#include "mpbios.h"
#include "pci.h"

struct bios_softc {
	struct device sc_dev;
};

void smbios_info(char *);
int bios_match(struct device *, void *, void *);
void bios_attach(struct device *, struct device *, void *);
int bios_print(void *, const char *);
char *fixstring(char *);

struct cfattach bios_ca = {
	sizeof(struct bios_softc), bios_match, bios_attach
};

struct cfdriver bios_cd = {
	NULL, "bios", DV_DULL
};

struct smbios_entry smbios_entry;
/*
 * used by hw_sysctl
 */
extern char *hw_vendor, *hw_prod, *hw_uuid, *hw_serial, *hw_ver;

const char *smbios_uninfo[] = {
	"System",
	"Not ",
	"To be",
	"SYS-"
};

int
bios_match(struct device *parent, void *match , void *aux)
{
	struct bios_attach_args *bia = aux;

	/* only one */
	if (bios_cd.cd_ndevs || strcmp(bia->ba_name, bios_cd.cd_name))
		return 0;
	return 1;
}

void
bios_attach(struct device *parent, struct device *self, void *aux)
{
	struct bios_softc *sc = (struct bios_softc *)self;
	struct smbios_struct_bios *sb;
	struct smbtable bios;
	char scratch[64];
	vaddr_t va;
	paddr_t pa, end;
	u_int8_t *p;
	int smbiosrev = 0;

	/* see if we have SMBIOS extentions */
	for (p = ISA_HOLE_VADDR(SMBIOS_START);
	    p < (u_int8_t *)ISA_HOLE_VADDR(SMBIOS_END); p+= 16) {
		struct smbhdr * hdr = (struct smbhdr *)p;
		u_int8_t chksum;
		int i;

		if (hdr->sig != SMBIOS_SIGNATURE)
			continue;
		i = hdr->len;
		for (chksum = 0; i--; chksum += p[i])
			;
		if (chksum != 0)
			continue;
		p += 0x10;
		if (p[0] != '_' && p[1] != 'D' && p[2] != 'M' &&
		    p[3] != 'I' && p[4] != '_')
			continue;
		for (chksum = 0, i = 0xf; i--; chksum += p[i])
			;
		if (chksum != 0)
			continue;

		pa = trunc_page(hdr->addr);
		end = round_page(hdr->addr + hdr->size);
		va = uvm_km_valloc(kernel_map, end-pa);
		if (va == 0)
			break;

		smbios_entry.addr = (u_int8_t *)(va + (hdr->addr & PGOFSET));
		smbios_entry.len = hdr->size;
		smbios_entry.mjr = hdr->majrev;
		smbios_entry.min = hdr->minrev;
		smbios_entry.count = hdr->count;

		for (; pa < end; pa+= NBPG, va+= NBPG)
			pmap_kenter_pa(va, pa, PROT_READ);

		printf(": SMBIOS rev. %d.%d @ 0x%x (%d entries)",
		    hdr->majrev, hdr->minrev, hdr->addr, hdr->count);

		smbiosrev = hdr->majrev * 100 + hdr->minrev;
		if (hdr->minrev < 10)
			smbiosrev = hdr->majrev * 100 + hdr->minrev * 10;

		bios.cookie = 0;
		if (smbios_find_table(SMBIOS_TYPE_BIOS, &bios)) {
			sb = bios.tblhdr;
			printf("\n%s:", sc->sc_dev.dv_xname);
			if ((smbios_get_string(&bios, sb->vendor,
			    scratch, sizeof(scratch))) != NULL)
				printf(" vendor %s",
				    fixstring(scratch));
			if ((smbios_get_string(&bios, sb->version,
			    scratch, sizeof(scratch))) != NULL)
				printf(" version \"%s\"",
				    fixstring(scratch));
			if ((smbios_get_string(&bios, sb->release,
			    scratch, sizeof(scratch))) != NULL)
				printf(" date %s", fixstring(scratch));
		}

		smbios_info(sc->sc_dev.dv_xname);
		break;
	}
	printf("\n");

	/* No SMBIOS extensions, go looking for Soekris comBIOS */
	if (smbiosrev == 0) {
		const char *signature = "Soekris Engineering";

		for (p = ISA_HOLE_VADDR(SMBIOS_START);
		    p <= (u_int8_t *)ISA_HOLE_VADDR(SMBIOS_END -
		    (strlen(signature) - 1)); p++)
			if (!memcmp(p, signature, strlen(signature))) {
				hw_vendor = malloc(strlen(signature) + 1,
				    M_DEVBUF, M_NOWAIT);
				if (hw_vendor)
					strlcpy(hw_vendor, signature,
					    strlen(signature) + 1);
				p += strlen(signature);
				break;
			}

		for (; hw_vendor &&
		    p <= (u_int8_t *)ISA_HOLE_VADDR(SMBIOS_END - 6); p++)
			/*
			 * Search only for "net6501" in the comBIOS as that's
			 * the only Soekris platform that can run amd64
			 */
			if (!memcmp(p, "net6501", 7)) {
				hw_prod = malloc(8, M_DEVBUF, M_NOWAIT);
				if (hw_prod) {
					memcpy(hw_prod, p, 7);
					hw_prod[7] = '\0';
				}
				break;
			}
	}

#if NACPI > 0
	{
		struct bios_attach_args ba;

		memset(&ba, 0, sizeof(ba));
		ba.ba_name = "acpi";
		ba.ba_iot = X86_BUS_SPACE_IO;
		ba.ba_memt = X86_BUS_SPACE_MEM;

		config_found(self, &ba, bios_print);
	}
#endif

#if NMPBIOS > 0
	if (mpbios_probe(self)) {
		struct bios_attach_args ba;

		memset(&ba, 0, sizeof(ba));
		ba.ba_name = "mpbios";
		ba.ba_iot = X86_BUS_SPACE_IO;
		ba.ba_memt = X86_BUS_SPACE_MEM;

		config_found(self, &ba, bios_print);
	}
#endif
}

/*
 * smbios_find_table() takes a caller supplied smbios struct type and
 * a pointer to a handle (struct smbtable) returning one if the structure
 * is successfully located and zero otherwise. Callers should take care
 * to initialize the cookie field of the smbtable structure to zero before
 * the first invocation of this function.
 * Multiple tables of the same type can be located by repeatedly calling
 * smbios_find_table with the same arguments.
 */
int
smbios_find_table(u_int8_t type, struct smbtable *st)
{
	u_int8_t *va, *end;
	struct smbtblhdr *hdr;
	int ret = 0, tcount = 1;

	va = smbios_entry.addr;
	end = va + smbios_entry.len;

	/*
	 * The cookie field of the smtable structure is used to locate
	 * multiple instances of a table of an arbitrary type. Following the
	 * successful location of a table, the type is encoded as bits 0:7 of
	 * the cookie value, the offset in terms of the number of structures
	 * preceding that referenced by the handle is encoded in bits 15:31.
	 */
	if ((st->cookie & 0xfff) == type && st->cookie >> 16) {
		if ((u_int8_t *)st->hdr >= va && (u_int8_t *)st->hdr < end) {
			hdr = st->hdr;
			if (hdr->type == type) {
				va = (u_int8_t *)hdr + hdr->size;
				for (; va + 1 < end; va++)
					if (*va == 0 && *(va + 1) == 0)
						break;
				va+= 2;
				tcount = st->cookie >> 16;
			}
		}
	}
	for (; va + sizeof(struct smbtblhdr) < end && tcount <=
	    smbios_entry.count; tcount++) {
		hdr = (struct smbtblhdr *)va;
		if (hdr->type == type) {
			ret = 1;
			st->hdr = hdr;
			st->tblhdr = va + sizeof(struct smbtblhdr);
			st->cookie = (tcount + 1) << 16 | type;
			break;
		}
		if (hdr->type == SMBIOS_TYPE_EOT)
			break;
		va+= hdr->size;
		for (; va + 1 < end; va++)
			if (*va == 0 && *(va + 1) == 0)
				break;
		va+=2;
	}
	return ret;
}

char *
smbios_get_string(struct smbtable *st, u_int8_t indx, char *dest, size_t len)
{
	u_int8_t *va, *end;
	char *ret = NULL;
	int i;

	va = (u_int8_t *)st->hdr + st->hdr->size;
	end = smbios_entry.addr + smbios_entry.len;
	for (i = 1; va < end && i < indx && *va; i++)
		while (*va++)
			;
	if (i == indx) {
		if (va + len < end) {
			ret = dest;
			memcpy(ret, va, len);
			ret[len-1] = '\0';
		}
	}

	return ret;
}

char *
fixstring(char *s)
{
	char *p, *e;
	int i;

	for (i = 0; i < nitems(smbios_uninfo); i++)
		if ((strncasecmp(s, smbios_uninfo[i],
		    strlen(smbios_uninfo[i])))==0)
			return NULL;
	/*
	 * Remove leading and trailing whitespace
	 */
	for (p = s; *p == ' '; p++)
		;
	/*
	 * Special case entire string is whitespace
	 */
	if (p == s + strlen(s))
		return NULL;
	for (e = s + strlen(s) - 1; e > s && *e == ' '; e--)
		;
	if (p > s || e < s + strlen(s) - 1) {
		memmove(s, p, e - p + 1);
		s[e - p + 1] = '\0';
	}

	return s;
}

void
smbios_info(char * str)
{
	char *sminfop, sminfo[64];
	struct smbtable stbl, btbl;
	struct smbios_sys *sys;
	struct smbios_board *board;
	int i, infolen, uuidf, havebb;
	char *p;

	if (smbios_entry.mjr < 2)
		return;
	/*
	 * According to the spec the system table among others is required,
	 * if it is not we do not bother with this smbios implementation.
	 */
	stbl.cookie = btbl.cookie = 0;
	if (!smbios_find_table(SMBIOS_TYPE_SYSTEM, &stbl))
		return;
	havebb = smbios_find_table(SMBIOS_TYPE_BASEBOARD, &btbl);

	sys = (struct smbios_sys *)stbl.tblhdr;
	if (havebb)
		board = (struct smbios_board *)btbl.tblhdr;
	/*
	 * Some smbios implementations have no system vendor or product strings,
	 * some have very uninformative data which is harder to work around
	 * and we must rely upon various heuristics to detect this. In both
	 * cases we attempt to fall back on the base board information in the
	 * perhaps naive belief that motherboard vendors will supply this
	 * information.
	 */
	sminfop = NULL;
	if ((p = smbios_get_string(&stbl, sys->vendor, sminfo,
	    sizeof(sminfo))) != NULL)
		sminfop = fixstring(p);
	if (sminfop == NULL) {
		if (havebb) {
			if ((p = smbios_get_string(&btbl, board->vendor,
			    sminfo, sizeof(sminfo))) != NULL)
				sminfop = fixstring(p);
		}
	}
	if (sminfop) {
		infolen = strlen(sminfop) + 1;
		hw_vendor = malloc(infolen, M_DEVBUF, M_NOWAIT);
		if (hw_vendor)
			strlcpy(hw_vendor, sminfop, infolen);
		sminfop = NULL;
	}
	if ((p = smbios_get_string(&stbl, sys->product, sminfo,
	    sizeof(sminfo))) != NULL)
		sminfop = fixstring(p);
	if (sminfop == NULL) {
		if (havebb) {
			if ((p = smbios_get_string(&btbl, board->product,
			    sminfo, sizeof(sminfo))) != NULL)
				sminfop = fixstring(p);
		}
	}
	if (sminfop) {
		infolen = strlen(sminfop) + 1;
		hw_prod = malloc(infolen, M_DEVBUF, M_NOWAIT);
		if (hw_prod)
			strlcpy(hw_prod, sminfop, infolen);
		sminfop = NULL;
	}
	if (hw_vendor != NULL && hw_prod != NULL)
		printf("\n%s: %s %s", str, hw_vendor, hw_prod);
	if ((p = smbios_get_string(&stbl, sys->version, sminfo,
	    sizeof(sminfo))) != NULL)
		sminfop = fixstring(p);
	if (sminfop) {
		infolen = strlen(sminfop) + 1;
		hw_ver = malloc(infolen, M_DEVBUF, M_NOWAIT);
		if (hw_ver)
			strlcpy(hw_ver, sminfop, infolen);
		sminfop = NULL;
	}
	if ((p = smbios_get_string(&stbl, sys->serial, sminfo,
	    sizeof(sminfo))) != NULL)
		sminfop = fixstring(p);
	if (sminfop) {
		infolen = strlen(sminfop) + 1;
		for (i = 0; i < infolen - 1; i++)
			add_timer_randomness(sminfop[i]);
		hw_serial = malloc(infolen, M_DEVBUF, M_NOWAIT);
		if (hw_serial)
			strlcpy(hw_serial, sminfop, infolen);
	}
	if (smbios_entry.mjr > 2 || (smbios_entry.mjr == 2 &&
	    smbios_entry.min >= 1)) {
		/*
		 * If the uuid value is all 0xff the uuid is present but not
		 * set, if its all 0 then the uuid isn't present at all.
		 */
		uuidf = SMBIOS_UUID_NPRESENT|SMBIOS_UUID_NSET;
		for (i = 0; i < sizeof(sys->uuid); i++) {
			if (sys->uuid[i] != 0xff)
				uuidf &= ~SMBIOS_UUID_NSET;
			if (sys->uuid[i] != 0)
				uuidf &= ~SMBIOS_UUID_NPRESENT;
		}

		if (uuidf & SMBIOS_UUID_NPRESENT)
			hw_uuid = NULL;
		else if (uuidf & SMBIOS_UUID_NSET)
			hw_uuid = "Not Set";
		else {
			for (i = 0; i < sizeof(sys->uuid); i++)
				add_timer_randomness(sys->uuid[i]);
			hw_uuid = malloc(SMBIOS_UUID_REPLEN, M_DEVBUF,
			    M_NOWAIT);
			if (hw_uuid) {
				snprintf(hw_uuid, SMBIOS_UUID_REPLEN,
				    SMBIOS_UUID_REP,
				    sys->uuid[0], sys->uuid[1], sys->uuid[2],
				    sys->uuid[3], sys->uuid[4], sys->uuid[5],
				    sys->uuid[6], sys->uuid[7], sys->uuid[8],
				    sys->uuid[9], sys->uuid[10], sys->uuid[11],
				    sys->uuid[12], sys->uuid[13], sys->uuid[14],
				    sys->uuid[15]);
			}
		}
	}
}

int
bios_print(void *aux, const char *pnp)
{
        struct bios_attach_args *ba = aux;

        if (pnp)
                printf("%s at %s",
                    ba->ba_name, pnp);
        return (UNCONF);
}


code:
/*	$OpenBSD: bus_dma.c,v 1.48 2015/01/27 05:10:30 dlg Exp $	*/
/*	$NetBSD: bus_dma.c,v 1.3 2003/05/07 21:33:58 fvdl Exp $	*/

/*-
 * Copyright (c) 1996, 1997, 1998 The NetBSD Foundation, Inc.
 * All rights reserved.
 *
 * This code is derived from software contributed to The NetBSD Foundation
 * by Charles M. Hannum and by Jason R. Thorpe of the Numerical Aerospace
 * Simulation Facility, NASA Ames Research Center.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

/*
 * The following is included because _bus_dma_uiomove is derived from
 * uiomove() in kern_subr.c.
 */

/*
 * Copyright (c) 1982, 1986, 1991, 1993
 *	The Regents of the University of California.  All rights reserved.
 * (c) UNIX System Laboratories, Inc.
 * All or some portions of this file are derived from material licensed
 * to the University of California by American Telephone and Telegraph
 * Co. or Unix System Laboratories, Inc. and are reproduced herein with
 * the permission of UNIX System Laboratories, Inc.
 *
 * Copyright (c) 1992, 1993
 *	The Regents of the University of California.  All rights reserved.
 *
 * This software was developed by the Computer Systems Engineering group
 * at Lawrence Berkeley Laboratory under DARPA contract BG 91-66 and
 * contributed to Berkeley.
 *
 * All advertising materials mentioning features or use of this software
 * must display the following acknowledgement:
 *	This product includes software developed by the University of
 *	California, Lawrence Berkeley Laboratory.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. All advertising materials mentioning features or use of this software
 *    must display the following acknowledgement:
 *	This product includes software developed by the University of
 *	California, Berkeley and its contributors.
 * 4. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/malloc.h>
#include <sys/mbuf.h>
#include <sys/proc.h>

#include <machine/bus.h>

#include <uvm/uvm_extern.h>

int _bus_dmamap_load_buffer(bus_dma_tag_t, bus_dmamap_t, void *, bus_size_t,
    struct proc *, int, paddr_t *, int *, int);

/*
 * Common function for DMA map creation.  May be called by bus-specific
 * DMA map creation functions.
 */
int
_bus_dmamap_create(bus_dma_tag_t t, bus_size_t size, int nsegments,
    bus_size_t maxsegsz, bus_size_t boundary, int flags, bus_dmamap_t *dmamp)
{
	struct bus_dmamap *map;
	void *mapstore;
	size_t mapsize;

	/*
	 * Allocate and initialize the DMA map.  The end of the map
	 * is a variable-sized array of segments, so we allocate enough
	 * room for them in one shot.
	 *
	 * Note we don't preserve the WAITOK or NOWAIT flags.  Preservation
	 * of ALLOCNOW notifies others that we've reserved these resources,
	 * and they are not to be freed.
	 *
	 * The bus_dmamap_t includes one bus_dma_segment_t, hence
	 * the (nsegments - 1).
	 */
	mapsize = sizeof(struct bus_dmamap) +
	    (sizeof(bus_dma_segment_t) * (nsegments - 1));
	if ((mapstore = malloc(mapsize, M_DEVBUF,
	    (flags & BUS_DMA_NOWAIT) ?
	        (M_NOWAIT|M_ZERO) : (M_WAITOK|M_ZERO))) == NULL)
		return (ENOMEM);

	map = (struct bus_dmamap *)mapstore;
	map->_dm_size = size;
	map->_dm_segcnt = nsegments;
	map->_dm_maxsegsz = maxsegsz;
	map->_dm_boundary = boundary;
	map->_dm_flags = flags & ~(BUS_DMA_WAITOK|BUS_DMA_NOWAIT);

	*dmamp = map;
	return (0);
}

/*
 * Common function for DMA map destruction.  May be called by bus-specific
 * DMA map destruction functions.
 */
void
_bus_dmamap_destroy(bus_dma_tag_t t, bus_dmamap_t map)
{

	free(map, M_DEVBUF, 0);
}

/*
 * Common function for loading a DMA map with a linear buffer.  May
 * be called by bus-specific DMA map load functions.
 */
int
_bus_dmamap_load(bus_dma_tag_t t, bus_dmamap_t map, void *buf,
    bus_size_t buflen, struct proc *p, int flags)
{
	bus_addr_t lastaddr = 0;
	int seg, error;

	/*
	 * Make sure that on error condition we return "no valid mappings".
	 */
	map->dm_mapsize = 0;
	map->dm_nsegs = 0;

	if (buflen > map->_dm_size)
		return (EINVAL);

	seg = 0;
	error = _bus_dmamap_load_buffer(t, map, buf, buflen, p, flags,
	    &lastaddr, &seg, 1);
	if (error == 0) {
		map->dm_mapsize = buflen;
		map->dm_nsegs = seg + 1;
	}
	return (error);
}

/*
 * Like _bus_dmamap_load(), but for mbufs.
 */
int
_bus_dmamap_load_mbuf(bus_dma_tag_t t, bus_dmamap_t map, struct mbuf *m0,
    int flags)
{
	paddr_t lastaddr = 0;
	int seg, error, first;
	struct mbuf *m;

	/*
	 * Make sure that on error condition we return "no valid mappings".
	 */
	map->dm_mapsize = 0;
	map->dm_nsegs = 0;

#ifdef DIAGNOSTIC
	if ((m0->m_flags & M_PKTHDR) == 0)
		panic("_bus_dmamap_load_mbuf: no packet header");
#endif

	if (m0->m_pkthdr.len > map->_dm_size)
		return (EINVAL);

	first = 1;
	seg = 0;
	error = 0;
	for (m = m0; m != NULL && error == 0; m = m->m_next) {
		if (m->m_len == 0)
			continue;
		error = _bus_dmamap_load_buffer(t, map, m->m_data, m->m_len,
		    NULL, flags, &lastaddr, &seg, first);
		first = 0;
	}
	if (error == 0) {
		map->dm_mapsize = m0->m_pkthdr.len;
		map->dm_nsegs = seg + 1;
	}
	return (error);
}

/*
 * Like _bus_dmamap_load(), but for uios.
 */
int
_bus_dmamap_load_uio(bus_dma_tag_t t, bus_dmamap_t map, struct uio *uio,
    int flags)
{
	paddr_t lastaddr = 0;
	int seg, i, error, first;
	bus_size_t minlen, resid;
	struct proc *p = NULL;
	struct iovec *iov;
	caddr_t addr;

	/*
	 * Make sure that on error condition we return "no valid mappings".
	 */
	map->dm_mapsize = 0;
	map->dm_nsegs = 0;

	resid = uio->uio_resid;
	iov = uio->uio_iov;

	if (uio->uio_segflg == UIO_USERSPACE) {
		p = uio->uio_procp;
#ifdef DIAGNOSTIC
		if (p == NULL)
			panic("_bus_dmamap_load_uio: USERSPACE but no proc");
#endif
	}

	first = 1;
	seg = 0;
	error = 0;
	for (i = 0; i < uio->uio_iovcnt && resid != 0 && error == 0; i++) {
		/*
		 * Now at the first iovec to load.  Load each iovec
		 * until we have exhausted the residual count.
		 */
		minlen = resid < iov[i].iov_len ? resid : iov[i].iov_len;
		addr = (caddr_t)iov[i].iov_base;

		error = _bus_dmamap_load_buffer(t, map, addr, minlen,
		    p, flags, &lastaddr, &seg, first);
		first = 0;

		resid -= minlen;
	}
	if (error == 0) {
		map->dm_mapsize = uio->uio_resid;
		map->dm_nsegs = seg + 1;
	}
	return (error);
}

/*
 * Like _bus_dmamap_load(), but for raw memory allocated with
 * bus_dmamem_alloc().
 */
int
_bus_dmamap_load_raw(bus_dma_tag_t t, bus_dmamap_t map, bus_dma_segment_t *segs,
    int nsegs, bus_size_t size, int flags)
{
	bus_addr_t paddr, baddr, bmask, lastaddr = 0;
	bus_size_t plen, sgsize, mapsize;
	int first = 1;
	int i, seg = 0;

	/*
	 * Make sure that on error condition we return "no valid mappings".
	 */
	map->dm_mapsize = 0;
	map->dm_nsegs = 0;

	if (nsegs > map->_dm_segcnt || size > map->_dm_size)
		return (EINVAL);

	mapsize = size;
	bmask  = ~(map->_dm_boundary - 1);

	for (i = 0; i < nsegs && size > 0; i++) {
		paddr = segs[i].ds_addr;
		plen = MIN(segs[i].ds_len, size);

		while (plen > 0) {
			/*
			 * Compute the segment size, and adjust counts.
			 */
			sgsize = PAGE_SIZE - ((u_long)paddr & PGOFSET);
			if (plen < sgsize)
				sgsize = plen;

			if (paddr > dma_constraint.ucr_high)
				panic("Non dma-reachable buffer at paddr %#lx(raw)",
				    paddr);

			/*
			 * Make sure we don't cross any boundaries.
			 */
			if (map->_dm_boundary > 0) {
				baddr = (paddr + map->_dm_boundary) & bmask;
				if (sgsize > (baddr - paddr))
					sgsize = (baddr - paddr);
			}

			/*
			 * Insert chunk into a segment, coalescing with
			 * previous segment if possible.
			 */
			if (first) {
				map->dm_segs[seg].ds_addr = paddr;
				map->dm_segs[seg].ds_len = sgsize;
				first = 0;
			} else {
				if (paddr == lastaddr &&
				    (map->dm_segs[seg].ds_len + sgsize) <=
				     map->_dm_maxsegsz &&
				    (map->_dm_boundary == 0 ||
				     (map->dm_segs[seg].ds_addr & bmask) ==
				     (paddr & bmask)))
					map->dm_segs[seg].ds_len += sgsize;
				else {
					if (++seg >= map->_dm_segcnt)
						return (EINVAL);
					map->dm_segs[seg].ds_addr = paddr;
					map->dm_segs[seg].ds_len = sgsize;
				}
			}

			paddr += sgsize;
			plen -= sgsize;
			size -= sgsize;

			lastaddr = paddr;
		}
	}

	map->dm_mapsize = mapsize;
	map->dm_nsegs = seg + 1;
	return (0);
}

/*
 * Common function for unloading a DMA map.  May be called by
 * bus-specific DMA map unload functions.
 */
void
_bus_dmamap_unload(bus_dma_tag_t t, bus_dmamap_t map)
{
	/*
	 * No resources to free; just mark the mappings as
	 * invalid.
	 */
	map->dm_mapsize = 0;
	map->dm_nsegs = 0;
}

/*
 * Common function for DMA map synchronization.  May be called
 * by bus-specific DMA map synchronization functions.
 */
void
_bus_dmamap_sync(bus_dma_tag_t t, bus_dmamap_t map, bus_addr_t addr,
    bus_size_t size, int op)
{
	/* Nothing to do here. */
}

/*
 * Common function for DMA-safe memory allocation.  May be called
 * by bus-specific DMA memory allocation functions.
 */
int
_bus_dmamem_alloc(bus_dma_tag_t t, bus_size_t size, bus_size_t alignment,
    bus_size_t boundary, bus_dma_segment_t *segs, int nsegs, int *rsegs,
    int flags)
{

	/*
	 * XXX in the presence of decent (working) iommus and bouncebuffers
	 * we can then fallback this allocation to a range of { 0, -1 }.
	 * However for now  we err on the side of caution and allocate dma
	 * memory under the 4gig boundary.
	 */
	return (_bus_dmamem_alloc_range(t, size, alignment, boundary,
	    segs, nsegs, rsegs, flags, (bus_addr_t)0, (bus_addr_t)0xffffffff));
}

/*
 * Common function for freeing DMA-safe memory.  May be called by
 * bus-specific DMA memory free functions.
 */
void
_bus_dmamem_free(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs)
{
	struct vm_page *m;
	bus_addr_t addr;
	struct pglist mlist;
	int curseg;

	/*
	 * Build a list of pages to free back to the VM system.
	 */
	TAILQ_INIT(&mlist);
	for (curseg = 0; curseg < nsegs; curseg++) {
		for (addr = segs[curseg].ds_addr;
		    addr < (segs[curseg].ds_addr + segs[curseg].ds_len);
		    addr += PAGE_SIZE) {
			m = PHYS_TO_VM_PAGE(addr);
			TAILQ_INSERT_TAIL(&mlist, m, pageq);
		}
	}

	uvm_pglistfree(&mlist);
}

/*
 * Common function for mapping DMA-safe memory.  May be called by
 * bus-specific DMA memory map functions.
 */
int
_bus_dmamem_map(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs,
    size_t size, caddr_t *kvap, int flags)
{
	vaddr_t va, sva;
	size_t ssize;
	bus_addr_t addr;
	int curseg, pmapflags = 0, error;
	const struct kmem_dyn_mode *kd;

	if (nsegs == 1 && (flags & BUS_DMA_NOCACHE) == 0) {
		*kvap = (caddr_t)PMAP_DIRECT_MAP(segs[0].ds_addr);
		return (0);
	}

	if (flags & BUS_DMA_NOCACHE)
		pmapflags |= PMAP_NOCACHE;

	size = round_page(size);
	kd = flags & BUS_DMA_NOWAIT ? &kd_trylock : &kd_waitok;
	va = (vaddr_t)km_alloc(size, &kv_any, &kp_none, kd);
	if (va == 0)
		return (ENOMEM);

	*kvap = (caddr_t)va;

	sva = va;
	ssize = size;
	for (curseg = 0; curseg < nsegs; curseg++) {
		for (addr = segs[curseg].ds_addr;
		    addr < (segs[curseg].ds_addr + segs[curseg].ds_len);
		    addr += PAGE_SIZE, va += PAGE_SIZE, size -= PAGE_SIZE) {
			if (size == 0)
				panic("_bus_dmamem_map: size botch");
			error = pmap_enter(pmap_kernel(), va, addr | pmapflags,
			    PROT_READ | PROT_WRITE,
			    PROT_READ | PROT_WRITE | PMAP_WIRED | PMAP_CANFAIL);
			if (error) {
				pmap_update(pmap_kernel());
				km_free((void *)sva, ssize, &kv_any, &kp_none);
				return (error);
			}
		}
	}
	pmap_update(pmap_kernel());

	return (0);
}

/*
 * Common function for unmapping DMA-safe memory.  May be called by
 * bus-specific DMA memory unmapping functions.
 */
void
_bus_dmamem_unmap(bus_dma_tag_t t, caddr_t kva, size_t size)
{

#ifdef DIAGNOSTIC
	if ((u_long)kva & PGOFSET)
		panic("_bus_dmamem_unmap");
#endif
	if (kva >= (caddr_t)PMAP_DIRECT_BASE && kva <= (caddr_t)PMAP_DIRECT_END)
		return;

	km_free(kva, round_page(size), &kv_any, &kp_none);
}

/*
 * Common function for mmap(2)'ing DMA-safe memory.  May be called by
 * bus-specific DMA mmap(2)'ing functions.
 */
paddr_t
_bus_dmamem_mmap(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs, off_t off,
    int prot, int flags)
{
	int i, pmapflags = 0;

	if (flags & BUS_DMA_NOCACHE)
		pmapflags |= PMAP_NOCACHE;

	for (i = 0; i < nsegs; i++) {
#ifdef DIAGNOSTIC
		if (off & PGOFSET)
			panic("_bus_dmamem_mmap: offset unaligned");
		if (segs[i].ds_addr & PGOFSET)
			panic("_bus_dmamem_mmap: segment unaligned");
		if (segs[i].ds_len & PGOFSET)
			panic("_bus_dmamem_mmap: segment size not multiple"
			    " of page size");
#endif
		if (off >= segs[i].ds_len) {
			off -= segs[i].ds_len;
			continue;
		}

		return ((segs[i].ds_addr + off) | pmapflags);
	}

	/* Page not found. */
	return (-1);
}

/**********************************************************************
 * DMA utility functions
 **********************************************************************/
/*
 * Utility function to load a linear buffer.  lastaddrp holds state
 * between invocations (for multiple-buffer loads).  segp contains
 * the starting segment on entrance, and the ending segment on exit.
 * first indicates if this is the first invocation of this function.
 */
int
_bus_dmamap_load_buffer(bus_dma_tag_t t, bus_dmamap_t map, void *buf,
    bus_size_t buflen, struct proc *p, int flags, paddr_t *lastaddrp, int *segp,
    int first)
{
	bus_size_t sgsize;
	bus_addr_t curaddr, lastaddr, baddr, bmask;
	vaddr_t vaddr = (vaddr_t)buf;
	int seg;
	pmap_t pmap;

	if (p != NULL)
		pmap = p->p_vmspace->vm_map.pmap;
	else
		pmap = pmap_kernel();

	lastaddr = *lastaddrp;
	bmask  = ~(map->_dm_boundary - 1);

	for (seg = *segp; buflen > 0 ; ) {
		/*
		 * Get the physical address for this segment.
		 */
		pmap_extract(pmap, vaddr, (paddr_t *)&curaddr);

		if (curaddr > dma_constraint.ucr_high)
			panic("Non dma-reachable buffer at curaddr %#lx(raw)",
			    curaddr);

		/*
		 * Compute the segment size, and adjust counts.
		 */
		sgsize = PAGE_SIZE - ((u_long)vaddr & PGOFSET);
		if (buflen < sgsize)
			sgsize = buflen;

		/*
		 * Make sure we don't cross any boundaries.
		 */
		if (map->_dm_boundary > 0) {
			baddr = (curaddr + map->_dm_boundary) & bmask;
			if (sgsize > (baddr - curaddr))
				sgsize = (baddr - curaddr);
		}

		/*
		 * Insert chunk into a segment, coalescing with
		 * previous segment if possible.
		 */
		if (first) {
			map->dm_segs[seg].ds_addr = curaddr;
			map->dm_segs[seg].ds_len = sgsize;
			first = 0;
		} else {
			if (curaddr == lastaddr &&
			    (map->dm_segs[seg].ds_len + sgsize) <=
			     map->_dm_maxsegsz &&
			    (map->_dm_boundary == 0 ||
			     (map->dm_segs[seg].ds_addr & bmask) ==
			     (curaddr & bmask)))
				map->dm_segs[seg].ds_len += sgsize;
			else {
				if (++seg >= map->_dm_segcnt)
					break;
				map->dm_segs[seg].ds_addr = curaddr;
				map->dm_segs[seg].ds_len = sgsize;
			}
		}

		lastaddr = curaddr + sgsize;
		vaddr += sgsize;
		buflen -= sgsize;
	}

	*segp = seg;
	*lastaddrp = lastaddr;

	/*
	 * Did we fit?
	 */
	if (buflen != 0)
		return (EFBIG);		/* XXX better return value here? */
	return (0);
}

/*
 * Allocate physical memory from the given physical address range.
 * Called by DMA-safe memory allocation methods.
 */
int
_bus_dmamem_alloc_range(bus_dma_tag_t t, bus_size_t size, bus_size_t alignment,
    bus_size_t boundary, bus_dma_segment_t *segs, int nsegs, int *rsegs,
    int flags, bus_addr_t low, bus_addr_t high)
{
	paddr_t curaddr, lastaddr;
	struct vm_page *m;
	struct pglist mlist;
	int curseg, error, plaflag;

	/* Always round the size. */
	size = round_page(size);

	segs[0]._ds_boundary = boundary;
	segs[0]._ds_align = alignment;

	/*
	 * Allocate pages from the VM system.
	 */
	plaflag = flags & BUS_DMA_NOWAIT ? UVM_PLA_NOWAIT : UVM_PLA_WAITOK;
	if (flags & BUS_DMA_ZERO)
		plaflag |= UVM_PLA_ZERO;

	TAILQ_INIT(&mlist);
	error = uvm_pglistalloc(size, low, high, alignment, boundary,
	    &mlist, nsegs, plaflag);
	if (error)
		return (error);

	/*
	 * Compute the location, size, and number of segments actually
	 * returned by the VM code.
	 */
	m = TAILQ_FIRST(&mlist);
	curseg = 0;
	lastaddr = segs[curseg].ds_addr = VM_PAGE_TO_PHYS(m);
	segs[curseg].ds_len = PAGE_SIZE;

	for (m = TAILQ_NEXT(m, pageq); m != NULL; m = TAILQ_NEXT(m, pageq)) {
		curaddr = VM_PAGE_TO_PHYS(m);
#ifdef DIAGNOSTIC
		if (curseg == nsegs) {
			printf("uvm_pglistalloc returned too many\n");
			panic("_bus_dmamem_alloc_range");
		}
		if (curaddr < low || curaddr >= high) {
			printf("uvm_pglistalloc returned non-sensical"
			    " address 0x%lx\n", curaddr);
			panic("_bus_dmamem_alloc_range");
		}
#endif
		if (curaddr == (lastaddr + PAGE_SIZE))
			segs[curseg].ds_len += PAGE_SIZE;
		else {
			curseg++;
			segs[curseg].ds_addr = curaddr;
			segs[curseg].ds_len = PAGE_SIZE;
		}
		lastaddr = curaddr;
	}

	*rsegs = curseg + 1;

	return (0);
}