code:
/*	$OpenBSD: bus_space.c,v 1.26 2015/04/25 21:31:24 guenther Exp $	*/
/*	$NetBSD: bus_space.c,v 1.2 2003/03/14 18:47:53 christos Exp $	*/

/*-
 * Copyright (c) 1996, 1997, 1998 The NetBSD Foundation, Inc.
 * All rights reserved.
 *
 * This code is derived from software contributed to The NetBSD Foundation
 * by Charles M. Hannum and by Jason R. Thorpe of the Numerical Aerospace
 * Simulation Facility, NASA Ames Research Center.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/malloc.h>
#include <sys/extent.h>

#include <uvm/uvm_extern.h>

#include <machine/bus.h>

#include <dev/isa/isareg.h>
#include <machine/isa_machdep.h>

/*
 * Extent maps to manage I/O and memory space.  Allocate
 * storage for 16 regions in each, initially.  Later, ioport_malloc_safe
 * will indicate that it's safe to use malloc() to dynamically allocate
 * region descriptors.
 *
 * N.B. At least two regions are _always_ allocated from the iomem
 * extent map; (0 -> ISA hole) and (end of ISA hole -> end of RAM).
 *
 * The extent maps are not static!  Machine-dependent ISA and EISA
 * routines need access to them for bus address space allocation.
 */
static	long ioport_ex_storage[EXTENT_FIXED_STORAGE_SIZE(16) / sizeof(long)];
static	long iomem_ex_storage[EXTENT_FIXED_STORAGE_SIZE(16) / sizeof(long)];
struct	extent *ioport_ex;
struct	extent *iomem_ex;
static	int ioport_malloc_safe;

int	x86_mem_add_mapping(bus_addr_t, bus_size_t,
	    int, bus_space_handle_t *);

u_int8_t	x86_bus_space_io_read_1(bus_space_handle_t, bus_size_t);
u_int16_t	x86_bus_space_io_read_2(bus_space_handle_t, bus_size_t);
u_int32_t	x86_bus_space_io_read_4(bus_space_handle_t, bus_size_t);
u_int64_t	x86_bus_space_io_read_8(bus_space_handle_t, bus_size_t);

void		x86_bus_space_io_read_multi_1(bus_space_handle_t, bus_size_t,
		    u_int8_t *, bus_size_t);
void		x86_bus_space_io_read_multi_2(bus_space_handle_t, bus_size_t,
		    u_int16_t *, bus_size_t);
void		x86_bus_space_io_read_multi_4(bus_space_handle_t, bus_size_t,
		    u_int32_t *, bus_size_t);
void		x86_bus_space_io_read_multi_8(bus_space_handle_t, bus_size_t,
		    u_int64_t *, bus_size_t);

void		x86_bus_space_io_read_region_1(bus_space_handle_t, bus_size_t,
		    u_int8_t *, bus_size_t);
void		x86_bus_space_io_read_region_2(bus_space_handle_t, bus_size_t,
		    u_int16_t *, bus_size_t);
void		x86_bus_space_io_read_region_4(bus_space_handle_t, bus_size_t,
		    u_int32_t *, bus_size_t);
void		x86_bus_space_io_read_region_8(bus_space_handle_t, bus_size_t,
		    u_int64_t *, bus_size_t);

void		x86_bus_space_io_write_1(bus_space_handle_t, bus_size_t,
		    u_int8_t);
void		x86_bus_space_io_write_2(bus_space_handle_t, bus_size_t,
		    u_int16_t);
void		x86_bus_space_io_write_4(bus_space_handle_t, bus_size_t,
		    u_int32_t);
void		x86_bus_space_io_write_8(bus_space_handle_t, bus_size_t,
		    u_int64_t);

void		x86_bus_space_io_write_multi_1(bus_space_handle_t,
		    bus_size_t, const u_int8_t *, bus_size_t);
void		x86_bus_space_io_write_multi_2(bus_space_handle_t,
		    bus_size_t, const u_int16_t *, bus_size_t);
void		x86_bus_space_io_write_multi_4(bus_space_handle_t,
		    bus_size_t, const u_int32_t *, bus_size_t);
void		x86_bus_space_io_write_multi_8(bus_space_handle_t,
		    bus_size_t, const u_int64_t *, bus_size_t);

void		x86_bus_space_io_write_region_1(bus_space_handle_t,
		    bus_size_t, const u_int8_t *, bus_size_t);
void		x86_bus_space_io_write_region_2(bus_space_handle_t,
		    bus_size_t, const u_int16_t *, bus_size_t);
void		x86_bus_space_io_write_region_4(bus_space_handle_t,
		    bus_size_t, const u_int32_t *, bus_size_t);
void		x86_bus_space_io_write_region_8(bus_space_handle_t,
		    bus_size_t, const u_int64_t *, bus_size_t);

void		x86_bus_space_io_set_multi_1(bus_space_handle_t, bus_size_t,
		    u_int8_t, size_t);
void		x86_bus_space_io_set_multi_2(bus_space_handle_t, bus_size_t,
		    u_int16_t, size_t);
void		x86_bus_space_io_set_multi_4(bus_space_handle_t, bus_size_t,
		    u_int32_t, size_t);
void		x86_bus_space_io_set_multi_8(bus_space_handle_t, bus_size_t,
		    u_int64_t, size_t);

void		x86_bus_space_io_set_region_1(bus_space_handle_t, bus_size_t,
		    u_int8_t, size_t);
void		x86_bus_space_io_set_region_2(bus_space_handle_t, bus_size_t,
		    u_int16_t, size_t);
void		x86_bus_space_io_set_region_4(bus_space_handle_t, bus_size_t,
		    u_int32_t, size_t);
void		x86_bus_space_io_set_region_8(bus_space_handle_t, bus_size_t,
		    u_int64_t, size_t);

void		x86_bus_space_io_copy_1(bus_space_handle_t, bus_size_t,
		    bus_space_handle_t, bus_size_t, size_t);
void		x86_bus_space_io_copy_2(bus_space_handle_t, bus_size_t,
		    bus_space_handle_t, bus_size_t, size_t);
void		x86_bus_space_io_copy_4(bus_space_handle_t, bus_size_t,
		    bus_space_handle_t, bus_size_t, size_t);
void		x86_bus_space_io_copy_8(bus_space_handle_t, bus_size_t,
		    bus_space_handle_t, bus_size_t, size_t);

void *		x86_bus_space_io_vaddr(bus_space_handle_t);
paddr_t		x86_bus_space_io_mmap(bus_addr_t, off_t, int, int);

const struct x86_bus_space_ops x86_bus_space_io_ops = {
	x86_bus_space_io_read_1,
	x86_bus_space_io_read_2,
	x86_bus_space_io_read_4,
	x86_bus_space_io_read_8,
	x86_bus_space_io_read_multi_1,
	x86_bus_space_io_read_multi_2,
	x86_bus_space_io_read_multi_4,
	x86_bus_space_io_read_multi_8,
	x86_bus_space_io_read_region_1,
	x86_bus_space_io_read_region_2,
	x86_bus_space_io_read_region_4,
	x86_bus_space_io_read_region_8,
	x86_bus_space_io_write_1,
	x86_bus_space_io_write_2,
	x86_bus_space_io_write_4,
	x86_bus_space_io_write_8,
	x86_bus_space_io_write_multi_1,
	x86_bus_space_io_write_multi_2,
	x86_bus_space_io_write_multi_4,
	x86_bus_space_io_write_multi_8,
	x86_bus_space_io_write_region_1,
	x86_bus_space_io_write_region_2,
	x86_bus_space_io_write_region_4,
	x86_bus_space_io_write_region_8,
	x86_bus_space_io_set_multi_1,
	x86_bus_space_io_set_multi_2,
	x86_bus_space_io_set_multi_4,
	x86_bus_space_io_set_multi_8,
	x86_bus_space_io_set_region_1,
	x86_bus_space_io_set_region_2,
	x86_bus_space_io_set_region_4,
	x86_bus_space_io_set_region_8,
	x86_bus_space_io_copy_1,
	x86_bus_space_io_copy_2,
	x86_bus_space_io_copy_4,
	x86_bus_space_io_copy_8,
	x86_bus_space_io_vaddr,
	x86_bus_space_io_mmap
};

u_int8_t	x86_bus_space_mem_read_1(bus_space_handle_t, bus_size_t);
u_int16_t	x86_bus_space_mem_read_2(bus_space_handle_t, bus_size_t);
u_int32_t	x86_bus_space_mem_read_4(bus_space_handle_t, bus_size_t);
u_int64_t	x86_bus_space_mem_read_8(bus_space_handle_t, bus_size_t);

void		x86_bus_space_mem_read_multi_1(bus_space_handle_t, bus_size_t,
		    u_int8_t *, bus_size_t);
void		x86_bus_space_mem_read_multi_2(bus_space_handle_t, bus_size_t,
		    u_int16_t *, bus_size_t);
void		x86_bus_space_mem_read_multi_4(bus_space_handle_t, bus_size_t,
		    u_int32_t *, bus_size_t);
void		x86_bus_space_mem_read_multi_8(bus_space_handle_t, bus_size_t,
		    u_int64_t *, bus_size_t);

void		x86_bus_space_mem_read_region_1(bus_space_handle_t, bus_size_t,
		    u_int8_t *, bus_size_t);
void		x86_bus_space_mem_read_region_2(bus_space_handle_t, bus_size_t,
		    u_int16_t *, bus_size_t);
void		x86_bus_space_mem_read_region_4(bus_space_handle_t, bus_size_t,
		    u_int32_t *, bus_size_t);
void		x86_bus_space_mem_read_region_8(bus_space_handle_t, bus_size_t,
		    u_int64_t *, bus_size_t);

void		x86_bus_space_mem_write_1(bus_space_handle_t, bus_size_t,
		    u_int8_t);
void		x86_bus_space_mem_write_2(bus_space_handle_t, bus_size_t,
		    u_int16_t);
void		x86_bus_space_mem_write_4(bus_space_handle_t, bus_size_t,
		    u_int32_t);
void		x86_bus_space_mem_write_8(bus_space_handle_t, bus_size_t,
		    u_int64_t);

void		x86_bus_space_mem_write_multi_1(bus_space_handle_t,
		    bus_size_t, const u_int8_t *, bus_size_t);
void		x86_bus_space_mem_write_multi_2(bus_space_handle_t,
		    bus_size_t, const u_int16_t *, bus_size_t);
void		x86_bus_space_mem_write_multi_4(bus_space_handle_t,
		    bus_size_t, const u_int32_t *, bus_size_t);
void		x86_bus_space_mem_write_multi_8(bus_space_handle_t,
		    bus_size_t, const u_int64_t *, bus_size_t);

void		x86_bus_space_mem_write_region_1(bus_space_handle_t,
		    bus_size_t, const u_int8_t *, bus_size_t);
void		x86_bus_space_mem_write_region_2(bus_space_handle_t,
		    bus_size_t, const u_int16_t *, bus_size_t);
void		x86_bus_space_mem_write_region_4(bus_space_handle_t,
		    bus_size_t, const u_int32_t *, bus_size_t);
void		x86_bus_space_mem_write_region_8(bus_space_handle_t,
		    bus_size_t, const u_int64_t *, bus_size_t);

void		x86_bus_space_mem_set_multi_1(bus_space_handle_t, bus_size_t,
		    u_int8_t, size_t);
void		x86_bus_space_mem_set_multi_2(bus_space_handle_t, bus_size_t,
		    u_int16_t, size_t);
void		x86_bus_space_mem_set_multi_4(bus_space_handle_t, bus_size_t,
		    u_int32_t, size_t);
void		x86_bus_space_mem_set_multi_8(bus_space_handle_t, bus_size_t,
		    u_int64_t, size_t);

void		x86_bus_space_mem_set_region_1(bus_space_handle_t, bus_size_t,
		    u_int8_t, size_t);
void		x86_bus_space_mem_set_region_2(bus_space_handle_t, bus_size_t,
		    u_int16_t, size_t);
void		x86_bus_space_mem_set_region_4(bus_space_handle_t, bus_size_t,
		    u_int32_t, size_t);
void		x86_bus_space_mem_set_region_8(bus_space_handle_t, bus_size_t,
		    u_int64_t, size_t);

void		x86_bus_space_mem_copy_1(bus_space_handle_t, bus_size_t,
		    bus_space_handle_t, bus_size_t, size_t);
void		x86_bus_space_mem_copy_2(bus_space_handle_t, bus_size_t,
		    bus_space_handle_t, bus_size_t, size_t);
void		x86_bus_space_mem_copy_4(bus_space_handle_t, bus_size_t,
		    bus_space_handle_t, bus_size_t, size_t);
void		x86_bus_space_mem_copy_8(bus_space_handle_t, bus_size_t,
		    bus_space_handle_t, bus_size_t, size_t);

void *		x86_bus_space_mem_vaddr(bus_space_handle_t);

paddr_t		x86_bus_space_mem_mmap(bus_addr_t, off_t, int, int);

const struct x86_bus_space_ops x86_bus_space_mem_ops = {
	x86_bus_space_mem_read_1,
	x86_bus_space_mem_read_2,
	x86_bus_space_mem_read_4,
	x86_bus_space_mem_read_8,
	x86_bus_space_mem_read_multi_1,
	x86_bus_space_mem_read_multi_2,
	x86_bus_space_mem_read_multi_4,
	x86_bus_space_mem_read_multi_8,
	x86_bus_space_mem_read_region_1,
	x86_bus_space_mem_read_region_2,
	x86_bus_space_mem_read_region_4,
	x86_bus_space_mem_read_region_8,
	x86_bus_space_mem_write_1,
	x86_bus_space_mem_write_2,
	x86_bus_space_mem_write_4,
	x86_bus_space_mem_write_8,
	x86_bus_space_mem_write_multi_1,
	x86_bus_space_mem_write_multi_2,
	x86_bus_space_mem_write_multi_4,
	x86_bus_space_mem_write_multi_8,
	x86_bus_space_mem_write_region_1,
	x86_bus_space_mem_write_region_2,
	x86_bus_space_mem_write_region_4,
	x86_bus_space_mem_write_region_8,
	x86_bus_space_mem_set_multi_1,
	x86_bus_space_mem_set_multi_2,
	x86_bus_space_mem_set_multi_4,
	x86_bus_space_mem_set_multi_8,
	x86_bus_space_mem_set_region_1,
	x86_bus_space_mem_set_region_2,
	x86_bus_space_mem_set_region_4,
	x86_bus_space_mem_set_region_8,
	x86_bus_space_mem_copy_1,
	x86_bus_space_mem_copy_2,
	x86_bus_space_mem_copy_4,
	x86_bus_space_mem_copy_8,
	x86_bus_space_mem_vaddr,
	x86_bus_space_mem_mmap
};

void
x86_bus_space_init(void)
{
	/*
	 * Initialize the I/O port and I/O mem extent maps.
	 * Note: we don't have to check the return value since
	 * creation of a fixed extent map will never fail (since
	 * descriptor storage has already been allocated).
	 *
	 * N.B. The iomem extent manages _all_ physical addresses
	 * on the machine.  When the amount of RAM is found, the two
	 * extents of RAM are allocated from the map (0 -> ISA hole
	 * and end of ISA hole -> end of RAM).
	 */
	ioport_ex = extent_create("ioport", 0x0, 0xffff, M_DEVBUF,
	    (caddr_t)ioport_ex_storage, sizeof(ioport_ex_storage),
	    EX_NOCOALESCE|EX_NOWAIT);
	iomem_ex = extent_create("iomem", 0x0, 0xffffffffffff, M_DEVBUF,
	    (caddr_t)iomem_ex_storage, sizeof(iomem_ex_storage),
	    EX_NOCOALESCE|EX_NOWAIT);
}

void
x86_bus_space_mallocok(void)
{
	ioport_malloc_safe = 1;
}

int
bus_space_map(bus_space_tag_t t, bus_addr_t bpa, bus_size_t size, int flags,
    bus_space_handle_t *bshp)
{
	int error;
	struct extent *ex;

	/*
	 * Pick the appropriate extent map.
	 */
	if (t == X86_BUS_SPACE_IO) {
		ex = ioport_ex;
		if (flags & BUS_SPACE_MAP_LINEAR)
			return (EINVAL);
	} else if (t == X86_BUS_SPACE_MEM)
		ex = iomem_ex;
	else
		panic("bus_space_map: bad bus space tag");

	/*
	 * Before we go any further, let's make sure that this
	 * region is available.
	 */
	error = extent_alloc_region(ex, bpa, size,
	    EX_NOWAIT | (ioport_malloc_safe ? EX_MALLOCOK : 0));
	if (error)
		return (error);

	/*
	 * For I/O space, that's all she wrote.
	 */
	if (t == X86_BUS_SPACE_IO) {
		*bshp = bpa;
		return (0);
	}

	if (bpa >= IOM_BEGIN && (bpa + size) <= IOM_END) {
		*bshp = (bus_space_handle_t)ISA_HOLE_VADDR(bpa);
		return(0);
	}

	/*
	 * For memory space, map the bus physical address to
	 * a kernel virtual address.
	 */
	error = x86_mem_add_mapping(bpa, size, flags, bshp);
	if (error) {
		if (extent_free(ex, bpa, size, EX_NOWAIT |
		    (ioport_malloc_safe ? EX_MALLOCOK : 0))) {
			printf("bus_space_map: pa 0x%lx, size 0x%lx\n",
			    bpa, size);
			printf("bus_space_map: can't free region\n");
		}
	}

	return (error);
}

int
_bus_space_map(bus_space_tag_t t, bus_addr_t bpa, bus_size_t size, int flags,
    bus_space_handle_t *bshp)
{

	/*
	 * For I/O space, just fill in the handle.
	 */
	if (t == X86_BUS_SPACE_IO) {
		*bshp = bpa;
		return (0);
	}

	/*
	 * For memory space, map the bus physical address to
	 * a kernel virtual address.
	 */
	return (x86_mem_add_mapping(bpa, size, flags, bshp));
}

int
bus_space_alloc(bus_space_tag_t t, bus_addr_t rstart, bus_addr_t rend,
    bus_size_t size, bus_size_t alignment, bus_size_t boundary, int flags,
    bus_addr_t *bpap, bus_space_handle_t *bshp)
{
	struct extent *ex;
	u_long bpa;
	int error;

	/*
	 * Pick the appropriate extent map.
	 */
	if (t == X86_BUS_SPACE_IO) {
		ex = ioport_ex;
	} else if (t == X86_BUS_SPACE_MEM)
		ex = iomem_ex;
	else
		panic("bus_space_alloc: bad bus space tag");

	/*
	 * Sanity check the allocation against the extent's boundaries.
	 */
	if (rstart < ex->ex_start || rend > ex->ex_end)
		panic("bus_space_alloc: bad region start/end");

	/*
	 * Do the requested allocation.
	 */
	error = extent_alloc_subregion(ex, rstart, rend, size, alignment,
	    0, boundary,
	    EX_FAST | EX_NOWAIT | (ioport_malloc_safe ?  EX_MALLOCOK : 0),
	    &bpa);

	if (error)
		return (error);

	/*
	 * For I/O space, that's all she wrote.
	 */
	if (t == X86_BUS_SPACE_IO) {
		*bshp = *bpap = bpa;
		return (0);
	}

	/*
	 * For memory space, map the bus physical address to
	 * a kernel virtual address.
	 */
	error = x86_mem_add_mapping(bpa, size, flags, bshp);
	if (error) {
		if (extent_free(iomem_ex, bpa, size, EX_NOWAIT |
		    (ioport_malloc_safe ? EX_MALLOCOK : 0))) {
			printf("bus_space_alloc: pa 0x%lx, size 0x%lx\n",
			    bpa, size);
			printf("bus_space_alloc: can't free region\n");
		}
	}

	*bpap = bpa;

	return (error);
}

int
x86_mem_add_mapping(bus_addr_t bpa, bus_size_t size, int flags,
    bus_space_handle_t *bshp)
{
	paddr_t pa, endpa;
	vaddr_t va;
	bus_size_t map_size;
	int pmap_flags = PMAP_NOCACHE;

	pa = trunc_page(bpa);
	endpa = round_page(bpa + size);

#ifdef DIAGNOSTIC
	if (endpa <= pa && endpa != 0)
		panic("bus_mem_add_mapping: overflow");
#endif

	map_size = endpa - pa;

	va = (vaddr_t)km_alloc(map_size, &kv_any, &kp_none, &kd_nowait);
	if (va == 0)
		return (ENOMEM);

	*bshp = (bus_space_handle_t)(va + (bpa & PGOFSET));

	if (flags & BUS_SPACE_MAP_CACHEABLE)
		pmap_flags = 0;
	else if (flags & BUS_SPACE_MAP_PREFETCHABLE)
		pmap_flags = PMAP_WC;

	for (; map_size > 0;
	    pa += PAGE_SIZE, va += PAGE_SIZE, map_size -= PAGE_SIZE)
		pmap_kenter_pa(va, pa | pmap_flags, PROT_READ | PROT_WRITE);
	pmap_update(pmap_kernel());

	return 0;
}

/*
 * void _bus_space_unmap(bus_space_tag bst, bus_space_handle bsh,
 *                        bus_size_t size, bus_addr_t *adrp)
 *
 *   This function unmaps memory- or io-space mapped by the function
 *   _bus_space_map().  This function works nearly as same as
 *   bus_space_unmap(), but this function does not ask kernel
 *   built-in extents and returns physical address of the bus space,
 *   for the convenience of the extra extent manager.
 */
void
_bus_space_unmap(bus_space_tag_t t, bus_space_handle_t bsh, bus_size_t size,
    bus_addr_t *adrp)
{
	u_long va, endva;
	bus_addr_t bpa;

	/*
	 * Find the correct bus physical address.
	 */
	if (t == X86_BUS_SPACE_IO) {
		bpa = bsh;
	} else if (t == X86_BUS_SPACE_MEM) {
		bpa = (bus_addr_t)ISA_PHYSADDR(bsh);
		if (IOM_BEGIN <= bpa && bpa <= IOM_END)
			goto ok;

		va = trunc_page(bsh);
		endva = round_page(bsh + size);

#ifdef DIAGNOSTIC
		if (endva <= va)
			panic("_bus_space_unmap: overflow");
#endif

		(void) pmap_extract(pmap_kernel(), va, &bpa);
		bpa += (bsh & PGOFSET);

		pmap_kremove(va, endva - va);
		pmap_update(pmap_kernel());

		/*
		 * Free the kernel virtual mapping.
		 */
		km_free((void *)va, endva - va, &kv_any, &kp_none);
	} else
		panic("bus_space_unmap: bad bus space tag");

ok:
	if (adrp != NULL)
		*adrp = bpa;
}

void
bus_space_unmap(bus_space_tag_t t, bus_space_handle_t bsh, bus_size_t size)
{
	struct extent *ex;
	u_long va, endva;
	bus_addr_t bpa;

	/*
	 * Find the correct extent and bus physical address.
	 */
	if (t == X86_BUS_SPACE_IO) {
		ex = ioport_ex;
		bpa = bsh;
	} else if (t == X86_BUS_SPACE_MEM) {
		ex = iomem_ex;
		bpa = (bus_addr_t)ISA_PHYSADDR(bsh);
		if (IOM_BEGIN <= bpa && bpa <= IOM_END)
			goto ok;

		va = trunc_page(bsh);
		endva = round_page(bsh + size);

#ifdef DIAGNOSTIC
		if (endva <= va)
			panic("bus_space_unmap: overflow");
#endif

		(void)pmap_extract(pmap_kernel(), va, &bpa);
		bpa += (bsh & PGOFSET);

		pmap_kremove(va, endva - va);
		pmap_update(pmap_kernel());

		/*
		 * Free the kernel virtual mapping.
		 */
		km_free((void *)va, endva - va, &kv_any, &kp_none);
	} else
		panic("bus_space_unmap: bad bus space tag");

ok:
	if (extent_free(ex, bpa, size,
	    EX_NOWAIT | (ioport_malloc_safe ? EX_MALLOCOK : 0))) {
		printf("bus_space_unmap: %s 0x%lx, size 0x%lx\n",
		    (t == X86_BUS_SPACE_IO) ? "port" : "pa", bpa, size);
		printf("bus_space_unmap: can't free region\n");
	}
}

void
bus_space_free(bus_space_tag_t t, bus_space_handle_t bsh, bus_size_t size)
{

	/* bus_space_unmap() does all that we need to do. */
	bus_space_unmap(t, bsh, size);
}

int
bus_space_subregion(bus_space_tag_t t, bus_space_handle_t bsh,
    bus_size_t offset, bus_size_t size, bus_space_handle_t *nbshp)
{

	*nbshp = bsh + offset;
	return (0);
}

u_int8_t
x86_bus_space_io_read_1(bus_space_handle_t h, bus_size_t o)
{
	return (inb(h + o));
}

u_int16_t
x86_bus_space_io_read_2(bus_space_handle_t h, bus_size_t o)
{
	return (inw(h + o));
}

u_int32_t
x86_bus_space_io_read_4(bus_space_handle_t h, bus_size_t o)
{
	return (inl(h + o));
}

u_int64_t
x86_bus_space_io_read_8(bus_space_handle_t h, bus_size_t o)
{
	panic("bus_space_read_8: invalid bus space tag");
}

void
x86_bus_space_io_read_multi_1(bus_space_handle_t h, bus_size_t o,
    u_int8_t *ptr, bus_size_t cnt)
{
	insb(h + o, ptr, cnt);
}

void
x86_bus_space_io_read_multi_2(bus_space_handle_t h, bus_size_t o,
    u_int16_t *ptr, bus_size_t cnt)
{
	insw(h + o, ptr, cnt);
}

void
x86_bus_space_io_read_multi_4(bus_space_handle_t h, bus_size_t o,
    u_int32_t *ptr, bus_size_t cnt)
{
	insl(h + o, ptr, cnt);
}

void
x86_bus_space_io_read_multi_8(bus_space_handle_t h, bus_size_t o,
    u_int64_t *ptr, bus_size_t cnt)
{
	panic("bus_space_multi_8: invalid bus space tag");
}

void
x86_bus_space_io_read_region_1(bus_space_handle_t h,
    bus_size_t o, u_int8_t *ptr, bus_size_t cnt)
{
	int dummy1;
	void *dummy2;
	int dummy3;
	int __x;
	u_int32_t port = h + o;
	__asm volatile(
	"1:	inb %w1,%%al				;"
	"	stosb					;"
	"	incl %1					;"
	"	loop 1b"				:
	    "=&a" (__x), "=d" (dummy1), "=D" (dummy2),
	    "=c" (dummy3)				:
	    "1" (port), "2" (ptr), "3" (cnt)	:
	    "memory");
}

void
x86_bus_space_io_read_region_2(bus_space_handle_t h,
    bus_size_t o, u_int16_t *ptr, bus_size_t cnt)
{
	int dummy1;
	void *dummy2;
	int dummy3;
	int __x;
	u_int32_t port = h + o;
	__asm volatile(
	"1:	inw %w1,%%ax				;"
	"	stosw					;"
	"	addl $2,%1				;"
	"	loop 1b"				:
	    "=&a" (__x), "=d" (dummy1), "=D" (dummy2),
	    "=c" (dummy3)				:
	    "1" ((port)), "2" ((ptr)), "3" ((cnt))	:
	    "memory");
}

void
x86_bus_space_io_read_region_4(bus_space_handle_t h,
    bus_size_t o, u_int32_t *ptr, bus_size_t cnt)
{
	int dummy1;
	void *dummy2;
	int dummy3;
	int __x;
	u_int32_t port = h + o;
	__asm volatile(
	"1:	inl %w1,%%eax				;"
	"	stosl					;"
	"	addl $4,%1				;"
	"	loop 1b"				:
	    "=&a" (__x), "=d" (dummy1), "=D" (dummy2),
	    "=c" (dummy3)				:
	    "1" (port), "2" (ptr), "3" (cnt)	:
	    "memory");
}

void
x86_bus_space_io_read_region_8(bus_space_handle_t h,
    bus_size_t o, u_int64_t *ptr, bus_size_t cnt)
{
	panic("bus_space_read_region_8: invalid bus space tag");
}

void
x86_bus_space_io_write_1(bus_space_handle_t h, bus_size_t o, u_int8_t v)
{
	outb(h + o, v);
}

void
x86_bus_space_io_write_2(bus_space_handle_t h, bus_size_t o, u_int16_t v)
{
	outw(h + o, v);
}

void
x86_bus_space_io_write_4(bus_space_handle_t h, bus_size_t o, u_int32_t v)
{
	outl(h + o, v);
}

void
x86_bus_space_io_write_8(bus_space_handle_t h, bus_size_t o, u_int64_t v)
{
	panic("bus_space_write_8: invalid bus space tag");
}

void
x86_bus_space_io_write_multi_1(bus_space_handle_t h,
    bus_size_t o, const u_int8_t *ptr, bus_size_t cnt)
{
	outsb(h + o, ptr, cnt);
}

void
x86_bus_space_io_write_multi_2(bus_space_handle_t h,
    bus_size_t o, const u_int16_t *ptr, bus_size_t cnt)
{
	outsw(h + o, ptr, cnt);
}

void
x86_bus_space_io_write_multi_4(bus_space_handle_t h,
    bus_size_t o, const u_int32_t *ptr, bus_size_t cnt)
{
	outsl(h + o, ptr, cnt);
}

void
x86_bus_space_io_write_multi_8(bus_space_handle_t h,
    bus_size_t o, const u_int64_t *ptr, bus_size_t cnt)
{
	panic("bus_space_write_multi_8: invalid bus space tag");
}

void
x86_bus_space_io_write_region_1(bus_space_handle_t h,
    bus_size_t o, const u_int8_t *ptr, bus_size_t cnt)
{
	int dummy1;
	void *dummy2;
	int dummy3;
	int __x;
	u_int32_t port = h + o;
	__asm volatile(
	"1:	lodsb					;"
	"	outb %%al,%w1				;"
	"	incl %1					;"
	"	loop 1b"				:
	    "=&a" (__x), "=d" (dummy1), "=S" (dummy2),
	    "=c" (dummy3)				:
	    "1" (port), "2" (ptr), "3" (cnt)	:
	    "memory");
}

void
x86_bus_space_io_write_region_2(bus_space_handle_t h,
    bus_size_t o, const u_int16_t *ptr, bus_size_t cnt)
{
	int dummy1;
	void *dummy2;
	int dummy3;
	int __x;
	u_int32_t port = h + o;
	__asm volatile(
	"1:	lodsw					;"
	"	outw %%ax,%w1				;"
	"	addl $2,%1				;"
	"	loop 1b"				:
	    "=&a" (__x), "=d" (dummy1), "=S" (dummy2),
	    "=c" (dummy3)				:
	    "1" (port), "2" (ptr), "3" (cnt)	:
	    "memory");
}

void
x86_bus_space_io_write_region_4(bus_space_handle_t h,
    bus_size_t o, const u_int32_t *ptr, bus_size_t cnt)
{
	int dummy1;
	void *dummy2;
	int dummy3;
	int __x;
	u_int32_t port = h + o;
	__asm volatile(
	"1:	lodsl					;"
	"	outl %%eax,%w1				;"
	"	addl $4,%1				;"
	"	loop 1b"				:
	    "=&a" (__x), "=d" (dummy1), "=S" (dummy2),
	    "=c" (dummy3)				:
	    "1" (port), "2" (ptr), "3" (cnt)	:
	    "memory");
}

void
x86_bus_space_io_write_region_8(bus_space_handle_t h,
    bus_size_t o, const u_int64_t *ptr, bus_size_t cnt)
{
	panic("bus_space_write_region_8: invalid bus space tag");
}

void
x86_bus_space_io_set_multi_1(bus_space_handle_t h, bus_size_t o,
    u_int8_t v, size_t c)
{
	bus_addr_t addr = h + o;

	while (c--)
		outb(addr, v);
}

void
x86_bus_space_io_set_multi_2(bus_space_handle_t h, bus_size_t o,
    u_int16_t v, size_t c)
{
	bus_addr_t addr = h + o;

	while (c--)
		outw(addr, v);
}

void
x86_bus_space_io_set_multi_4(bus_space_handle_t h, bus_size_t o,
    u_int32_t v, size_t c)
{
	bus_addr_t addr = h + o;

	while (c--)
		outl(addr, v);
}

void
x86_bus_space_io_set_multi_8(bus_space_handle_t h, bus_size_t o,
    u_int64_t v, size_t c)
{
	panic("bus_space_set_multi_8: invalid bus space tag");
}

void
x86_bus_space_io_set_region_1(bus_space_handle_t h, bus_size_t o,
    u_int8_t v, size_t c)
{
	bus_addr_t addr = h + o;

	for (; c != 0; c--, addr++)
		outb(addr, v);
}

void
x86_bus_space_io_set_region_2(bus_space_handle_t h, bus_size_t o,
    u_int16_t v, size_t c)
{
	bus_addr_t addr = h + o;

	for (; c != 0; c--, addr += sizeof(v))
		outw(addr, v);
}

void
x86_bus_space_io_set_region_4(bus_space_handle_t h, bus_size_t o,
    u_int32_t v, size_t c)
{
	bus_addr_t addr = h + o;

	for (; c != 0; c--, addr += sizeof(v))
		outl(addr, v);
}

void
x86_bus_space_io_set_region_8(bus_space_handle_t h, bus_size_t o,
    u_int64_t v, size_t c)
{
	panic("bus_space_set_region_8: invalid bus space tag");
}

void
x86_bus_space_io_copy_1(bus_space_handle_t h1, bus_size_t o1,
    bus_space_handle_t h2, bus_size_t o2, size_t c)
{
	bus_addr_t addr1 = h1 + o1;
	bus_addr_t addr2 = h2 + o2;

	if (addr1 >= addr2) {
		/* src after dest: copy forward */
		for (; c != 0; c--, addr1++, addr2++)
			outb(addr2, inb(addr1));
	} else {
		/* dest after src: copy backwards */
		for (addr1 += (c - 1), addr2 += (c - 1);
		    c != 0; c--, addr1--, addr2--)
			outb(addr2, inb(addr1));
	}
}

void
x86_bus_space_io_copy_2(bus_space_handle_t h1, bus_size_t o1,
    bus_space_handle_t h2, bus_size_t o2, size_t c)
{
	bus_addr_t addr1 = h1 + o1;
	bus_addr_t addr2 = h2 + o2;

	if (addr1 >= addr2) {
		/* src after dest: copy forward */
		for (; c != 0; c--, addr1 += 2, addr2 += 2)
			outw(addr2, inw(addr1));
	} else {
		/* dest after src: copy backwards */
		for (addr1 += 2 * (c - 1), addr2 += 2 * (c - 1);
		    c != 0; c--, addr1 -= 2, addr2 -= 2)
			outw(addr2, inw(addr1));
	}
}

void
x86_bus_space_io_copy_4(bus_space_handle_t h1, bus_size_t o1,
    bus_space_handle_t h2, bus_size_t o2, size_t c)
{
	bus_addr_t addr1 = h1 + o1;
	bus_addr_t addr2 = h2 + o2;

	if (addr1 >= addr2) {
		/* src after dest: copy forward */
		for (; c != 0; c--, addr1 += 4, addr2 += 4)
			outl(addr2, inl(addr1));
	} else {
		/* dest after src: copy backwards */
		for (addr1 += 4 * (c - 1), addr2 += 4 * (c - 1);
		    c != 0; c--, addr1 -= 4, addr2 -= 4)
			outl(addr2, inl(addr1));
	}
}

void
x86_bus_space_io_copy_8(bus_space_handle_t h1, bus_size_t o1,
    bus_space_handle_t h2, bus_size_t o2, size_t c)
{
	panic("bus_space_set_region_8: invalid bus space tag");
}

void *
x86_bus_space_io_vaddr(bus_space_handle_t h)
{
	return (NULL);
}

paddr_t
x86_bus_space_io_mmap(bus_addr_t addr, off_t off, int prot, int flags)
{
	/* Can't mmap I/O space. */
	return (-1);
}

void
x86_bus_space_mem_write_1(bus_space_handle_t h, bus_size_t o, u_int8_t v)
{
	*(volatile u_int8_t *)(h + o) = v;
}

void
x86_bus_space_mem_write_2(bus_space_handle_t h, bus_size_t o, u_int16_t v)
{
	*(volatile u_int16_t *)(h + o) = v;
}

u_int8_t
x86_bus_space_mem_read_1(bus_space_handle_t h, bus_size_t o)
{
	return (*(volatile u_int8_t *)(h + o));
}

u_int16_t
x86_bus_space_mem_read_2(bus_space_handle_t h, bus_size_t o)
{
	return (*(volatile u_int16_t *)(h + o));
}

u_int32_t
x86_bus_space_mem_read_4(bus_space_handle_t h, bus_size_t o)
{
	return (*(volatile u_int32_t *)(h + o));
}

u_int64_t
x86_bus_space_mem_read_8(bus_space_handle_t h, bus_size_t o)
{
	return (*(volatile u_int64_t *)(h + o));
}

void
x86_bus_space_mem_read_multi_1(bus_space_handle_t h, bus_size_t o,
    u_int8_t *ptr, bus_size_t cnt)
{
	void *dummy1;
	int dummy2;
	void *dummy3;
	int __x;
	__asm volatile(
	"1:	movb (%2),%%al				;"
	"	stosb					;"
	"	loop 1b"				:
	    "=D" (dummy1), "=c" (dummy2), "=r" (dummy3), "=&a" (__x) :
	    "0" ((ptr)), "1" ((cnt)), "2" (h + o)       :
	    "memory");
}

void
x86_bus_space_mem_read_multi_2(bus_space_handle_t h, bus_size_t o,
    u_int16_t *ptr, bus_size_t cnt)
{
	void *dummy1;
	int dummy2;
	void *dummy3;
	int __x;
	__asm volatile(
	"1:	movw (%2),%%ax				;"
	"	stosw					;"
	"	loop 1b"				:
	    "=D" (dummy1), "=c" (dummy2), "=r" (dummy3), "=&a" (__x) :
	    "0" ((ptr)), "1" ((cnt)), "2" (h + o)       :
	    "memory");
}

void
x86_bus_space_mem_read_multi_4(bus_space_handle_t h, bus_size_t o,
    u_int32_t *ptr, bus_size_t cnt)
{
	void *dummy1;
	int dummy2;
	void *dummy3;
	int __x;
	__asm volatile(
	"1:	movl (%2),%%eax				;"
	"	stosl					;"
	"	loop 1b"				:
	    "=D" (dummy1), "=c" (dummy2), "=r" (dummy3), "=&a" (__x) :
	    "0" ((ptr)), "1" ((cnt)), "2" (h + o)       :
	    "memory");
}

void
x86_bus_space_mem_read_multi_8(bus_space_handle_t h, bus_size_t o,
    u_int64_t *ptr, bus_size_t cnt)
{
	void *dummy1;
	int dummy2;
	void *dummy3;
	int __x;
	__asm volatile(
	"1:	movq (%2),%%rax				;"
	"	stosq					;"
	"	loop 1b"				:
	    "=D" (dummy1), "=c" (dummy2), "=r" (dummy3), "=&a" (__x) :
	    "0" ((ptr)), "1" ((cnt)), "2" (h + o)       :
	    "memory");
}

void
x86_bus_space_mem_read_region_1(bus_space_handle_t h,
    bus_size_t o, u_int8_t *ptr, bus_size_t cnt)
{
	int dummy1;
	void *dummy2;
	int dummy3;
	__asm volatile(
	"	repne					;"
	"	movsb"					:
	    "=S" (dummy1), "=D" (dummy2), "=c" (dummy3)	:
	    "0" (h + o), "1" (ptr), "2" (cnt)	:
	    "memory");
}

void
x86_bus_space_mem_read_region_2(bus_space_handle_t h,
    bus_size_t o, u_int16_t *ptr, bus_size_t cnt)
{
	int dummy1;
	void *dummy2;
	int dummy3;
	__asm volatile(
	"	repne					;"
	"	movsw"					:
	    "=S" (dummy1), "=D" (dummy2), "=c" (dummy3)	:
	    "0" (h + o), "1" (ptr), "2" (cnt)	:
	    "memory");
}

void
x86_bus_space_mem_read_region_4(bus_space_handle_t h,
    bus_size_t o, u_int32_t *ptr, bus_size_t cnt)
{
	int dummy1;
	void *dummy2;
	int dummy3;
	__asm volatile(
	"	repne					;"
	"	movsl"					:
	    "=S" (dummy1), "=D" (dummy2), "=c" (dummy3)	:
	    "0" (h + o), "1" (ptr), "2" (cnt)	:
	    "memory");
}

void
x86_bus_space_mem_read_region_8(bus_space_handle_t h,
    bus_size_t o, u_int64_t *ptr, bus_size_t cnt)
{
	int dummy1;
	void *dummy2;
	int dummy3;
	__asm volatile(
	"	repne					;"
	"	movsq"					:
	    "=S" (dummy1), "=D" (dummy2), "=c" (dummy3)	:
	    "0" (h + o), "1" (ptr), "2" (cnt)	:
	    "memory");
}

void
x86_bus_space_mem_write_4(bus_space_handle_t h, bus_size_t o, u_int32_t v)
{
	*(volatile u_int32_t *)(h + o) = v;
}

void
x86_bus_space_mem_write_8(bus_space_handle_t h, bus_size_t o, u_int64_t v)
{
	*(volatile u_int64_t *)(h + o) = v;
}

void
x86_bus_space_mem_write_multi_1(bus_space_handle_t h,
    bus_size_t o, const u_int8_t *ptr, bus_size_t cnt)
{
	void *dummy1;
	int dummy2;
	void *dummy3;
	int __x;
	__asm volatile(
	"1:	lodsb					;"
	"	movb %%al,(%2)				;"
	"	loop 1b"				:
	    "=S" (dummy1), "=c" (dummy2), "=r" (dummy3), "=&a" (__x) :
	    "0" (ptr), "1" (cnt), "2" (h + o));
}

void
x86_bus_space_mem_write_multi_2(bus_space_handle_t h,
    bus_size_t o, const u_int16_t *ptr, bus_size_t cnt)
{
	void *dummy1;
	int dummy2;
	void *dummy3;
	int __x;
	__asm volatile(
	"1:	lodsw					;"
	"	movw %%ax,(%2)				;"
	"	loop 1b"				:
	    "=S" (dummy1), "=c" (dummy2), "=r" (dummy3), "=&a" (__x) :
	    "0" (ptr), "1" (cnt), "2" (h + o));
}

void
x86_bus_space_mem_write_multi_4(bus_space_handle_t h,
    bus_size_t o, const u_int32_t *ptr, bus_size_t cnt)
{
	void *dummy1;
	int dummy2;
	void *dummy3;
	int __x;
	__asm volatile(
	"1:	lodsl					;"
	"	movl %%eax,(%2)				;"
	"	loop 1b"				:
	    "=S" (dummy1), "=c" (dummy2), "=r" (dummy3), "=&a" (__x) :
	    "0" (ptr), "1" (cnt), "2" (h + o));
}

void
x86_bus_space_mem_write_multi_8(bus_space_handle_t h,
    bus_size_t o, const u_int64_t *ptr, bus_size_t cnt)
{
	void *dummy1;
	int dummy2;
	void *dummy3;
	int __x;
	__asm volatile(
	"1:	lodsq					;"
	"	movq %%rax,(%2)				;"
	"	loop 1b"				:
	    "=S" (dummy1), "=c" (dummy2), "=r" (dummy3), "=&a" (__x) :
	    "0" (ptr), "1" (cnt), "2" (h + o));
}

void
x86_bus_space_mem_write_region_1(bus_space_handle_t h,
    bus_size_t o, const u_int8_t *ptr, bus_size_t cnt)
{
	int dummy1;
	void *dummy2;
	int dummy3;
	__asm volatile(
	"	repne					;"
	"	movsb"					:
	    "=D" (dummy1), "=S" (dummy2), "=c" (dummy3)	:
	    "0" (h + o), "1" (ptr), "2" (cnt)	:
	    "memory");
}

void
x86_bus_space_mem_write_region_2(bus_space_handle_t h,
    bus_size_t o, const u_int16_t *ptr, bus_size_t cnt)
{
	int dummy1;
	void *dummy2;
	int dummy3;
	__asm volatile(
	"	repne					;"
	"	movsw"					:
	    "=D" (dummy1), "=S" (dummy2), "=c" (dummy3)	:
	    "0" (h + o), "1" (ptr), "2" (cnt)	:
	    "memory");
}

void
x86_bus_space_mem_write_region_4(bus_space_handle_t h,
    bus_size_t o, const u_int32_t *ptr, bus_size_t cnt)
{
	int dummy1;
	void *dummy2;
	int dummy3;
	__asm volatile(
	"	repne					;"
	"	movsl"					:
	    "=D" (dummy1), "=S" (dummy2), "=c" (dummy3)	:
	    "0" (h + o), "1" (ptr), "2" (cnt)	:
	    "memory");
}

void
x86_bus_space_mem_write_region_8(bus_space_handle_t h,
    bus_size_t o, const u_int64_t *ptr, bus_size_t cnt)
{
	int dummy1;
	void *dummy2;
	int dummy3;
	__asm volatile(
	"	repne					;"
	"	movsq"					:
	    "=D" (dummy1), "=S" (dummy2), "=c" (dummy3)	:
	    "0" (h + o), "1" (ptr), "2" (cnt)	:
	    "memory");
}

void
x86_bus_space_mem_set_multi_1(bus_space_handle_t h, bus_size_t o,
    u_int8_t v, size_t c)
{
	bus_addr_t addr = h + o;

	while (c--)
		*(volatile u_int8_t *)(addr) = v;
}

void
x86_bus_space_mem_set_multi_2(bus_space_handle_t h, bus_size_t o,
    u_int16_t v, size_t c)
{
	bus_addr_t addr = h + o;

	while (c--)
		*(volatile u_int16_t *)(addr) = v;
}

void
x86_bus_space_mem_set_multi_4(bus_space_handle_t h, bus_size_t o,
    u_int32_t v, size_t c)
{
	bus_addr_t addr = h + o;

	while (c--)
		*(volatile u_int32_t *)(addr) = v;
}

void
x86_bus_space_mem_set_multi_8(bus_space_handle_t h, bus_size_t o,
    u_int64_t v, size_t c)
{
	bus_addr_t addr = h + o;

	while (c--)
		*(volatile u_int64_t *)(addr) = v;
}

void
x86_bus_space_mem_set_region_1(bus_space_handle_t h, bus_size_t o,
    u_int8_t v, size_t c)
{
	bus_addr_t addr = h + o;

	for (; c != 0; c--, addr++)
		*(volatile u_int8_t *)(addr) = v;
}

void
x86_bus_space_mem_set_region_2(bus_space_handle_t h, bus_size_t o,
    u_int16_t v, size_t c)
{
	bus_addr_t addr = h + o;

	for (; c != 0; c--, addr += sizeof(v))
		*(volatile u_int16_t *)(addr) = v;
}

void
x86_bus_space_mem_set_region_4(bus_space_handle_t h, bus_size_t o,
    u_int32_t v, size_t c)
{
	bus_addr_t addr = h + o;

	for (; c != 0; c--, addr += sizeof(v))
		*(volatile u_int32_t *)(addr) = v;
}

void
x86_bus_space_mem_set_region_8(bus_space_handle_t h, bus_size_t o,
    u_int64_t v, size_t c)
{
	bus_addr_t addr = h + o;

	for (; c != 0; c--, addr += sizeof(v))
		*(volatile u_int64_t *)(addr) = v;
}

void
x86_bus_space_mem_copy_1( bus_space_handle_t h1, bus_size_t o1,
    bus_space_handle_t h2, bus_size_t o2, size_t c)
{
	bus_addr_t addr1 = h1 + o1;
	bus_addr_t addr2 = h2 + o2;

	if (addr1 >= addr2) {
		/* src after dest: copy forward */
		for (; c != 0; c--, addr1++, addr2++)
			*(volatile u_int8_t *)(addr2) =
			    *(volatile u_int8_t *)(addr1);
	} else {
		/* dest after src: copy backwards */
		for (addr1 += (c - 1), addr2 += (c - 1);
		    c != 0; c--, addr1--, addr2--)
			*(volatile u_int8_t *)(addr2) =
			    *(volatile u_int8_t *)(addr1);
	}
}

void
x86_bus_space_mem_copy_2(bus_space_handle_t h1, bus_size_t o1,
    bus_space_handle_t h2, bus_size_t o2, size_t c)
{
	bus_addr_t addr1 = h1 + o1;
	bus_addr_t addr2 = h2 + o2;

	if (addr1 >= addr2) {
		/* src after dest: copy forward */
		for (; c != 0; c--, addr1 += 2, addr2 += 2)
			*(volatile u_int16_t *)(addr2) =
			    *(volatile u_int16_t *)(addr1);
	} else {
		/* dest after src: copy backwards */
		for (addr1 += 2 * (c - 1), addr2 += 2 * (c - 1);
		    c != 0; c--, addr1 -= 2, addr2 -= 2)
			*(volatile u_int16_t *)(addr2) =
			    *(volatile u_int16_t *)(addr1);
	}
}

void
x86_bus_space_mem_copy_4(bus_space_handle_t h1, bus_size_t o1,
    bus_space_handle_t h2, bus_size_t o2, size_t c)
{
	bus_addr_t addr1 = h1 + o1;
	bus_addr_t addr2 = h2 + o2;

	if (addr1 >= addr2) {
		/* src after dest: copy forward */
		for (; c != 0; c--, addr1 += 4, addr2 += 4)
			*(volatile u_int32_t *)(addr2) =
			    *(volatile u_int32_t *)(addr1);
	} else {
		/* dest after src: copy backwards */
		for (addr1 += 4 * (c - 1), addr2 += 4 * (c - 1);
		    c != 0; c--, addr1 -= 4, addr2 -= 4)
			*(volatile u_int32_t *)(addr2) =
			    *(volatile u_int32_t *)(addr1);
	}
}

void
x86_bus_space_mem_copy_8(bus_space_handle_t h1, bus_size_t o1,
    bus_space_handle_t h2, bus_size_t o2, size_t c)
{
	bus_addr_t addr1 = h1 + o1;
	bus_addr_t addr2 = h2 + o2;

	if (addr1 >= addr2) {
		/* src after dest: copy forward */
		for (; c != 0; c--, addr1 += 8, addr2 += 8)
			*(volatile u_int64_t *)(addr2) =
			    *(volatile u_int64_t *)(addr1);
	} else {
		/* dest after src: copy backwards */
		for (addr1 += 8 * (c - 1), addr2 += 8 * (c - 1);
		    c != 0; c--, addr1 -= 8, addr2 -= 8)
			*(volatile u_int64_t *)(addr2) =
			    *(volatile u_int64_t *)(addr1);
	}
}

void *
x86_bus_space_mem_vaddr(bus_space_handle_t h)
{
	return ((void *)h);
}

paddr_t
x86_bus_space_mem_mmap(bus_addr_t addr, off_t off, int prot, int flags)
{
	/*
	 * "addr" is the base address of the device we're mapping.
	 * "off" is the offset into that device.
	 *
	 * Note we are called for each "page" in the device that
	 * the upper layers want to map.
	 */
	return (addr + off);
}


code:
/*	$OpenBSD: cacheinfo.c,v 1.6 2012/03/16 01:53:00 haesbaert Exp $	*/

/*-
 * Copyright (c) 2000 The NetBSD Foundation, Inc.
 * All rights reserved.
 *
 * This code is derived from software contributed to The NetBSD Foundation
 * by Jason R. Thorpe.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#include <sys/param.h>
#include <sys/systm.h>

#include <machine/cpu.h>
#include <machine/specialreg.h>

static char *print_cache_config(struct cpu_info *, int, char *, char *);
static char *print_tlb_config(struct cpu_info *, int, char *, char *);

static char *
print_cache_config(struct cpu_info *ci, int cache_tag, char *name, char *sep)
{
	struct x86_cache_info *cai = &ci->ci_cinfo[cache_tag];

	if (cai->cai_totalsize == 0)
		return sep;

	if (sep == NULL)
		printf("%s: ", ci->ci_dev->dv_xname);
	else
		printf("%s", sep);

	if (cai->cai_string != NULL)
		printf("%s ", cai->cai_string);
	else if (cai->cai_totalsize >= 1024*1024)
		printf("%dMB %db/line ", cai->cai_totalsize / 1024 / 1024,
		    cai->cai_linesize);
	else
		printf("%dKB %db/line ", cai->cai_totalsize / 1024,
		    cai->cai_linesize);

	switch (cai->cai_associativity) {
	case    0:
		printf("disabled");
		break;
	case    1:
		printf("direct-mapped");
		break;
	case 0xff:
		printf("fully associative");
		break;
	default:
		printf("%d-way", cai->cai_associativity);
		break;
	}

	if (name != NULL)
		printf(" %s", name);

	return ", ";
}

static char *
print_tlb_config(struct cpu_info *ci, int cache_tag, char *name, char *sep)
{
	struct x86_cache_info *cai = &ci->ci_cinfo[cache_tag];

	if (cai->cai_totalsize == 0)
		return sep;

	if (sep == NULL)
		printf("%s: ", ci->ci_dev->dv_xname);
	else
		printf("%s", sep);
	if (name != NULL)
		printf("%s ", name);

	if (cai->cai_string != NULL) {
		printf("%s", cai->cai_string);
	} else {
		if (cai->cai_linesize >= 1024*1024)
			printf("%d %dMB entries ", cai->cai_totalsize,
			    cai->cai_linesize / 1024 / 1024);
		else
			printf("%d %dKB entries ", cai->cai_totalsize,
			    cai->cai_linesize / 1024);
		switch (cai->cai_associativity) {
		case 0:
			printf("disabled");
			break;
		case 1:
			printf("direct-mapped");
			break;
		case 0xff:
			printf("fully associative");
			break;
		default:
			printf("%d-way", cai->cai_associativity);
			break;
		}
	}
	return ", ";
}

const struct x86_cache_info *
cache_info_lookup(const struct x86_cache_info *cai, u_int8_t desc)
{
	int i;

	for (i = 0; cai[i].cai_desc != 0; i++) {
		if (cai[i].cai_desc == desc)
			return (&cai[i]);
	}

	return (NULL);
}

static const struct x86_cache_info amd_cpuid_l2cache_assoc_info[] = {
	{ 0, 0x01,    1 },
	{ 0, 0x02,    2 },
	{ 0, 0x04,    4 },
	{ 0, 0x06,    8 },
	{ 0, 0x08,   16 },
	{ 0, 0x0a,   32 },
	{ 0, 0x0b,   48 },
	{ 0, 0x0c,   64 },
	{ 0, 0x0d,   96 },
	{ 0, 0x0e,  128 },
	{ 0, 0x0f, 0xff },
	{ 0, 0x00,    0 },
};

void
amd_cpu_cacheinfo(struct cpu_info *ci)
{
	const struct x86_cache_info *cp;
	struct x86_cache_info *cai;
	int family, model;
	u_int descs[4];
	u_int lfunc;

	family = ci->ci_family;
	model = ci->ci_model;

	/*
	 * K5 model 0 has none of this info.
	 */
	if (family == 5 && model == 0)
		return;

	/*
	 * Determine the largest extended function value.
	 */
	CPUID(0x80000000, descs[0], descs[1], descs[2], descs[3]);
	lfunc = descs[0];

	/*
	 * Determine L1 cache/TLB info.
	 */
	if (lfunc < 0x80000005) {
		/* No L1 cache info available. */
		return;
	}

	CPUID(0x80000005, descs[0], descs[1], descs[2], descs[3]);

	/*
	 * K6-III and higher have large page TLBs.
	 */
	if ((family == 5 && model >= 9) || family >= 6) {
		cai = &ci->ci_cinfo[CAI_ITLB2];
		cai->cai_totalsize = AMD_L1_EAX_ITLB_ENTRIES(descs[0]);
		cai->cai_associativity = AMD_L1_EAX_ITLB_ASSOC(descs[0]);
		cai->cai_linesize = (4 * 1024 * 1024);

		cai = &ci->ci_cinfo[CAI_DTLB2];
		cai->cai_totalsize = AMD_L1_EAX_DTLB_ENTRIES(descs[0]);
		cai->cai_associativity = AMD_L1_EAX_DTLB_ASSOC(descs[0]);
		cai->cai_linesize = (4 * 1024 * 1024);
	}

	cai = &ci->ci_cinfo[CAI_ITLB];
	cai->cai_totalsize = AMD_L1_EBX_ITLB_ENTRIES(descs[1]);
	cai->cai_associativity = AMD_L1_EBX_ITLB_ASSOC(descs[1]);
	cai->cai_linesize = (4 * 1024);

	cai = &ci->ci_cinfo[CAI_DTLB];
	cai->cai_totalsize = AMD_L1_EBX_DTLB_ENTRIES(descs[1]);
	cai->cai_associativity = AMD_L1_EBX_DTLB_ASSOC(descs[1]);
	cai->cai_linesize = (4 * 1024);

	cai = &ci->ci_cinfo[CAI_DCACHE];
	cai->cai_totalsize = AMD_L1_ECX_DC_SIZE(descs[2]);
	cai->cai_associativity = AMD_L1_ECX_DC_ASSOC(descs[2]);
	cai->cai_linesize = AMD_L1_EDX_IC_LS(descs[2]);

	cai = &ci->ci_cinfo[CAI_ICACHE];
	cai->cai_totalsize = AMD_L1_EDX_IC_SIZE(descs[3]);
	cai->cai_associativity = AMD_L1_EDX_IC_ASSOC(descs[3]);
	cai->cai_linesize = AMD_L1_EDX_IC_LS(descs[3]);

	/*
	 * Determine L2 cache/TLB info.
	 */
	if (lfunc < 0x80000006) {
		/* No L2 cache info available. */
		return;
	}

	CPUID(0x80000006, descs[0], descs[1], descs[2], descs[3]);

	cai = &ci->ci_cinfo[CAI_L2CACHE];
	cai->cai_totalsize = AMD_L2_ECX_C_SIZE(descs[2]);
	cai->cai_associativity = AMD_L2_ECX_C_ASSOC(descs[2]);
	cai->cai_linesize = AMD_L2_ECX_C_LS(descs[2]);

	cp = cache_info_lookup(amd_cpuid_l2cache_assoc_info,
	    cai->cai_associativity);
	if (cp != NULL)
		cai->cai_associativity = cp->cai_associativity;
	else
		cai->cai_associativity = 0;	/* XXX Unknown/reserved */

	/*
	 * Determine L3 cache, Intel is different
	 */
	if (!strcmp(cpu_vendor, "AuthenticAMD") && family >= 0xf) {
		cai = &ci->ci_cinfo[CAI_L3CACHE];
		cai->cai_totalsize = AMD_L3_EDX_C_SIZE(descs[3]);
		cai->cai_associativity = AMD_L3_EDX_C_ASSOC(descs[3]);
		cai->cai_linesize = AMD_L3_EDX_C_LS(descs[3]);
		cp = cache_info_lookup(amd_cpuid_l2cache_assoc_info,
		    cai->cai_associativity);
		if (cp != NULL)
			cai->cai_associativity = cp->cai_associativity;
		else
			cai->cai_associativity = 0;	/* XXX Unknown/reserved */
	}
}

void
x86_print_cacheinfo(struct cpu_info *ci)
{
	char *sep;

	sep = print_cache_config(ci, CAI_ICACHE, "I-cache", NULL);
	sep = print_cache_config(ci, CAI_DCACHE, "D-cache", sep);
	sep = print_cache_config(ci, CAI_L2CACHE, "L2 cache", sep);
	sep = print_cache_config(ci, CAI_L3CACHE, "L3 cache", sep);
	if (sep != NULL)
		printf("\n");
	sep = print_tlb_config(ci, CAI_ITLB, "ITLB", NULL);
	sep = print_tlb_config(ci, CAI_ITLB2, NULL, sep);
	if (sep != NULL)
		printf("\n");
	sep = print_tlb_config(ci, CAI_DTLB, "DTLB", NULL);
	sep = print_tlb_config(ci, CAI_DTLB2, NULL, sep);
	if (sep != NULL)
		printf("\n");
}

license: unknown


code:
/*      $OpenBSD: codepatch.c,v 1.2 2015/04/19 06:30:20 sf Exp $    */
/*
 * Copyright (c) 2014-2015 Stefan Fritsch <sf@sfritsch.de>
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

#include <sys/param.h>
#include <machine/codepatch.h>
#include <uvm/uvm_extern.h>

#ifdef CODEPATCH_DEBUG
#define DBGPRINT(fmt, args...)	printf("%s: " fmt "\n", __func__, ## args)
#else
#define DBGPRINT(fmt, args...)	do {} while (0)
#endif

struct codepatch {
	uint32_t offset;
	uint16_t len;
	uint16_t tag;
};

extern struct codepatch codepatch_begin;
extern struct codepatch codepatch_end;

#define NOP_LEN_MAX	9

static const unsigned char nops[][NOP_LEN_MAX] = {
	{ 0x90 },
	{ 0x66, 0x90 },
	{ 0x0F, 0x1F, 0x00 },
	{ 0x0F, 0x1F, 0x40, 0x00 },
	{ 0x0F, 0x1F, 0x44, 0x00, 0x00 },
	{ 0x66, 0x0F, 0x1F, 0x44, 0x00, 0x00 },
	{ 0x0F, 0x1F, 0x80, 0x00, 0x00, 0x00, 0x00 },
	{ 0x0F, 0x1F, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00 },
	{ 0x66, 0x0F, 0x1F, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00},
};

void
codepatch_fill_nop(void *caddr, uint16_t len)
{
	unsigned char *addr = caddr;
	uint16_t nop_len;

	while (len > 0) {
		if (len <= NOP_LEN_MAX)
			nop_len = len;
		else
			nop_len = NOP_LEN_MAX;
		memcpy(addr, nops[nop_len-1], nop_len);
		addr += nop_len;
		len -= nop_len;
	}
}

/*
 * Create writeable aliases of memory we need
 * to write to as kernel is mapped read-only
 */
void *codepatch_maprw(vaddr_t *nva, vaddr_t dest)
{
	paddr_t kva = trunc_page((paddr_t)dest);
	paddr_t po = (paddr_t)dest & PAGE_MASK;
	paddr_t pa1, pa2;

	if (*nva == 0)
		*nva = (vaddr_t)km_alloc(2 * PAGE_SIZE, &kv_any, &kp_none,
					&kd_waitok);

	pmap_extract(pmap_kernel(), kva, &pa1);
	pmap_extract(pmap_kernel(), kva + PAGE_SIZE, &pa2);
	pmap_kenter_pa(*nva, pa1, PROT_READ | PROT_WRITE);
	pmap_kenter_pa(*nva + PAGE_SIZE, pa2, PROT_READ | PROT_WRITE);
	pmap_update(pmap_kernel());

	return (void *)(*nva + po);
}

void codepatch_unmaprw(vaddr_t nva)
{
	if (nva == 0)
		return;
	pmap_kremove(nva, 2 * PAGE_SIZE);
	km_free((void *)nva, 2 * PAGE_SIZE, &kv_any, &kp_none);
}

/* Patch with NOPs */
void
codepatch_nop(uint16_t tag)
{
	struct codepatch *patch;
	unsigned char *rwaddr;
	vaddr_t addr, rwmap = 0;
	int i = 0;

	DBGPRINT("patching tag %u", tag);

	for (patch = &codepatch_begin; patch < &codepatch_end; patch++) {
		if (patch->tag != tag)
			continue;
		addr = KERNBASE + patch->offset;
		rwaddr = codepatch_maprw(&rwmap, addr);
		codepatch_fill_nop(rwaddr, patch->len);
		i++;
	}
	codepatch_unmaprw(rwmap);
	DBGPRINT("patched %d places", i);
}

/* Patch with alternative code */
void
codepatch_replace(uint16_t tag, void *code, size_t len)
{
	struct codepatch *patch;
	unsigned char *rwaddr;
	vaddr_t addr, rwmap = 0;
	int i = 0;

	DBGPRINT("patching tag %u with %p", tag, code);

	for (patch = &codepatch_begin; patch < &codepatch_end; patch++) {
		if (patch->tag != tag)
			continue;
		addr = KERNBASE + patch->offset;

		if (len > patch->len) {
			panic("%s: can't replace len %u with %zu at %#lx",
			    __func__, patch->len, len, addr);
		}
		rwaddr = codepatch_maprw(&rwmap, addr);
		memcpy(rwaddr, code, len);
		codepatch_fill_nop(rwaddr + len, patch->len - len);
		i++;
	}
	codepatch_unmaprw(rwmap);
	DBGPRINT("patched %d places", i);
}

/* Patch with calls to func */
void
codepatch_call(uint16_t tag, void *func)
{
	struct codepatch *patch;
	unsigned char *rwaddr;
	int32_t offset;
	int i = 0;
	vaddr_t addr, rwmap = 0;

	DBGPRINT("patching tag %u with call %p", tag, func);

	for (patch = &codepatch_begin; patch < &codepatch_end; patch++) {
		if (patch->tag != tag)
			continue;
		addr = KERNBASE + patch->offset;
		if (patch->len < 5)
			panic("%s: can't replace len %u with call at %#lx",
			    __func__, patch->len, addr);

		offset = (vaddr_t)func - (addr + 5);
		rwaddr = codepatch_maprw(&rwmap, addr);
		rwaddr[0] = 0xe8; /* call near */
		memcpy(rwaddr + 1, &offset, sizeof(offset));
		codepatch_fill_nop(rwaddr + 5, patch->len - 5);
		i++;
	}
	codepatch_unmaprw(rwmap);
	DBGPRINT("patched %d places", i);
}


code:
/*	$OpenBSD: conf.c,v 1.50 2015/03/14 03:38:46 jsg Exp $	*/

/*
 * Copyright (c) 1994, 1995 Charles M. Hannum.  All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. All advertising materials mentioning features or use of this software
 *    must display the following acknowledgement:
 *	This product includes software developed by Charles Hannum.
 * 4. The name of the author may not be used to endorse or promote products
 *    derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/device.h>
#include <sys/disklabel.h>
#include <sys/ioctl.h>
#include <sys/tty.h>
#include <sys/vnode.h>

#include <machine/conf.h>

#include "wd.h"
bdev_decl(wd);
#include "fdc.h"
#include "fd.h"
bdev_decl(fd);
#include "sd.h"
#include "st.h"
#include "cd.h"
#include "uk.h"
#include "vnd.h"
#include "rd.h"

struct bdevsw	bdevsw[] =
{
	bdev_disk_init(NWD,wd),		/* 0: ST506/ESDI/IDE disk */
	bdev_swap_init(1,sw),		/* 1: swap pseudo-device */
	bdev_disk_init(NFD,fd),		/* 2: floppy diskette */
	bdev_notdef(),			/* 3 */
	bdev_disk_init(NSD,sd),		/* 4: SCSI disk */
	bdev_tape_init(NST,st),		/* 5: SCSI tape */
	bdev_disk_init(NCD,cd),		/* 6: SCSI CD-ROM */
	bdev_notdef(),			/* 7 */
	bdev_notdef(),			/* 8 */
	bdev_notdef(),			/* 9 */
	bdev_notdef(),			/* 10 */
	bdev_notdef(),			/* 11 */
	bdev_notdef(),			/* 12 */
	bdev_notdef(),			/* 13 */
	bdev_disk_init(NVND,vnd),	/* 14: vnode disk driver */
	bdev_notdef(),			/* 15: was: Sony CD-ROM */
	bdev_notdef(),			/* 16: was: concatenated disk driver */
	bdev_disk_init(NRD,rd),		/* 17: ram disk driver */
	bdev_notdef(),			/* 18 */
	bdev_notdef(),			/* 19 was: RAIDframe disk driver */
};
int	nblkdev = nitems(bdevsw);

/* open, close, read, write, ioctl, tty, mmap */
#define cdev_pc_init(c,n) { \
	dev_init(c,n,open), dev_init(c,n,close), dev_init(c,n,read), \
	dev_init(c,n,write), dev_init(c,n,ioctl), dev_init(c,n,stop), \
	dev_init(c,n,tty), ttselect, dev_init(c,n,mmap), D_TTY }

/* open, close, read, ioctl */
#define cdev_joy_init(c,n) { \
	dev_init(c,n,open), dev_init(c,n,close), dev_init(c,n,read), \
	(dev_type_write((*))) enodev, dev_init(c,n,ioctl), \
	(dev_type_stop((*))) enodev, 0, seltrue, \
	(dev_type_mmap((*))) enodev }

/* open, close, ioctl, select -- XXX should be a generic device */
#define cdev_ocis_init(c,n) { \
        dev_init(c,n,open), dev_init(c,n,close), (dev_type_read((*))) enodev, \
        (dev_type_write((*))) enodev, dev_init(c,n,ioctl), \
        (dev_type_stop((*))) enodev, 0,  dev_init(c,n,poll), \
        (dev_type_mmap((*))) enodev, 0 }

/* open, close, read */
#define cdev_nvram_init(c,n) { \
	dev_init(c,n,open), dev_init(c,n,close), dev_init(c,n,read), \
	(dev_type_write((*))) enodev, (dev_type_ioctl((*))) enodev, \
	(dev_type_stop((*))) enodev, 0, seltrue, \
	(dev_type_mmap((*))) enodev, 0 }

#define	mmread	mmrw
#define	mmwrite	mmrw
cdev_decl(mm);
cdev_decl(wd);
#include "systrace.h"
#include "bio.h"
#include "pty.h"
#include "com.h"
cdev_decl(com);
cdev_decl(fd);
#include "lpt.h"
cdev_decl(lpt);
#include "ch.h"
#include "bpfilter.h"
#if 0
#include "pcmcia.h"
cdev_decl(pcmcia);
#endif
#include "spkr.h"
cdev_decl(spkr);
#if 0 /* old (non-wsmouse) drivers */
#include "mms.h"
cdev_decl(mms);
#include "lms.h"
cdev_decl(lms);
#include "opms.h"
cdev_decl(pms);
#endif
#include "cy.h"
cdev_decl(cy);
#include "tun.h"
#include "audio.h"
#include "video.h"
#include "midi.h"
#include "acpi.h"
#include "pctr.h"
#include "bktr.h"
#include "ksyms.h"
#include "usb.h"
#include "uhid.h"
#include "ugen.h"
#include "ulpt.h"
#include "ucom.h"
#include "cz.h"
cdev_decl(cztty);
#include "radio.h"
#include "nvram.h"
cdev_decl(nvram);
#include "drm.h"
cdev_decl(drm);

#include "wsdisplay.h"
#include "wskbd.h"
#include "wsmouse.h"
#include "wsmux.h"

#ifdef USER_PCICONF
#include "pci.h"
cdev_decl(pci);
#endif

#include "pf.h"
#include "hotplug.h"
#include "gpio.h"
#include "vscsi.h"
#include "pppx.h"
#include "fuse.h"

struct cdevsw	cdevsw[] =
{
	cdev_cn_init(1,cn),		/* 0: virtual console */
	cdev_ctty_init(1,ctty),		/* 1: controlling terminal */
	cdev_mm_init(1,mm),		/* 2: /dev/{null,mem,kmem,...} */
	cdev_disk_init(NWD,wd),		/* 3: ST506/ESDI/IDE disk */
	cdev_notdef(),			/* 4 was /dev/drum */
	cdev_tty_init(NPTY,pts),	/* 5: pseudo-tty slave */
	cdev_ptc_init(NPTY,ptc),	/* 6: pseudo-tty master */
	cdev_log_init(1,log),		/* 7: /dev/klog */
	cdev_tty_init(NCOM,com),	/* 8: serial port */
	cdev_disk_init(NFD,fd),		/* 9: floppy disk */
	cdev_notdef(),			/* 10 */
	cdev_notdef(),			/* 11: Sony CD-ROM */
	cdev_wsdisplay_init(NWSDISPLAY,	/* 12: frame buffers, etc. */
	    wsdisplay),
	cdev_disk_init(NSD,sd),		/* 13: SCSI disk */
	cdev_tape_init(NST,st),		/* 14: SCSI tape */
	cdev_disk_init(NCD,cd),		/* 15: SCSI CD-ROM */
	cdev_lpt_init(NLPT,lpt),	/* 16: parallel printer */
	cdev_ch_init(NCH,ch),		/* 17: SCSI autochanger */
	cdev_notdef(),			/* 18: was: concatenated disk driver */
	cdev_notdef(),			/* 19 */
	cdev_uk_init(NUK,uk),		/* 20: unknown SCSI */
	cdev_notdef(),			/* 21 */
	cdev_fd_init(1,filedesc),	/* 22: file descriptor pseudo-device */
	cdev_bpf_init(NBPFILTER,bpf),	/* 23: Berkeley packet filter */
	cdev_notdef(),			/* 24 */
#if 0
	cdev_ocis_init(NPCMCIA,pcmcia), /* 25: PCMCIA Bus */
#else
	cdev_notdef(),			/* 25 */
#endif
	cdev_notdef(),			/* 26 */
	cdev_spkr_init(NSPKR,spkr),	/* 27: PC speaker */
	cdev_notdef(),			/* 28 was LKM */
	cdev_notdef(),			/* 29 */
	cdev_notdef(),			/* 30 */
	cdev_notdef(),			/* 31 */
	cdev_notdef(),			/* 32 */
	cdev_notdef(),			/* 33 */
	cdev_notdef(),			/* 34 */
	cdev_notdef(),			/* 35: Microsoft mouse */
	cdev_notdef(),			/* 36: Logitech mouse */
	cdev_notdef(),			/* 37: Extended PS/2 mouse */
	cdev_tty_init(NCY,cy),		/* 38: Cyclom serial port */
	cdev_notdef(),			/* 39: Mitsumi CD-ROM */
	cdev_tun_init(NTUN,tun),	/* 40: network tunnel */
	cdev_disk_init(NVND,vnd),	/* 41: vnode disk driver */
	cdev_audio_init(NAUDIO,audio),	/* 42: generic audio I/O */
	cdev_notdef(),			/* 43 */
	cdev_video_init(NVIDEO,video),	/* 44: generic video I/O */
	cdev_random_init(1,random),	/* 45: random data source */
	cdev_ocis_init(NPCTR,pctr),	/* 46: performance counters */
	cdev_disk_init(NRD,rd),		/* 47: ram disk driver */
	cdev_notdef(),			/* 48 */
	cdev_bktr_init(NBKTR,bktr),     /* 49: Bt848 video capture device */
	cdev_ksyms_init(NKSYMS,ksyms),	/* 50: Kernel symbols device */
	cdev_notdef(),			/* 51 */
	cdev_midi_init(NMIDI,midi),	/* 52: MIDI I/O */
	cdev_notdef(),			/* 53 was: sequencer I/O */
	cdev_notdef(),			/* 54 was: RAIDframe disk driver */
	cdev_notdef(),			/* 55: */
	/* The following slots are reserved for isdn4bsd. */
	cdev_notdef(),			/* 56: i4b main device */
	cdev_notdef(),			/* 57: i4b control device */
	cdev_notdef(),			/* 58: i4b raw b-channel access */
	cdev_notdef(),			/* 59: i4b trace device */
	cdev_notdef(),			/* 60: i4b phone device */
	/* End of reserved slots for isdn4bsd. */
	cdev_usb_init(NUSB,usb),	/* 61: USB controller */
	cdev_usbdev_init(NUHID,uhid),	/* 62: USB generic HID */
	cdev_usbdev_init(NUGEN,ugen),	/* 63: USB generic driver */
	cdev_ulpt_init(NULPT,ulpt),	/* 64: USB printers */
	cdev_notdef(),			/* 65: urio */
	cdev_tty_init(NUCOM,ucom),	/* 66: USB tty */
	cdev_mouse_init(NWSKBD, wskbd),	/* 67: keyboards */
	cdev_mouse_init(NWSMOUSE,	/* 68: mice */
	    wsmouse),
	cdev_mouse_init(NWSMUX, wsmux),	/* 69: ws multiplexor */
	cdev_notdef(),			/* 70: was: /dev/crypto */
	cdev_tty_init(NCZ,cztty),	/* 71: Cyclades-Z serial port */
#ifdef USER_PCICONF
	cdev_pci_init(NPCI,pci),        /* 72: PCI user */
#else
	cdev_notdef(),
#endif
	cdev_pf_init(NPF,pf),		/* 73: packet filter */
	cdev_notdef(),			/* 74: ALTQ (deprecated) */
	cdev_notdef(),
	cdev_radio_init(NRADIO, radio), /* 76: generic radio I/O */
	cdev_notdef(),			/* 77: was USB scanners */
	cdev_systrace_init(NSYSTRACE,systrace),	/* 78: system call tracing */
	cdev_bio_init(NBIO,bio),	/* 79: ioctl tunnel */
	cdev_notdef(),			/* 80: gpr? XXX */
	cdev_ptm_init(NPTY,ptm),	/* 81: pseudo-tty ptm device */
	cdev_hotplug_init(NHOTPLUG,hotplug), /* 82: devices hot plugging */
	cdev_acpi_init(NACPI,acpi),	/* 83: ACPI */
	cdev_notdef(),
	cdev_nvram_init(NNVRAM,nvram),	/* 85: NVRAM interface */
	cdev_notdef(),			/* 86 */
	cdev_drm_init(NDRM,drm),	/* 87: drm */
	cdev_gpio_init(NGPIO,gpio),	/* 88: gpio */
	cdev_vscsi_init(NVSCSI,vscsi),	/* 89: vscsi */
	cdev_disk_init(1,diskmap),	/* 90: disk mapper */
	cdev_pppx_init(NPPPX,pppx),     /* 91: pppx */
	cdev_fuse_init(NFUSE,fuse),	/* 92: fuse */
};
int	nchrdev = nitems(cdevsw);

int	mem_no = 2;	/* major device number of memory special file */

/*
 * Swapdev is a fake device implemented
 * in sw.c used only internally to get to swstrategy.
 * It cannot be provided to the users, because the
 * swstrategy routine munches the b_dev and b_blkno entries
 * before calling the appropriate driver.  This would horribly
 * confuse, e.g. the hashing routines. Instead, /dev/drum is
 * provided as a character (raw) device.
 */
dev_t	swapdev = makedev(1, 0);

/*
 * Returns true if dev is /dev/mem or /dev/kmem.
 */
int
iskmemdev(dev_t dev)
{

	return (major(dev) == mem_no && (minor(dev) < 2 || minor(dev) == 14));
}

/*
 * Returns true if dev is /dev/zero.
 */
int
iszerodev(dev_t dev)
{

	return (major(dev) == mem_no && minor(dev) == 12);
}

dev_t
getnulldev(void)
{
	return makedev(mem_no, 2);
}

int chrtoblktbl[] = {
	/*VCHR*/	/*VBLK*/
	/*  0 */	NODEV,
	/*  1 */	NODEV,
	/*  2 */	NODEV,
	/*  3 */	0,		/* wd */
	/*  4 */	NODEV,
	/*  5 */	NODEV,
	/*  6 */	NODEV,
	/*  7 */	NODEV,
	/*  8 */	NODEV,
	/*  9 */	2,		/* fd */
	/* 10 */	NODEV,
	/* 11 */	NODEV,
	/* 12 */	NODEV,
	/* 13 */	4,		/* sd */
	/* 14 */	5,		/* st */
	/* 15 */	6,		/* cd */
	/* 16 */	NODEV,
	/* 17 */	NODEV,
	/* 18 */	NODEV,
	/* 19 */	NODEV,
	/* 20 */	NODEV,
	/* 21 */	NODEV,
	/* 22 */	NODEV,
	/* 23 */	NODEV,
	/* 24 */	NODEV,
	/* 25 */	NODEV,
	/* 26 */	NODEV,
	/* 27 */	NODEV,
	/* 28 */	NODEV,
	/* 29 */	NODEV,
	/* 30 */	NODEV,
	/* 31 */	NODEV,
	/* 32 */	NODEV,
	/* 33 */	NODEV,
	/* 34 */	NODEV,
	/* 35 */	NODEV,
	/* 36 */	NODEV,
	/* 37 */	NODEV,
	/* 38 */	NODEV,
	/* 39 */	NODEV,
	/* 40 */	NODEV,
	/* 41 */	14,		/* vnd */
	/* 42 */	NODEV,
	/* 43 */	NODEV,
	/* 44 */	NODEV,
	/* 45 */	NODEV,
	/* 46 */	NODEV,
	/* 47 */	17,		/* rd */
	/* 48 */	NODEV,
	/* 49 */	NODEV,
	/* 50 */	NODEV,
	/* 51 */	NODEV,
	/* 52 */	NODEV,
	/* 53 */	NODEV,
	/* 54 */	19,		/* raid */
};

int nchrtoblktbl = nitems(chrtoblktbl);

/*
 * In order to map BSD bdev numbers of disks to their BIOS equivalents
 * we use several heuristics, one being using checksums of the first
 * few blocks of a disk to get a signature we can match with /boot's
 * computed signatures.  To know where from to read, we must provide a
 * disk driver name -> bdev major number table, which follows.
 * Note: floppies are not included as those are differentiated by the BIOS.
 */
int findblkmajor(struct device *dv);
dev_t dev_rawpart(struct device *);	/* XXX */

dev_t
dev_rawpart(struct device *dv)
{
	int majdev;

	majdev = findblkmajor(dv);

	switch (majdev) {
	/* add here any device you want to be checksummed on boot */
	case 0:		/* wd */
	case 4:		/* sd */
		return (MAKEDISKDEV(majdev, dv->dv_unit, RAW_PART));
		break;
	default:
		;
	}

	return (NODEV);
}

#include <dev/cons.h>

cons_decl(com);
cons_decl(ws);

struct	consdev constab[] = {
#if NWSDISPLAY > 0
	cons_init(ws),
#endif
#if NCOM > 0
	cons_init(com),
#endif
	{ 0 },
};


code:
/*	$OpenBSD: consinit.c,v 1.5 2015/03/14 03:38:46 jsg Exp $	*/
/*	$NetBSD: consinit.c,v 1.2 2003/03/02 18:27:14 fvdl Exp $	*/

/*
 * Copyright (c) 1998
 *	Matthias Drochner.  All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 */

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/device.h>
#include <machine/bus.h>

void
consinit()
{
	/* already done in machdep.c */
}


code:
/*	$OpenBSD: copy.S,v 1.7 2015/04/25 21:31:24 guenther Exp $	*/
/*	$NetBSD: copy.S,v 1.1 2003/04/26 18:39:26 fvdl Exp $	*/

/*
 * Copyright (c) 2001 Wasabi Systems, Inc.
 * All rights reserved.
 *
 * Written by Frank van der Linden for Wasabi Systems, Inc.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. All advertising materials mentioning features or use of this software
 *    must display the following acknowledgement:
 *      This product includes software developed for the NetBSD Project by
 *      Wasabi Systems, Inc.
 * 4. The name of Wasabi Systems, Inc. may not be used to endorse
 *    or promote products derived from this software without specific prior
 *    written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL WASABI SYSTEMS, INC
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#include "assym.h"

#include <sys/errno.h>
#include <sys/syscall.h>

#include <machine/asm.h>
#include <machine/codepatch.h>

/*
 * As stac/clac SMAP instructions are 3 bytes, we want the fastest
 * 3 byte nop sequence possible here.  This will be replaced by
 * stac/clac instructions if SMAP is detected after booting.
 *
 * This would be 'nop (%rax)' if binutils could cope.
 * Intel documents multi-byte NOP sequences as being available
 * on all family 0x6 and 0xf processors (ie 686+)
 */
#define SMAP_NOP	.byte 0x0f, 0x1f, 0x00
#define SMAP_STAC	CODEPATCH_START			;\
			SMAP_NOP			;\
			CODEPATCH_END(CPTAG_STAC)
#define SMAP_CLAC	CODEPATCH_START			;\
			SMAP_NOP			;\
			CODEPATCH_END(CPTAG_CLAC)

/*
 * Copy routines from and to userland, plus a few more. See the
 * section 9 manpages for info. Some cases can be optimized more.
 *
 * I wonder if it's worthwhile to make these use SSE2 registers.
 */

ENTRY(kcopy)
	movq	CPUVAR(CURPCB),%rax
	pushq	PCB_ONFAULT(%rax)
	leaq	_C_LABEL(copy_fault)(%rip),%r11
	movq	%r11, PCB_ONFAULT(%rax)

	xchgq	%rdi,%rsi
	movq	%rdx,%rcx

	movq	%rdi,%rax
	subq	%rsi,%rax
	cmpq	%rcx,%rax		# overlapping?
	jb	1f
	shrq	$3,%rcx			# nope, copy forward by 64-bit words
	rep
	movsq

	movq	%rdx,%rcx
	andl	$7,%ecx			# any bytes left?
	rep
	movsb

	movq	CPUVAR(CURPCB),%rdx
	popq	PCB_ONFAULT(%rdx)
	xorq	%rax,%rax
	ret

1:	addq	%rcx,%rdi		# copy backward
	addq	%rcx,%rsi
	std
	andq	$7,%rcx			# any fractional bytes?
	decq	%rdi
	decq	%rsi
	rep
	movsb
	movq	%rdx,%rcx		# copy remainder by 64-bit words
	shrq	$3,%rcx
	subq	$7,%rsi
	subq	$7,%rdi
	rep
	movsq
	cld

	movq	CPUVAR(CURPCB),%rdx
	popq	PCB_ONFAULT(%rdx)
	xorq	%rax,%rax
	ret

ENTRY(copyout)
	pushq	$0

	xchgq	%rdi,%rsi
	movq	%rdx,%rax

	movq	%rdi,%rdx
	addq	%rax,%rdx
	jc	_C_LABEL(copy_efault)
	movq	$VM_MAXUSER_ADDRESS,%r8
	cmpq	%r8,%rdx
	ja	_C_LABEL(copy_efault)

	movq	CPUVAR(CURPCB),%rdx
	leaq	_C_LABEL(copy_fault)(%rip),%r11
	movq	%r11,PCB_ONFAULT(%rdx)
	SMAP_STAC
	movq	%rax,%rcx
	shrq	$3,%rcx
	rep
	movsq
	movb	%al,%cl
	andb	$7,%cl
	rep
	movsb
	SMAP_CLAC
	popq	PCB_ONFAULT(%rdx)
	xorl	%eax,%eax
	ret

ENTRY(copyin)
	movq	CPUVAR(CURPCB),%rax
	pushq	$0
	leaq	_C_LABEL(copy_fault)(%rip),%r11
	movq	%r11,PCB_ONFAULT(%rax)
	SMAP_STAC
	xchgq	%rdi,%rsi
	movq	%rdx,%rax

	movq	%rsi,%rdx
	addq	%rax,%rdx
	jc	_C_LABEL(copy_efault)
	movq	$VM_MAXUSER_ADDRESS,%r8
	cmpq	%r8,%rdx
	ja	_C_LABEL(copy_efault)

3:	/* bcopy(%rsi, %rdi, %rax); */
	movq	%rax,%rcx
	shrq	$3,%rcx
	rep
	movsq
	movb	%al,%cl
	andb	$7,%cl
	rep
	movsb

	SMAP_CLAC
	movq	CPUVAR(CURPCB),%rdx
	popq	PCB_ONFAULT(%rdx)
	xorl	%eax,%eax
	ret

NENTRY(copy_efault)
	movq	$EFAULT,%rax

NENTRY(copy_fault)
	SMAP_CLAC
	movq	CPUVAR(CURPCB),%rdx
	popq	PCB_ONFAULT(%rdx)
	ret

ENTRY(copyoutstr)
	xchgq	%rdi,%rsi
	movq	%rdx,%r8
	movq	%rcx,%r9

5:	movq	CPUVAR(CURPCB),%rax
	leaq	_C_LABEL(copystr_fault)(%rip),%r11
	movq	%r11,PCB_ONFAULT(%rax)
	SMAP_STAC
	/*
	 * Get min(%rdx, VM_MAXUSER_ADDRESS-%rdi).
	 */
	movq	$VM_MAXUSER_ADDRESS,%rax
	subq	%rdi,%rax
	jbe	_C_LABEL(copystr_efault)	/* die if CF == 1 || ZF == 1 */
	cmpq	%rdx,%rax
	jae	1f
	movq	%rax,%rdx
	movq	%rax,%r8

1:	incq	%rdx

1:	decq	%rdx
	jz	2f
	lodsb
	stosb
	testb	%al,%al
	jnz	1b

	/* Success -- 0 byte reached. */
	decq	%rdx
	xorq	%rax,%rax
	jmp	copystr_return

2:	/* rdx is zero -- return EFAULT or ENAMETOOLONG. */
	movq	$VM_MAXUSER_ADDRESS,%r11
	cmpq	%r11,%rdi
	jae	_C_LABEL(copystr_efault)
	movq	$ENAMETOOLONG,%rax
	jmp	copystr_return

ENTRY(copyinstr)
	xchgq	%rdi,%rsi
	movq	%rdx,%r8
	movq	%rcx,%r9

	movq	CPUVAR(CURPCB),%rcx
	leaq	_C_LABEL(copystr_fault)(%rip),%r11
	movq	%r11,PCB_ONFAULT(%rcx)
	SMAP_STAC

	/*
	 * Get min(%rdx, VM_MAXUSER_ADDRESS-%rsi).
	 */
	movq	$VM_MAXUSER_ADDRESS,%rax
	subq	%rsi,%rax
	jbe	_C_LABEL(copystr_efault)	/* die if CF == 1 || ZF == 1 */
	cmpq	%rdx,%rax
	jae	1f
	movq	%rax,%rdx
	movq	%rax,%r8

1:	incq	%rdx

1:	decq	%rdx
	jz	2f
	lodsb
	stosb
	testb	%al,%al
	jnz	1b

	/* Success -- 0 byte reached. */
	decq	%rdx
	xorq	%rax,%rax
	jmp	copystr_return

2:	/* edx is zero -- return EFAULT or ENAMETOOLONG. */
	movq	$VM_MAXUSER_ADDRESS,%r11
	cmpq	%r11,%rsi
	jae	_C_LABEL(copystr_efault)
	movq	$ENAMETOOLONG,%rax
	jmp	copystr_return

ENTRY(copystr_efault)
	movl	$EFAULT,%eax

ENTRY(copystr_fault)
copystr_return:
	SMAP_CLAC
	/* Set *lencopied and return %eax. */
	movq	CPUVAR(CURPCB),%rcx
	movq	$0,PCB_ONFAULT(%rcx)
	testq	%r9,%r9
	jz	8f
	subq	%rdx,%r8
	movq	%r8,(%r9)

8:	ret

ENTRY(copystr)
	xchgq	%rdi,%rsi
	movq	%rdx,%r8

	incq	%rdx

1:	decq	%rdx
	jz	4f
	lodsb
	stosb
	testb	%al,%al
	jnz	1b

	/* Success -- 0 byte reached. */
	decq	%rdx
	xorl	%eax,%eax
	jmp	6f

4:	/* edx is zero -- return ENAMETOOLONG. */
	movl	$ENAMETOOLONG,%eax

6:	/* Set *lencopied and return %eax. */
	testq	%rcx,%rcx
	jz	7f
	subq	%rdx,%r8
	movq	%r8,(%rcx)

7:	ret

.globl _C_LABEL(_stac)
_C_LABEL(_stac):
	stac

.globl _C_LABEL(_clac)
_C_LABEL(_clac):
	clac


code:
/*	$OpenBSD: cpu.c,v 1.88 2015/07/18 19:21:02 sf Exp $	*/
/* $NetBSD: cpu.c,v 1.1 2003/04/26 18:39:26 fvdl Exp $ */

/*-
 * Copyright (c) 2000 The NetBSD Foundation, Inc.
 * All rights reserved.
 *
 * This code is derived from software contributed to The NetBSD Foundation
 * by RedBack Networks Inc.
 *
 * Author: Bill Sommerfeld
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

/*
 * Copyright (c) 1999 Stefan Grefen
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. All advertising materials mentioning features or use of this software
 *    must display the following acknowledgement:
 *      This product includes software developed by the NetBSD
 *      Foundation, Inc. and its contributors.
 * 4. Neither the name of The NetBSD Foundation nor the names of its
 *    contributors may be used to endorse or promote products derived
 *    from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND ANY
 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR AND CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */

#include "lapic.h"
#include "ioapic.h"

#include <sys/param.h>
#include <sys/timeout.h>
#include <sys/systm.h>
#include <sys/device.h>
#include <sys/malloc.h>
#include <sys/memrange.h>
#include <dev/rndvar.h>
#include <sys/atomic.h>

#include <uvm/uvm_extern.h>

#include <machine/codepatch.h>
#include <machine/cpu.h>
#include <machine/cpufunc.h>
#include <machine/cpuvar.h>
#include <machine/pmap.h>
#include <machine/vmparam.h>
#include <machine/mpbiosvar.h>
#include <machine/pcb.h>
#include <machine/specialreg.h>
#include <machine/segments.h>
#include <machine/gdt.h>
#include <machine/pio.h>

#if NLAPIC > 0
#include <machine/apicvar.h>
#include <machine/i82489reg.h>
#include <machine/i82489var.h>
#endif

#if NIOAPIC > 0
#include <machine/i82093var.h>
#endif

#include <dev/ic/mc146818reg.h>
#include <amd64/isa/nvram.h>
#include <dev/isa/isareg.h>

#ifdef HIBERNATE
#include <sys/hibernate.h>
#include <machine/hibernate.h>
#endif /* HIBERNATE */

int     cpu_match(struct device *, void *, void *);
void    cpu_attach(struct device *, struct device *, void *);
int     cpu_activate(struct device *, int);
void	patinit(struct cpu_info *ci);

struct cpu_softc {
	struct device sc_dev;		/* device tree glue */
	struct cpu_info *sc_info;	/* pointer to CPU info */
};

#ifndef SMALL_KERNEL
void	replacesmap(void);

extern long _stac;
extern long _clac;

void
replacesmap(void)
{
	static int replacedone = 0;
	int s;

	if (replacedone)
		return;
	replacedone = 1;

	s = splhigh();

	codepatch_replace(CPTAG_STAC, &_stac, 3);
	codepatch_replace(CPTAG_CLAC, &_clac, 3);

	splx(s);
}
#endif /* !SMALL_KERNEL */

#ifdef MULTIPROCESSOR
int mp_cpu_start(struct cpu_info *);
void mp_cpu_start_cleanup(struct cpu_info *);
struct cpu_functions mp_cpu_funcs = { mp_cpu_start, NULL,
				      mp_cpu_start_cleanup };
#endif /* MULTIPROCESSOR */

struct cfattach cpu_ca = {
	sizeof(struct cpu_softc), cpu_match, cpu_attach, NULL, cpu_activate
};

struct cfdriver cpu_cd = {
	NULL, "cpu", DV_DULL
};

/*
 * Statically-allocated CPU info for the primary CPU (or the only
 * CPU, on uniprocessors).  The CPU info list is initialized to
 * point at it.
 */
struct cpu_info cpu_info_primary = { 0, &cpu_info_primary };

struct cpu_info *cpu_info_list = &cpu_info_primary;

#ifdef MULTIPROCESSOR
/*
 * Array of CPU info structures.  Must be statically-allocated because
 * curproc, etc. are used early.
 */
struct cpu_info *cpu_info[MAXCPUS] = { &cpu_info_primary };

void    	cpu_hatch(void *);
void    	cpu_boot_secondary(struct cpu_info *ci);
void    	cpu_start_secondary(struct cpu_info *ci);
#endif

int
cpu_match(struct device *parent, void *match, void *aux)
{
	struct cfdata *cf = match;
	struct cpu_attach_args *caa = aux;

	if (strcmp(caa->caa_name, cf->cf_driver->cd_name) != 0)
		return 0;

	if (cf->cf_unit >= MAXCPUS)
		return 0;

	return 1;
}

static void
cpu_vm_init(struct cpu_info *ci)
{
	int ncolors = 2, i;

	for (i = CAI_ICACHE; i <= CAI_L2CACHE; i++) {
		struct x86_cache_info *cai;
		int tcolors;

		cai = &ci->ci_cinfo[i];

		tcolors = atop(cai->cai_totalsize);
		switch(cai->cai_associativity) {
		case 0xff:
			tcolors = 1; /* fully associative */
			break;
		case 0:
		case 1:
			break;
		default:
			tcolors /= cai->cai_associativity;
		}
		ncolors = max(ncolors, tcolors);
	}

#ifdef notyet
	/*
	 * Knowing the size of the largest cache on this CPU, re-color
	 * our pages.
	 */
	if (ncolors <= uvmexp.ncolors)
		return;
	printf("%s: %d page colors\n", ci->ci_dev->dv_xname, ncolors);
	uvm_page_recolor(ncolors);
#endif
}

void	cpu_idle_mwait_cycle(void);
void	cpu_init_mwait(struct cpu_softc *);

u_int	cpu_mwait_size, cpu_mwait_states;

void
cpu_idle_mwait_cycle(void)
{
	struct cpu_info *ci = curcpu();

	if ((read_rflags() & PSL_I) == 0)
		panic("idle with interrupts blocked!");

	/* something already queued? */
	if (ci->ci_schedstate.spc_whichqs != 0)
		return;

	/*
	 * About to idle; setting the MWAIT_IN_IDLE bit tells
	 * cpu_unidle() that it can't be a no-op and tells cpu_kick()
	 * that it doesn't need to use an IPI.  We also set the
	 * MWAIT_KEEP_IDLING bit: those routines clear it to stop
	 * the mwait.  Once they're set, we do a final check of the
	 * queue, in case another cpu called setrunqueue() and added
	 * something to the queue and called cpu_unidle() between
	 * the check in sched_idle() and here.
	 */
	atomic_setbits_int(&ci->ci_mwait, MWAIT_IDLING | MWAIT_ONLY);
	if (ci->ci_schedstate.spc_whichqs == 0) {
		monitor(&ci->ci_mwait, 0, 0);
		if ((ci->ci_mwait & MWAIT_IDLING) == MWAIT_IDLING)
			mwait(0, 0);
	}

	/* done idling; let cpu_kick() know that an IPI is required */
	atomic_clearbits_int(&ci->ci_mwait, MWAIT_IDLING);
}

void
cpu_init_mwait(struct cpu_softc *sc)
{
	unsigned int smallest, largest, extensions, c_substates;

	if ((cpu_ecxfeature & CPUIDECX_MWAIT) == 0)
		return;

	/* get the monitor granularity */
	CPUID(0x5, smallest, largest, extensions, cpu_mwait_states);
	smallest &= 0xffff;
	largest  &= 0xffff;

	printf("%s: mwait min=%u, max=%u", sc->sc_dev.dv_xname,
	    smallest, largest);
	if (extensions & 0x1) {
		if (cpu_mwait_states > 0) {
			c_substates = cpu_mwait_states;
			printf(", C-substates=%u", 0xf & c_substates);
			while ((c_substates >>= 4) > 0)
				printf(".%u", 0xf & c_substates);
		}
		if (extensions & 0x2)
			printf(", IBE");
	} else {
		/* substates not supported, forge the default: just C1 */
		cpu_mwait_states = 1 << 4;
	}

	/* paranoia: check the values */
	if (smallest < sizeof(int) || largest < smallest ||
	    (largest & (sizeof(int)-1)))
		printf(" (bogus)");
	else
		cpu_mwait_size = largest;
	printf("\n");

	/* enable use of mwait; may be overriden by acpicpu later */
	if (cpu_mwait_size > 0)
		cpu_idle_cycle_fcn = &cpu_idle_mwait_cycle;
}

void
cpu_attach(struct device *parent, struct device *self, void *aux)
{
	struct cpu_softc *sc = (void *) self;
	struct cpu_attach_args *caa = aux;
	struct cpu_info *ci;
#if defined(MULTIPROCESSOR)
	int cpunum = sc->sc_dev.dv_unit;
	vaddr_t kstack;
	struct pcb *pcb;
#endif

	/*
	 * If we're an Application Processor, allocate a cpu_info
	 * structure, otherwise use the primary's.
	 */
	if (caa->cpu_role == CPU_ROLE_AP) {
		ci = malloc(sizeof(*ci), M_DEVBUF, M_WAITOK|M_ZERO);
#if defined(MULTIPROCESSOR)
		if (cpu_info[cpunum] != NULL)
			panic("cpu at apic id %d already attached?", cpunum);
		cpu_info[cpunum] = ci;
#endif
#ifdef TRAPLOG
		ci->ci_tlog_base = malloc(sizeof(struct tlog),
		    M_DEVBUF, M_WAITOK);
#endif
	} else {
		ci = &cpu_info_primary;
#if defined(MULTIPROCESSOR)
		if (caa->cpu_number != lapic_cpu_number()) {
			panic("%s: running cpu is at apic %d"
			    " instead of at expected %d",
			    sc->sc_dev.dv_xname, lapic_cpu_number(), caa->cpu_number);
		}
#endif
	}

	ci->ci_self = ci;
	sc->sc_info = ci;

	ci->ci_dev = self;
	ci->ci_apicid = caa->cpu_number;
#ifdef MULTIPROCESSOR
	ci->ci_cpuid = cpunum;
#else
	ci->ci_cpuid = 0;	/* False for APs, but they're not used anyway */
#endif
	ci->ci_func = caa->cpu_func;
	ci->ci_handled_intr_level = IPL_NONE;

#if defined(MULTIPROCESSOR)
	/*
	 * Allocate UPAGES contiguous pages for the idle PCB and stack.
	 */
	kstack = uvm_km_alloc (kernel_map, USPACE);
	if (kstack == 0) {
		if (caa->cpu_role != CPU_ROLE_AP) {
			panic("cpu_attach: unable to allocate idle stack for"
			    " primary");
		}
		printf("%s: unable to allocate idle stack\n",
		    sc->sc_dev.dv_xname);
		return;
	}
	pcb = ci->ci_idle_pcb = (struct pcb *) kstack;
	memset(pcb, 0, USPACE);

	pcb->pcb_kstack = kstack + USPACE - 16;
	pcb->pcb_rbp = pcb->pcb_rsp = kstack + USPACE - 16;
	pcb->pcb_pmap = pmap_kernel();
	pcb->pcb_cr0 = rcr0();
	pcb->pcb_cr3 = pcb->pcb_pmap->pm_pdirpa;
#endif

	/* further PCB init done later. */

	printf(": ");

	switch (caa->cpu_role) {
	case CPU_ROLE_SP:
		printf("(uniprocessor)\n");
		ci->ci_flags |= CPUF_PRESENT | CPUF_SP | CPUF_PRIMARY;
		cpu_intr_init(ci);
		identifycpu(ci);
#ifdef MTRR
		mem_range_attach();
#endif /* MTRR */
		cpu_init(ci);
		cpu_init_mwait(sc);
		break;

	case CPU_ROLE_BP:
		printf("apid %d (boot processor)\n", caa->cpu_number);
		ci->ci_flags |= CPUF_PRESENT | CPUF_BSP | CPUF_PRIMARY;
		cpu_intr_init(ci);
		identifycpu(ci);
#ifdef MTRR
		mem_range_attach();
#endif /* MTRR */
		cpu_init(ci);

#if NLAPIC > 0
		/*
		 * Enable local apic
		 */
		lapic_enable();
		lapic_calibrate_timer(ci);
#endif
#if NIOAPIC > 0
		ioapic_bsp_id = caa->cpu_number;
#endif
		cpu_init_mwait(sc);
		break;

	case CPU_ROLE_AP:
		/*
		 * report on an AP
		 */
		printf("apid %d (application processor)\n", caa->cpu_number);

#if defined(MULTIPROCESSOR)
		cpu_intr_init(ci);
		gdt_alloc_cpu(ci);
		sched_init_cpu(ci);
		cpu_start_secondary(ci);
		ncpus++;
		if (ci->ci_flags & CPUF_PRESENT) {
			ci->ci_next = cpu_info_list->ci_next;
			cpu_info_list->ci_next = ci;
		}
#else
		printf("%s: not started\n", sc->sc_dev.dv_xname);
#endif
		break;

	default:
		panic("unknown processor type??");
	}
	cpu_vm_init(ci);

#if defined(MULTIPROCESSOR)
	if (mp_verbose) {
		printf("%s: kstack at 0x%lx for %d bytes\n",
		    sc->sc_dev.dv_xname, kstack, USPACE);
		printf("%s: idle pcb at %p, idle sp at 0x%llx\n",
		    sc->sc_dev.dv_xname, pcb, pcb->pcb_rsp);
	}
#endif
}

/*
 * Initialize the processor appropriately.
 */

void
cpu_init(struct cpu_info *ci)
{
	u_int cr4;

	/* configure the CPU if needed */
	if (ci->cpu_setup != NULL)
		(*ci->cpu_setup)(ci);
	/*
	 * We do this here after identifycpu() because errata may affect
	 * what we do.
	 */
	patinit(ci);

	lcr0(rcr0() | CR0_WP);
	cr4 = rcr4() | CR4_DEFAULT;
	if (ci->ci_feature_sefflags & SEFF0EBX_SMEP)
		cr4 |= CR4_SMEP;
#ifndef SMALL_KERNEL
	if (ci->ci_feature_sefflags & SEFF0EBX_SMAP)
		cr4 |= CR4_SMAP;
	if (ci->ci_feature_sefflags & SEFF0EBX_FSGSBASE)
		cr4 |= CR4_FSGSBASE;
#endif
	if (cpu_ecxfeature & CPUIDECX_XSAVE)
		cr4 |= CR4_OSXSAVE;
	lcr4(cr4);

	if (cpu_ecxfeature & CPUIDECX_XSAVE) {
		u_int32_t eax, ebx, ecx, edx;

		xsave_mask = XCR0_X87 | XCR0_SSE;
		CPUID_LEAF(0xd, 0, eax, ebx, ecx, edx);
		if (eax & XCR0_AVX)
			xsave_mask |= XCR0_AVX;
		xsetbv(0, xsave_mask);
		CPUID_LEAF(0xd, 0, eax, ebx, ecx, edx);
		fpu_save_len = ebx;
	}

#ifdef MULTIPROCESSOR
	ci->ci_flags |= CPUF_RUNNING;
	tlbflushg();
#endif
}

#ifdef MULTIPROCESSOR
void
cpu_boot_secondary_processors(void)
{
	struct cpu_info *ci;
	u_long i;

	for (i=0; i < MAXCPUS; i++) {
		ci = cpu_info[i];
		if (ci == NULL)
			continue;
		if (ci->ci_idle_pcb == NULL)
			continue;
		if ((ci->ci_flags & CPUF_PRESENT) == 0)
			continue;
		if (ci->ci_flags & (CPUF_BSP | CPUF_SP | CPUF_PRIMARY))
			continue;
		ci->ci_randseed = (arc4random() & 0x7fffffff) + 1;
		cpu_boot_secondary(ci);
	}
}

void
cpu_init_idle_pcbs(void)
{
	struct cpu_info *ci;
	u_long i;

	for (i=0; i < MAXCPUS; i++) {
		ci = cpu_info[i];
		if (ci == NULL)
			continue;
		if (ci->ci_idle_pcb == NULL)
			continue;
		if ((ci->ci_flags & CPUF_PRESENT) == 0)
			continue;
		x86_64_init_pcb_tss_ldt(ci);
	}
}

void
cpu_start_secondary(struct cpu_info *ci)
{
	int i;

	ci->ci_flags |= CPUF_AP;

	pmap_kenter_pa(MP_TRAMPOLINE, MP_TRAMPOLINE, PROT_READ | PROT_EXEC);
	pmap_kenter_pa(MP_TRAMP_DATA, MP_TRAMP_DATA, PROT_READ | PROT_WRITE);

	CPU_STARTUP(ci);

	/*
	 * wait for it to become ready
	 */
	for (i = 100000; (!(ci->ci_flags & CPUF_PRESENT)) && i>0;i--) {
		delay(10);
	}
	if (! (ci->ci_flags & CPUF_PRESENT)) {
		printf("%s: failed to become ready\n", ci->ci_dev->dv_xname);
#if defined(MPDEBUG) && defined(DDB)
		printf("dropping into debugger; continue from here to resume boot\n");
		Debugger();
#endif
	}

	if ((ci->ci_flags & CPUF_IDENTIFIED) == 0) {
		atomic_setbits_int(&ci->ci_flags, CPUF_IDENTIFY);

		/* wait for it to identify */
		for (i = 100000; (ci->ci_flags & CPUF_IDENTIFY) && i > 0; i--)
			delay(10);

		if (ci->ci_flags & CPUF_IDENTIFY)
			printf("%s: failed to identify\n",
			    ci->ci_dev->dv_xname);
	}

	CPU_START_CLEANUP(ci);

	pmap_kremove(MP_TRAMPOLINE, PAGE_SIZE);
	pmap_kremove(MP_TRAMP_DATA, PAGE_SIZE);
}

void
cpu_boot_secondary(struct cpu_info *ci)
{
	int i;

	atomic_setbits_int(&ci->ci_flags, CPUF_GO);

	for (i = 100000; (!(ci->ci_flags & CPUF_RUNNING)) && i>0;i--) {
		delay(10);
	}
	if (! (ci->ci_flags & CPUF_RUNNING)) {
		printf("cpu failed to start\n");
#if defined(MPDEBUG) && defined(DDB)
		printf("dropping into debugger; continue from here to resume boot\n");
		Debugger();
#endif
	}
}

/*
 * The CPU ends up here when it's ready to run
 * This is called from code in mptramp.s; at this point, we are running
 * in the idle pcb/idle stack of the new cpu.  When this function returns,
 * this processor will enter the idle loop and start looking for work.
 *
 * XXX should share some of this with init386 in machdep.c
 */
void
cpu_hatch(void *v)
{
	struct cpu_info *ci = (struct cpu_info *)v;
	int s;

	cpu_init_msrs(ci);

#ifdef DEBUG
	if (ci->ci_flags & CPUF_PRESENT)
		panic("%s: already running!?", ci->ci_dev->dv_xname);
#endif

	ci->ci_flags |= CPUF_PRESENT;

	lapic_enable();
	lapic_startclock();

	if ((ci->ci_flags & CPUF_IDENTIFIED) == 0) {
		/*
		 * We need to wait until we can identify, otherwise dmesg
		 * output will be messy.
		 */
		while ((ci->ci_flags & CPUF_IDENTIFY) == 0)
			delay(10);

		identifycpu(ci);

		/* Signal we're done */
		atomic_clearbits_int(&ci->ci_flags, CPUF_IDENTIFY);
		/* Prevent identifycpu() from running again */
		atomic_setbits_int(&ci->ci_flags, CPUF_IDENTIFIED);
	}

	while ((ci->ci_flags & CPUF_GO) == 0)
		delay(10);
#ifdef HIBERNATE
	if ((ci->ci_flags & CPUF_PARK) != 0) {
		atomic_clearbits_int(&ci->ci_flags, CPUF_PARK);
		hibernate_drop_to_real_mode();
	}
#endif /* HIBERNATE */

#ifdef DEBUG
	if (ci->ci_flags & CPUF_RUNNING)
		panic("%s: already running!?", ci->ci_dev->dv_xname);
#endif

	lcr0(ci->ci_idle_pcb->pcb_cr0);
	cpu_init_idt();
	lapic_set_lvt();
	gdt_init_cpu(ci);
	fpuinit(ci);

	lldt(0);

	cpu_init(ci);

	/* Re-initialise memory range handling on AP */
	if (mem_range_softc.mr_op != NULL)
		mem_range_softc.mr_op->initAP(&mem_range_softc);

	s = splhigh();
	lcr8(0);
	enable_intr();

	nanouptime(&ci->ci_schedstate.spc_runtime);
	splx(s);

	SCHED_LOCK(s);
	cpu_switchto(NULL, sched_chooseproc());
}

#if defined(DDB)

#include <ddb/db_output.h>
#include <machine/db_machdep.h>

/*
 * Dump cpu information from ddb.
 */
void
cpu_debug_dump(void)
{
	struct cpu_info *ci;
	CPU_INFO_ITERATOR cii;

	db_printf("addr		dev	id	flags	ipis	curproc		fpcurproc\n");
	CPU_INFO_FOREACH(cii, ci) {
		db_printf("%p	%s	%u	%x	%x	%10p	%10p\n",
		    ci,
		    ci->ci_dev == NULL ? "BOOT" : ci->ci_dev->dv_xname,
		    ci->ci_cpuid,
		    ci->ci_flags, ci->ci_ipis,
		    ci->ci_curproc,
		    ci->ci_fpcurproc);
	}
}
#endif

int
mp_cpu_start(struct cpu_info *ci)
{
	unsigned short dwordptr[2];

	/*
	 * "The BSP must initialize CMOS shutdown code to 0Ah ..."
	 */

	outb(IO_RTC, NVRAM_RESET);
	outb(IO_RTC+1, NVRAM_RESET_JUMP);

	/*
	 * "and the warm reset vector (DWORD based at 40:67) to point
	 * to the AP startup code ..."
	 */

	dwordptr[0] = 0;
	dwordptr[1] = MP_TRAMPOLINE >> 4;

	pmap_kenter_pa(0, 0, PROT_READ | PROT_WRITE);
	memcpy((u_int8_t *) 0x467, dwordptr, 4);
	pmap_kremove(0, PAGE_SIZE);

#if NLAPIC > 0
	/*
	 * ... prior to executing the following sequence:"
	 */

	if (ci->ci_flags & CPUF_AP) {
		x86_ipi_init(ci->ci_apicid);

		delay(10000);

		if (cpu_feature & CPUID_APIC) {
			x86_ipi(MP_TRAMPOLINE/PAGE_SIZE, ci->ci_apicid,
			    LAPIC_DLMODE_STARTUP);
			delay(200);

			x86_ipi(MP_TRAMPOLINE/PAGE_SIZE, ci->ci_apicid,
			    LAPIC_DLMODE_STARTUP);
			delay(200);
		}
	}
#endif
	return 0;
}

void
mp_cpu_start_cleanup(struct cpu_info *ci)
{
	/*
	 * Ensure the NVRAM reset byte contains something vaguely sane.
	 */

	outb(IO_RTC, NVRAM_RESET);
	outb(IO_RTC+1, NVRAM_RESET_RST);
}
#endif	/* MULTIPROCESSOR */

typedef void (vector)(void);
extern vector Xsyscall, Xsyscall32;

void
cpu_init_msrs(struct cpu_info *ci)
{
	wrmsr(MSR_STAR,
	    ((uint64_t)GSEL(GCODE_SEL, SEL_KPL) << 32) |
	    ((uint64_t)GSEL(GUCODE32_SEL, SEL_UPL) << 48));
	wrmsr(MSR_LSTAR, (uint64_t)Xsyscall);
	wrmsr(MSR_CSTAR, (uint64_t)Xsyscall32);
	wrmsr(MSR_SFMASK, PSL_NT|PSL_T|PSL_I|PSL_C|PSL_D);

	wrmsr(MSR_FSBASE, 0);
	wrmsr(MSR_GSBASE, (u_int64_t)ci);
	wrmsr(MSR_KERNELGSBASE, 0);
}

void
patinit(struct cpu_info *ci)
{
	extern int	pmap_pg_wc;
	u_int64_t	reg;

	if ((ci->ci_feature_flags & CPUID_PAT) == 0)
		return;
#define	PATENTRY(n, type)	(type << ((n) * 8))
#define	PAT_UC		0x0UL
#define	PAT_WC		0x1UL
#define	PAT_WT		0x4UL
#define	PAT_WP		0x5UL
#define	PAT_WB		0x6UL
#define	PAT_UCMINUS	0x7UL
	/*
	 * Set up PAT bits.
	 * The default pat table is the following:
	 * WB, WT, UC- UC, WB, WT, UC-, UC
	 * We change it to:
	 * WB, WC, UC-, UC, WB, WC, UC-, UC.
	 * i.e change the WT bit to be WC.
	 */
	reg = PATENTRY(0, PAT_WB) | PATENTRY(1, PAT_WC) |
	    PATENTRY(2, PAT_UCMINUS) | PATENTRY(3, PAT_UC) |
	    PATENTRY(4, PAT_WB) | PATENTRY(5, PAT_WC) |
	    PATENTRY(6, PAT_UCMINUS) | PATENTRY(7, PAT_UC);

	wrmsr(MSR_CR_PAT, reg);
	pmap_pg_wc = PG_WC;
}

struct timeout rdrand_tmo;
void rdrand(void *);

void
rdrand(void *v)
{
	struct timeout *tmo = v;
	extern int	has_rdrand;
	union {
		uint64_t u64;
		uint32_t u32[2];
	} r;
	uint64_t valid;
	int i;

	if (has_rdrand == 0)
		return;
	for (i = 0; i < 2; i++) {
		__asm volatile(
		    "xor	%1, %1\n\t"
		    "rdrand	%0\n\t"
		    "rcl	$1, %1\n"
		    : "=r" (r.u64), "=r" (valid) );

		if (valid) {
			add_true_randomness(r.u32[0]);
			add_true_randomness(r.u32[1]);
		}
	}

	if (tmo)
		timeout_add_msec(tmo, 10);
}

int
cpu_activate(struct device *self, int act)
{
	struct cpu_softc *sc = (struct cpu_softc *)self;

	switch (act) {
	case DVACT_RESUME:
		if (sc->sc_info->ci_cpuid == 0)
			rdrand(NULL);
		break;
	}

	return (0);
}