code:
/* $OpenBSD: tc_dma_3000_300.c,v 1.2 2008/06/26 05:42:09 ray Exp $ */
/* $NetBSD: tc_dma_3000_300.c,v 1.3 1997/09/02 13:20:29 thorpej Exp $ */

/*-
 * Copyright (c) 1997 The NetBSD Foundation, Inc.
 * All rights reserved.
 *
 * This code is derived from software contributed to The NetBSD Foundation
 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
 * NASA Ames Research Center.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#include <sys/param.h>

#include <machine/bus.h>

#include <alpha/tc/tc_dma_3000_300.h>

/*
 * Return the DMA tag for the given slot.
 */
bus_dma_tag_t
tc_dma_get_tag_3000_300(slot)
	int slot;
{
	extern struct alpha_bus_dma_tag tc_dmat_direct;

	/* Pelicans don't have SGMAPs. */
	return (&tc_dmat_direct);
}


code:
/* $OpenBSD: tc_dma_3000_300.h,v 1.2 2008/06/26 05:42:09 ray Exp $ */
/* $NetBSD: tc_dma_3000_300.h,v 1.2 1997/06/07 00:02:17 thorpej Exp $ */

/*-
 * Copyright (c) 1997 The NetBSD Foundation, Inc.
 * All rights reserved.
 *
 * This code is derived from software contributed to The NetBSD Foundation
 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
 * NASA Ames Research Center.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

bus_dma_tag_t	tc_dma_get_tag_3000_300(int);

license: unknown


code:
/* $OpenBSD: tc_dma_3000_500.c,v 1.6 2014/12/13 21:05:32 doug Exp $ */
/* $NetBSD: tc_dma_3000_500.c,v 1.13 2001/07/19 06:40:03 thorpej Exp $ */

/*-
 * Copyright (c) 1997, 1998 The NetBSD Foundation, Inc.
 * All rights reserved.
 *
 * This code is derived from software contributed to The NetBSD Foundation
 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
 * NASA Ames Research Center.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#define _ALPHA_BUS_DMA_PRIVATE

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/device.h>
#include <sys/kernel.h>
#include <sys/malloc.h>

#include <uvm/uvm_extern.h>

#include <machine/bus.h>

#include <dev/tc/tcvar.h>
#include <alpha/tc/tc_sgmap.h>
#include <alpha/tc/tc_dma_3000_500.h>

struct alpha_bus_dma_tag tc_dmat_sgmap = {
	NULL,				/* _cookie */
	0,				/* _wbase */
	0,				/* _wsize */
	NULL,				/* _next_window */
	0,				/* _boundary */
	NULL,				/* _sgmap */
	NULL,				/* _get_tag */
	tc_bus_dmamap_create_sgmap,
	tc_bus_dmamap_destroy_sgmap,
	tc_bus_dmamap_load_sgmap,
	tc_bus_dmamap_load_mbuf_sgmap,
	tc_bus_dmamap_load_uio_sgmap,
	tc_bus_dmamap_load_raw_sgmap,
	tc_bus_dmamap_unload_sgmap,
	_bus_dmamap_sync,
	_bus_dmamem_alloc,
	_bus_dmamem_free,
	_bus_dmamem_map,
	_bus_dmamem_unmap,
	_bus_dmamem_mmap,
};

struct tc_dma_slot_info {
	struct alpha_sgmap tdsi_sgmap;		/* sgmap for slot */
	struct alpha_bus_dma_tag tdsi_dmat;	/* dma tag for slot */
};
struct tc_dma_slot_info *tc_dma_slot_info;

void
tc_dma_init_3000_500(nslots)
	int nslots;
{
	extern struct alpha_bus_dma_tag tc_dmat_direct;
	int i;

	/* Allocate per-slot DMA info. */
	tc_dma_slot_info = mallocarray(nslots, sizeof(struct tc_dma_slot_info),
	    M_DEVBUF, M_NOWAIT | M_ZERO);
	if (tc_dma_slot_info == NULL)
		panic("tc_dma_init: can't allocate per-slot DMA info");

	/* Default all slots to direct-mapped. */
	for (i = 0; i < nslots; i++)
		memcpy(&tc_dma_slot_info[i].tdsi_dmat, &tc_dmat_direct,
		    sizeof(tc_dma_slot_info[i].tdsi_dmat));
}

/*
 * Return the DMA tag for the given slot.
 */
bus_dma_tag_t
tc_dma_get_tag_3000_500(slot)
	int slot;
{

	return (&tc_dma_slot_info[slot].tdsi_dmat);
}

/*
 * Create a TurboChannel SGMAP-mapped DMA map.
 */
int
tc_bus_dmamap_create_sgmap(t, size, nsegments, maxsegsz, boundary,
    flags, dmamp)
	bus_dma_tag_t t;
	bus_size_t size;
	int nsegments;
	bus_size_t maxsegsz;
	bus_size_t boundary;
	int flags;
	bus_dmamap_t *dmamp;
{
	bus_dmamap_t map;
	int error;

	error = _bus_dmamap_create(t, size, nsegments, maxsegsz,
	    boundary, flags, dmamp);
	if (error)
		return (error);

	map = *dmamp;

	/* XXX BUS_DMA_ALLOCNOW */

	return (error);
}

/*
 * Destroy a TurboChannel SGMAP-mapped DMA map.
 */
void
tc_bus_dmamap_destroy_sgmap(t, map)
	bus_dma_tag_t t;
	bus_dmamap_t map;
{

	KASSERT(map->dm_mapsize == 0);

	_bus_dmamap_destroy(t, map);
}

/*
 * Load a TurboChannel SGMAP-mapped DMA map with a linear buffer.
 */
int
tc_bus_dmamap_load_sgmap(t, map, buf, buflen, p, flags)
	bus_dma_tag_t t;
	bus_dmamap_t map;
	void *buf;
	bus_size_t buflen;
	struct proc *p;
	int flags;
{
	struct tc_dma_slot_info *tdsi = t->_cookie;

	return (tc_sgmap_load(t, map, buf, buflen, p, flags,
	    &tdsi->tdsi_sgmap));
}

/*
 * Load a TurboChannel SGMAP-mapped DMA map with an mbuf chain.
 */
int
tc_bus_dmamap_load_mbuf_sgmap(t, map, m, flags)
	bus_dma_tag_t t;
	bus_dmamap_t map;
	struct mbuf *m;
	int flags;
{
	struct tc_dma_slot_info *tdsi = t->_cookie;

	return (tc_sgmap_load_mbuf(t, map, m, flags, &tdsi->tdsi_sgmap));
}

/*
 * Load a TurboChannel SGMAP-mapped DMA map with a uio.
 */
int
tc_bus_dmamap_load_uio_sgmap(t, map, uio, flags)
	bus_dma_tag_t t;
	bus_dmamap_t map;
	struct uio *uio;
	int flags;
{
	struct tc_dma_slot_info *tdsi = t->_cookie;

	return (tc_sgmap_load_uio(t, map, uio, flags, &tdsi->tdsi_sgmap));
}

/*
 * Load a TurboChannel SGMAP-mapped DMA map with raw memory.
 */
int
tc_bus_dmamap_load_raw_sgmap(t, map, segs, nsegs, size, flags)
	bus_dma_tag_t t;
	bus_dmamap_t map;
	bus_dma_segment_t *segs;
	int nsegs;
	bus_size_t size;
	int flags;
{
	struct tc_dma_slot_info *tdsi = t->_cookie;

	return (tc_sgmap_load_raw(t, map, segs, nsegs, size, flags,
	    &tdsi->tdsi_sgmap));
}

/*
 * Unload a TurboChannel SGMAP-mapped DMA map.
 */
void
tc_bus_dmamap_unload_sgmap(t, map)
	bus_dma_tag_t t;
	bus_dmamap_t map;
{
	struct tc_dma_slot_info *tdsi = t->_cookie;

	/*
	 * Invalidate any SGMAP page table entries used by this
	 * mapping.
	 */
	tc_sgmap_unload(t, map, &tdsi->tdsi_sgmap);

	/*
	 * Do the generic bits of the unload.
	 */
	_bus_dmamap_unload(t, map);
}


code:
/* $OpenBSD: tc_dma_3000_500.h,v 1.2 2008/06/26 05:42:09 ray Exp $ */
/* $NetBSD: tc_dma_3000_500.h,v 1.2 1997/06/07 00:02:19 thorpej Exp $ */

/*-
 * Copyright (c) 1997 The NetBSD Foundation, Inc.
 * All rights reserved.
 *
 * This code is derived from software contributed to The NetBSD Foundation
 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
 * NASA Ames Research Center.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

void	tc_dma_init_3000_500(int);
bus_dma_tag_t tc_dma_get_tag_3000_500(int);

int	tc_bus_dmamap_create_sgmap(bus_dma_tag_t, bus_size_t, int,
	    bus_size_t, bus_size_t, int, bus_dmamap_t *);
void	tc_bus_dmamap_destroy_sgmap(bus_dma_tag_t, bus_dmamap_t);
int	tc_bus_dmamap_load_sgmap(bus_dma_tag_t, bus_dmamap_t, void *,
	    bus_size_t, struct proc *, int);
int	tc_bus_dmamap_load_mbuf_sgmap(bus_dma_tag_t, bus_dmamap_t,
	    struct mbuf *, int);
int	tc_bus_dmamap_load_uio_sgmap(bus_dma_tag_t, bus_dmamap_t,
	    struct uio *, int);
int	tc_bus_dmamap_load_raw_sgmap(bus_dma_tag_t, bus_dmamap_t,
	    bus_dma_segment_t *, int, bus_size_t, int);
void	tc_bus_dmamap_unload_sgmap(bus_dma_tag_t, bus_dmamap_t);


code:
/* $OpenBSD: tc_sgmap.c,v 1.2 2008/06/26 05:42:09 ray Exp $ */
/* $NetBSD: tc_sgmap.c,v 1.5 2001/07/19 06:40:03 thorpej Exp $ */

/*-
 * Copyright (c) 1997 The NetBSD Foundation, Inc.
 * All rights reserved.
 *
 * This code is derived from software contributed to The NetBSD Foundation
 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
 * NASA Ames Research Center.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/device.h>
#include <sys/malloc.h>
#include <sys/mbuf.h>
#include <sys/proc.h>

#include <uvm/uvm_extern.h>

#include <machine/bus.h>

#include <alpha/tc/tc_sgmap.h>

#include <alpha/dev/sgmap_typedep.c>


code:
/* $OpenBSD: tc_sgmap.h,v 1.2 2008/06/26 05:42:09 ray Exp $ */
/* $NetBSD: tc_sgmap.h,v 1.2 1997/06/07 00:02:20 thorpej Exp $ */

/*-
 * Copyright (c) 1997 The NetBSD Foundation, Inc.
 * All rights reserved.
 *
 * This code is derived from software contributed to The NetBSD Foundation
 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
 * NASA Ames Research Center.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#define	SGMAP_TYPE		tc_sgmap
#define	SGMAP_PTE_TYPE		u_int32_t
#define	SGMAP_PTE_SPACING	2

/*
 * A TurboChannel SGMAP page table entry looks like this:
 *
 * 31                    23  22  21 20           4 3    0
 * |     Discarded     | V | F | P | Page address | UNP |
 *
 * The page address is bits <29:13> of the physical address of the
 * page.  The V bit is set if the PTE holds a valid mapping.
 * The F (funny) bit forces a parity error.  The P bit is a
 * hardware-generated parity bit.
 */
#define	SGPTE_PGADDR_SHIFT	9
#define	SGPTE_VALID		0x00800000

#include <alpha/dev/sgmapvar.h>
#include <alpha/dev/sgmap_typedep.h>


code:
/* $OpenBSD: tcasic.c,v 1.16 2010/11/11 17:54:52 miod Exp $ */
/* $NetBSD: tcasic.c,v 1.36 2001/08/23 01:16:52 nisimura Exp $ */

/*
 * Copyright (c) 1994, 1995, 1996 Carnegie-Mellon University.
 * All rights reserved.
 *
 * Author: Chris G. Demetriou
 *
 * Permission to use, copy, modify and distribute this software and
 * its documentation is hereby granted, provided that both the copyright
 * notice and this permission notice appear in all copies of the
 * software, derivative works or modified versions, and any portions
 * thereof, and that both notices appear in supporting documentation.
 *
 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
 * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
 *
 * Carnegie Mellon requests users of this software to return to
 *
 *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
 *  School of Computer Science
 *  Carnegie Mellon University
 *  Pittsburgh PA 15213-3890
 *
 * any improvements or extensions that they make and grant Carnegie the
 * rights to redistribute these changes.
 */

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/device.h>

#include <machine/autoconf.h>
#include <machine/rpb.h>
#include <machine/cpu.h>

#include <dev/tc/tcvar.h>
#include <alpha/tc/tc_conf.h>

/* Definition of the driver for autoconfig. */
int	tcasicmatch(struct device *, void *, void *);
void	tcasicattach(struct device *, struct device *, void *);

struct cfattach tcasic_ca = {
	sizeof (struct device), tcasicmatch, tcasicattach,
};

struct cfdriver tcasic_cd = {
	NULL, "tcasic", DV_DULL,
};

int	tcasicprint(void *, const char *);

/* There can be only one. */
int	tcasicfound;

int
tcasicmatch(parent, cfdata, aux)
	struct device *parent;
	void *cfdata, *aux;
{
	struct mainbus_attach_args *ma = aux;

        /* Make sure that we're looking for a TurboChannel ASIC. */
        if (strcmp(ma->ma_name, tcasic_cd.cd_name))
                return (0);

	if (tcasicfound)
		return (0);

	return (1);
}

void
tcasicattach(parent, self, aux)
	struct device *parent;
	struct device *self;
	void *aux;
{
	struct tcbus_attach_args tba;
	void (*intr_setup)(void);
	void (*iointr)(void *, unsigned long);

	printf("\n");
	tcasicfound = 1;

	switch (cputype) {
#ifdef DEC_3000_500
	case ST_DEC_3000_500:

		intr_setup = tc_3000_500_intr_setup;
		iointr = tc_3000_500_iointr;

		if ((hwrpb->rpb_type & SV_ST_MASK) == SV_ST_SANDPIPER)
			tba.tba_speed = TC_SPEED_22_5_MHZ;
		else
			tba.tba_speed = TC_SPEED_25_MHZ;
		tba.tba_nslots = tc_3000_500_nslots;
		tba.tba_slots = tc_3000_500_slots;
		if (hwrpb->rpb_variation & SV_GRAPHICS) {
			tba.tba_nbuiltins = tc_3000_500_graphics_nbuiltins;
			tba.tba_builtins = tc_3000_500_graphics_builtins;
		} else {
			tba.tba_nbuiltins = tc_3000_500_nographics_nbuiltins;
			tba.tba_builtins = tc_3000_500_nographics_builtins;
		}
		tba.tba_intr_establish = tc_3000_500_intr_establish;
		tba.tba_intr_disestablish = tc_3000_500_intr_disestablish;
		tba.tba_get_dma_tag = tc_dma_get_tag_3000_500;

		/* Do 3000/500-specific DMA setup now. */
		tc_dma_init_3000_500(tc_3000_500_nslots);
		break;
#endif /* DEC_3000_500 */

#ifdef DEC_3000_300
	case ST_DEC_3000_300:

		intr_setup = tc_3000_300_intr_setup;
		iointr = tc_3000_300_iointr;

		tba.tba_speed = TC_SPEED_12_5_MHZ;
		tba.tba_nslots = tc_3000_300_nslots;
		tba.tba_slots = tc_3000_300_slots;
		tba.tba_nbuiltins = tc_3000_300_nbuiltins;
		tba.tba_builtins = tc_3000_300_builtins;
		tba.tba_intr_establish = tc_3000_300_intr_establish;
		tba.tba_intr_disestablish = tc_3000_300_intr_disestablish;
		tba.tba_get_dma_tag = tc_dma_get_tag_3000_300;
		break;
#endif /* DEC_3000_300 */

	default:
		panic("tcasicattach: bad cputype");
	}

	tba.tba_busname = "tc";
	tba.tba_memt = tc_bus_mem_init(NULL);

	tc_dma_init();

	(*intr_setup)();

	/* They all come in at 0x800. */
	scb_set(0x800, iointr, NULL);

	config_found(self, &tba, tcasicprint);
}

int
tcasicprint(aux, pnp)
	void *aux;
	const char *pnp;
{

	/* only TCs can attach to tcasics; easy. */
	if (pnp)
		printf("tc at %s", pnp);
	return (UNCONF);
}

#ifdef notyet

#include "wsdisplay.h"

#if NWSDISPLAY > 0

#include "sfb.h"
#include "sfbp.h"
#include "cfb.h"
#include "mfb.h"
#include "tfb.h"
#include "px.h"
#include "pxg.h"

extern void	sfb_cnattach(tc_addr_t);
extern void	sfbp_cnattach(tc_addr_t);
extern void	cfb_cnattach(tc_addr_t);
extern void	mfb_cnattach(tc_addr_t);
extern void	tfb_cnattach(tc_addr_t);
extern void	px_cnattach(tc_addr_t);
extern void	pxg_cnattach(tc_addr_t);
extern int	tc_checkslot(tc_addr_t, char *);

struct cnboards {
	const char	*cb_tcname;
	void	(*cb_cnattach)(tc_addr_t);
} static const cnboards[] = {
#if NSFB > 0
	{ "PMAGB-BA", sfb_cnattach },
#endif
#if NSFBP > 0
	{ "PMAGD   ", sfbp_cnattach },
#endif
#if NCFB > 0
	{ "PMAG-BA ", cfb_cnattach },
#endif
#if NMFB > 0
	{ "PMAG-AA ", mfb_cnattach },
#endif
#if NTFB > 0
	{ "PMAG-JA ", tfb_cnattach },
#endif
#if NPX > 0
	{ "PMAG-CA ", px_cnattach },
#endif
#if NPXG > 0
	{ "PMAG-DA ", pxg_cnattach },
	{ "PMAG-FA ", pxg_cnattach },
	{ "PMAG-FB ", pxg_cnattach },
	{ "PMAGB-FA", pxg_cnattach },
	{ "PMAGB-FB", pxg_cnattach },
#endif
};

/*
 * tc_fb_cnattach --
 *	Attempt to attach the appropriate display driver to the
 * output console.
 */
int
tc_fb_cnattach(tcaddr)
	tc_addr_t tcaddr;
{
	char tcname[TC_ROM_LLEN];
	int i;

	if (tc_badaddr(tcaddr) || (tc_checkslot(tcaddr, tcname) == 0))
		return (EINVAL);

	for (i = 0; i < sizeof(cnboards) / sizeof(cnboards[0]); i++)
		if (strncmp(tcname, cnboards[i].cb_tcname, TC_ROM_LLEN) == 0)
			break;

	if (i == sizeof(cnboards) / sizeof(cnboards[0]))
		return (ENXIO);

	(cnboards[i].cb_cnattach)(tcaddr);
	return (0);
}
#endif /* if NWSDISPLAY > 0 */

#else

int
tc_fb_cnattach(tcaddr)
	tc_addr_t tcaddr;
{
		return (ENXIO);
}
#endif


code:
/Makefile/1.10/Sun Dec  8 14:46:39 2013//TOPENBSD_5_8_BASE
D/amd64////
D/compile////
D/conf////
D/include////
D/isa////
D/pci////
D/stand////


code:
#	$OpenBSD: Makefile,v 1.10 2013/12/08 14:46:39 espie Exp $

S=	${.CURDIR}/../..
KFILE=	GENERIC
.if exists(conf/GENERIC.MP)
KFILE=	GENERIC.MP
.endif
TDIRS=	${_arch} include pci isa
TAGS=	${.CURDIR}/tags

NOPROG=
NOMAN=
NOOBJ=
SUBDIR=	stand

# config the fattest kernel we can find into a temporary dir
# to create a Makefile.  Then use make to pull some variables
# out and push them into the sub-shell to expand the paths,
# and finally run ctags.
tags::
	TDIR=`mktemp -d /tmp/_tagXXXXXXXXXX` || exit 1; \
	eval "S=${S}" && \
	config -s ${S} -b $${TDIR} ${.CURDIR}/conf/${KFILE} && \
	eval "_arch=\"`make -V _arch -f $${TDIR}/Makefile`\"" && \
	eval "_mach=\"`make -V _mach -f $${TDIR}/Makefile`\"" && \
	eval "_machdir=\$S/arch/$${_mach}" && \
	eval "_archdir=\$S/arch/$${_arch}" && \
	eval "HFILES=\"`find $S \( -path $S/'arch' -o -path $S/stand -o -path $S/lib/libsa -o -path $S'/lib/libkern/arch' \) -prune -o -name '*.h'; find $${_machdir} $${_archdir} $S/lib/libkern/arch/$${_mach} \( -name boot -o -name stand \) -prune -o -name '*.h'`\"" && \
	eval "SFILES=\"`make -V SFILES -f $${TDIR}/Makefile`\"" && \
	eval "CFILES=\"`make -V CFILES -f $${TDIR}/Makefile`\"" && \
	eval "AFILES=\"`make -V AFILES -f $${TDIR}/Makefile`\"" && \
	ctags -wd -f ${TAGS} $${CFILES} $${HFILES} && \
	egrep "^[_A-Z]*ENTRY[_A-Z]*\(.*\)" $${SFILES} $${AFILES} | \
	    sed "s;\\([^:]*\\):\\([^(]*\\)(\\([^, )]*\\)\\(.*\\);\\3	\\1	/^\\2(\\3\\4$$/;" \
	    >> ${TAGS} && \
	sort -o ${TAGS} ${TAGS} && \
	rm -rf $${TDIR}

links:
	-for i in conf ${TDIRS}; do \
	    (cd $$i && rm -f tags; ln -s tags tags); done

obj:	_SUBDIRUSE

.include <bsd.prog.mk>


code:
/*	$OpenBSD: acpi_machdep.c,v 1.70 2015/03/16 20:31:47 deraadt Exp $	*/
/*
 * Copyright (c) 2005 Thorsten Lockert <tholo@sigmasoft.com>
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/device.h>
#include <sys/malloc.h>
#include <sys/memrange.h>
#include <sys/proc.h>
#include <sys/user.h>
#include <sys/reboot.h>
#include <sys/hibernate.h>

#include <uvm/uvm_extern.h>

#include <machine/bus.h>
#include <machine/biosvar.h>
#include <machine/isa_machdep.h>

#include <machine/cpu.h>
#include <machine/cpufunc.h>
#include <machine/cpuvar.h>

#include <dev/isa/isareg.h>
#include <dev/acpi/acpireg.h>
#include <dev/acpi/acpivar.h>
#include <dev/acpi/acpidev.h>

#include "isa.h"
#include "ioapic.h"
#include "lapic.h"

#if NIOAPIC > 0
#include <machine/i82093var.h>
#endif

#if NLAPIC > 0
#include <machine/i82489reg.h>
#include <machine/i82489var.h>
#endif

extern u_char acpi_real_mode_resume[], acpi_resume_end[];
extern u_char acpi_tramp_data_start[], acpi_tramp_data_end[];
extern u_int32_t acpi_pdirpa;
extern paddr_t tramp_pdirpa;

extern int acpi_savecpu(void) __returns_twice;

#define ACPI_BIOS_RSDP_WINDOW_BASE        0xe0000
#define ACPI_BIOS_RSDP_WINDOW_SIZE        0x20000

u_int8_t	*acpi_scan(struct acpi_mem_map *, paddr_t, size_t);

int
acpi_map(paddr_t pa, size_t len, struct acpi_mem_map *handle)
{
	paddr_t pgpa = trunc_page(pa);
	paddr_t endpa = round_page(pa + len);
	vaddr_t va = uvm_km_valloc(kernel_map, endpa - pgpa);

	if (va == 0)
		return (ENOMEM);

	handle->baseva = va;
	handle->va = (u_int8_t *)(va + (pa & PGOFSET));
	handle->vsize = endpa - pgpa;
	handle->pa = pa;

	do {
		pmap_kenter_pa(va, pgpa, PROT_READ | PROT_WRITE);
		va += NBPG;
		pgpa += NBPG;
	} while (pgpa < endpa);

	return 0;
}

void
acpi_unmap(struct acpi_mem_map *handle)
{
	pmap_kremove(handle->baseva, handle->vsize);
	uvm_km_free(kernel_map, handle->baseva, handle->vsize);
}

u_int8_t *
acpi_scan(struct acpi_mem_map *handle, paddr_t pa, size_t len)
{
	size_t i;
	u_int8_t *ptr;
	struct acpi_rsdp1 *rsdp;

	if (acpi_map(pa, len, handle))
		return (NULL);
	for (ptr = handle->va, i = 0; i < len; ptr += 16, i += 16) {
		/* is there a valid signature? */
		if (memcmp(ptr, RSDP_SIG, sizeof(RSDP_SIG) - 1))
			continue;

		/* is the checksum valid? */
		if (acpi_checksum(ptr, sizeof(struct acpi_rsdp1)) != 0)
			continue;

		/* check the extended checksum as needed */
		rsdp = (struct acpi_rsdp1 *)ptr;
		if (rsdp->revision == 0)
			return (ptr);
		else if (rsdp->revision >= 2 && rsdp->revision <= 4)
			if (acpi_checksum(ptr, sizeof(struct acpi_rsdp)) == 0)
				return (ptr);
	}
	acpi_unmap(handle);

	return (NULL);
}

int
acpi_probe(struct device *parent, struct cfdata *match,
    struct bios_attach_args *ba)
{
	struct acpi_mem_map handle;
	u_int8_t *ptr;
	paddr_t ebda;

	/*
	 * First try to find ACPI table entries in the EBDA
	 */
	if (acpi_map(0, NBPG, &handle))
		printf("acpi: failed to map BIOS data area\n");
	else {
		ebda = *(const u_int16_t *)(&handle.va[0x40e]);
		ebda <<= 4;
		acpi_unmap(&handle);

		if (ebda && ebda < IOM_BEGIN) {
			if ((ptr = acpi_scan(&handle, ebda, 1024)))
				goto havebase;
		}
	}

	/*
	 * Next try to find the ACPI table entries in the
	 * BIOS memory
	 */
	if ((ptr = acpi_scan(&handle, ACPI_BIOS_RSDP_WINDOW_BASE,
	    ACPI_BIOS_RSDP_WINDOW_SIZE)))
		goto havebase;

	return (0);

havebase:
	ba->ba_acpipbase = ptr - handle.va + handle.pa;
	acpi_unmap(&handle);

	return (1);
}

/*
 * Acquire the global lock.  If busy, set the pending bit.  The caller
 * will wait for notification from the BIOS that the lock is available
 * and then attempt to acquire it again.
 */
int
acpi_acquire_glk(uint32_t *lock)
{
	uint32_t	new, old;

	do {
		old = *lock;
		new = (old & ~GL_BIT_PENDING) | GL_BIT_OWNED;
		if ((old & GL_BIT_OWNED) != 0)
			new |= GL_BIT_PENDING;
	} while (atomic_cas_uint(lock, old, new) != old);

	return ((new & GL_BIT_PENDING) == 0);
}

/*
 * Release the global lock, returning whether there is a waiter pending.
 * If the BIOS set the pending bit, OSPM must notify the BIOS when it
 * releases the lock.
 */
int
acpi_release_glk(uint32_t *lock)
{
	uint32_t	new, old;

	do {
		old = *lock;
		new = old & ~(GL_BIT_PENDING | GL_BIT_OWNED);
	} while (atomic_cas_uint(lock, old, new) != old);

	return ((old & GL_BIT_PENDING) != 0);
}

void
acpi_attach_machdep(struct acpi_softc *sc)
{
	extern void (*cpuresetfn)(void);

	sc->sc_interrupt = isa_intr_establish(NULL, sc->sc_fadt->sci_int,
	    IST_LEVEL, IPL_TTY, acpi_interrupt, sc, sc->sc_dev.dv_xname);
	cpuresetfn = acpi_reset;

#ifndef SMALL_KERNEL
	/*
	 * Sanity check before setting up trampoline.
	 * Ensure the trampoline size is < PAGE_SIZE
	 */
	KASSERT(acpi_resume_end - acpi_real_mode_resume < PAGE_SIZE);

	/* Map ACPI tramp code and data pages RW for copy */
	pmap_kenter_pa(ACPI_TRAMPOLINE, ACPI_TRAMPOLINE,
	    PROT_READ | PROT_WRITE);
	pmap_kenter_pa(ACPI_TRAMP_DATA, ACPI_TRAMP_DATA,
	    PROT_READ | PROT_WRITE);

	memcpy((caddr_t)ACPI_TRAMPOLINE, acpi_real_mode_resume,
	    acpi_resume_end - acpi_real_mode_resume);
	memcpy((caddr_t)ACPI_TRAMP_DATA, acpi_tramp_data_start,
	    acpi_tramp_data_end - acpi_tramp_data_start);

	acpi_pdirpa = tramp_pdirpa;

	/* Unmap, will be remapped in acpi_sleep_cpu */
	pmap_kremove(ACPI_TRAMPOLINE, PAGE_SIZE);
	pmap_kremove(ACPI_TRAMP_DATA, PAGE_SIZE);
#endif /* SMALL_KERNEL */
}

#ifndef SMALL_KERNEL

void
acpi_sleep_clocks(struct acpi_softc *sc, int state)
{
	rtcstop();

#if NLAPIC > 0
	lapic_disable();
#endif
}

/*
 * We repair the interrupt hardware so that any events which occur
 * will cause the least number of unexpected side effects.  We re-start
 * the clocks early because we will soon run AML whigh might do DELAY.
 */
void
acpi_resume_clocks(struct acpi_softc *sc)
{
#if NISA > 0
	i8259_default_setup();
#endif
	intr_calculatemasks(curcpu());

#if NIOAPIC > 0
	ioapic_enable();
#endif

#if NLAPIC > 0
	lapic_enable();
	if (initclock_func == lapic_initclocks)
		lapic_startclock();
	lapic_set_lvt();
#endif

	i8254_startclock();
	if (initclock_func == i8254_initclocks)
		rtcstart();		/* in i8254 mode, rtc is profclock */
}

/*
 * This function may not have local variables due to a bug between
 * acpi_savecpu() and the resume path.
 */
int
acpi_sleep_cpu(struct acpi_softc *sc, int state)
{
	/*
	 * ACPI defines two wakeup vectors. One is used for ACPI 1.0
	 * implementations - it's in the FACS table as wakeup_vector and
	 * indicates a 32-bit physical address containing real-mode wakeup
	 * code.
	 *
	 * The second wakeup vector is in the FACS table as
	 * x_wakeup_vector and indicates a 64-bit physical address
	 * containing protected-mode wakeup code.
	 */
	sc->sc_facs->wakeup_vector = (u_int32_t)ACPI_TRAMPOLINE;
	if (sc->sc_facs->length > 32 && sc->sc_facs->version >= 1)
		sc->sc_facs->x_wakeup_vector = 0;

	/* Map trampoline and data page */
	pmap_kenter_pa(ACPI_TRAMPOLINE, ACPI_TRAMPOLINE, PROT_READ | PROT_EXEC);
	pmap_kenter_pa(ACPI_TRAMP_DATA, ACPI_TRAMP_DATA,
		PROT_READ | PROT_WRITE);

	/*
	 * Copy the current cpu registers into a safe place for resume.
	 * acpi_savecpu actually returns twice - once in the suspend
	 * path and once in the resume path (see setjmp(3)).
	 */
	if (acpi_savecpu()) {
		/* Suspend path */
		fpusave_cpu(curcpu(), 1);
		wbinvd();

#ifdef HIBERNATE
		if (state == ACPI_STATE_S4) {
			if (hibernate_suspend()) {
				printf("%s: hibernate_suspend failed",
				    DEVNAME(sc));
				hibernate_free();
				return (ECANCELED);
			}
		}
#endif

		/*
		 * XXX
		 * Flag to disk drivers that they should "power down" the disk
		 * when we get to DVACT_POWERDOWN.
		 */
		boothowto |= RB_POWERDOWN;
		config_suspend_all(DVACT_POWERDOWN);
		boothowto &= ~RB_POWERDOWN;

		acpi_sleep_pm(sc, state);
		printf("%s: acpi_sleep_pm failed", DEVNAME(sc));
		return (ECANCELED);
	}
	/* Resume path */

	/* Reset the vectors */
	sc->sc_facs->wakeup_vector = 0;
	if (sc->sc_facs->length > 32 && sc->sc_facs->version >= 1)
		sc->sc_facs->x_wakeup_vector = 0;

	pmap_kremove(ACPI_TRAMPOLINE, PAGE_SIZE);
	pmap_kremove(ACPI_TRAMP_DATA, PAGE_SIZE);

	return (0);
}

void
acpi_resume_cpu(struct acpi_softc *sc)
{
	fpuinit(&cpu_info_primary);

	cpu_init(&cpu_info_primary);

	/* Re-initialise memory range handling on BSP */
	if (mem_range_softc.mr_op != NULL)
		mem_range_softc.mr_op->initAP(&mem_range_softc);
}

#ifdef MULTIPROCESSOR
void
acpi_sleep_mp(void)
{
	int i;

	sched_stop_secondary_cpus();
	KASSERT(CPU_IS_PRIMARY(curcpu()));

	/*
	 * Wait for cpus to halt so we know their FPU state has been
	 * saved and their caches have been written back.
	 */
	x86_broadcast_ipi(X86_IPI_HALT);
	for (i = 0; i < ncpus; i++) {
		struct cpu_info *ci = cpu_info[i];

		if (CPU_IS_PRIMARY(ci))
			continue;
		while (ci->ci_flags & CPUF_RUNNING)
			;
	}
}

void
acpi_resume_mp(void)
{
	void	cpu_start_secondary(struct cpu_info *ci);
	struct cpu_info *ci;
	struct proc *p;
	struct pcb *pcb;
	struct trapframe *tf;
	struct switchframe *sf;
	int i;

	/* XXX refactor with matching code in cpu.c */

	for (i = 0; i < MAXCPUS; i++) {
		ci = cpu_info[i];
		if (ci == NULL)
			continue;
		if (ci->ci_idle_pcb == NULL)
			continue;
		if (ci->ci_flags & (CPUF_BSP|CPUF_SP|CPUF_PRIMARY))
			continue;
		KASSERT((ci->ci_flags & CPUF_RUNNING) == 0);

		p = ci->ci_schedstate.spc_idleproc;
		pcb = &p->p_addr->u_pcb;

		tf = (struct trapframe *)pcb->pcb_kstack - 1;
		sf = (struct switchframe *)tf - 1;
		sf->sf_r12 = (u_int64_t)sched_idle;
		sf->sf_r13 = (u_int64_t)ci;
		sf->sf_rip = (u_int64_t)proc_trampoline;
		pcb->pcb_rsp = (u_int64_t)sf;
		pcb->pcb_rbp = 0;

		ci->ci_idepth = 0;
		ci->ci_handled_intr_level = IPL_NONE;

		ci->ci_flags &= ~CPUF_PRESENT;
		cpu_start_secondary(ci);
	}

	cpu_boot_secondary_processors();
	sched_start_secondary_cpus();
}
#endif /* MULTIPROCESSOR */

#endif /* ! SMALL_KERNEL */


code:
/* $OpenBSD: acpi_wakecode.S,v 1.36 2015/06/22 20:06:11 bluhm Exp $ */
/*
 * Copyright (c) 2001 Takanori Watanabe <takawata@jp.freebsd.org>
 * Copyright (c) 2001 Mitsuru IWASAKI <iwasaki@jp.freebsd.org>
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */
/*
 * Copyright (c) 2008, 2009 Mike Larkin <mlarkin@openbsd.org>
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

#define _ACPI_WAKECODE

#include "assym.h"
#include <machine/asm.h>
#ifdef HIBERNATE
#include <machine/hibernate_var.h>
#endif /* HIBERNATE */
#include <machine/specialreg.h>
#include <machine/param.h>
#include <machine/segments.h>
#include <dev/acpi/acpivar.h>

#define _ACPI_TRMP_LABEL(a) a = . - _C_LABEL(acpi_real_mode_resume) + \
	ACPI_TRAMPOLINE
#define _ACPI_TRMP_OFFSET(a) a = . - _C_LABEL(acpi_real_mode_resume)
#define _ACPI_TRMP_DATA_LABEL(a) a = . - _C_LABEL(acpi_tramp_data_start) + \
	ACPI_TRAMP_DATA
#define _ACPI_TRMP_DATA_OFFSET(a) a = . - _C_LABEL(acpi_tramp_data_start)
#define _ACPI_RM_CODE_SEG (ACPI_TRAMPOLINE >> 4)
#define _ACPI_RM_DATA_SEG (ACPI_TRAMP_DATA >> 4)

/*
 * On wakeup, we'll start executing at acpi_real_mode_resume.
 * This is based on the wakeup vector previously stored with
 * ACPI before we went to sleep. ACPI's wakeup vector is a
 * physical address - in our case, it's calculated and mapped
 * by the kernel and stuffed into a low page early in the boot
 * process.
 *
 * We wakeup in real mode, at some phys addr based on the ACPI
 * specification (cs = phys>>8, ip = phys & 0xF). For example,
 * if our phys addr is 0x13000, we'd have cs=0x1300,ip=0
 *
 * The wakeup code needs to do the following:
 *     1. Reenable the video display
 *     2. Enter 32 bit protected mode
 *     3. Reenable paging
 *     4. Enter long mode
 *     5. Restore saved CPU registers
 */
	.text
	.code16
	.align 4
	.global _C_LABEL(acpi_real_mode_resume)
	.global _C_LABEL(acpi_protected_mode_resume)
	.global _C_LABEL(acpi_long_mode_resume)
	.global _C_LABEL(acpi_resume_end)
	.global _C_LABEL(acpi_pdirpa)
	.global _C_LABEL(acpi_tramp_data_start)
	.global _C_LABEL(acpi_tramp_data_end)
_C_LABEL(acpi_real_mode_resume):
_ACPI_TRMP_OFFSET(acpi_s3_vector_real)
	nop
	cli
	cld

	/*
	 * Set up segment registers for real mode.
	 * We'll only be in real mode for a moment, and we don't have
	 * ant real dependencies on data or stack, so we'll just use
	 * the code segment for data and stack (eg, a 64k memory space).
	 */
	movw	$(_ACPI_RM_DATA_SEG), %ax
	movw	%ax, %ds
	movw	%ax, %ss
	movw	%cs, %ax
	movw	%ax, %es
	lidtl	clean_idt

	/*
	 * Set up stack to grow down from offset 0x0FFE.
	 * We will only be doing a few push/pops and no calls in real
	 * mode, so as long as the real mode code in the segment
	 * plus stack doesn't exceed 0x0FFE (4094) bytes, we'll be ok.
	 */
	movw	$0x0FFE,%sp

	/*
	 * Clear flags
	 */
	pushl	$0
	popfl

	/*
	 * Flush instruction prefetch queue
	 */
	jmp     1f
1:	jmp     1f
1:

	/*
	 * We're about to enter protected mode, so we need a GDT for that.
	 * Set up a temporary GDT describing 2 segments, one for code
	 * extending from 0x00000000-0xffffffff and one for data
	 * with the same range. This GDT will only be in use for a short
	 * time, until we restore the saved GDT that we had when we went
	 * to sleep.
	 */
	data32 addr32 lgdt	tmp_gdt

	/*
	 * Enable protected mode by setting the PE bit in CR0
	 */
	mov	%cr0,%eax
	orl	$(CR0_PE),%eax
	mov	%eax,%cr0

	/*
	 * Force CPU into protected mode by making an intersegment jump (to
	 * ourselves, just a few lines down from here). We rely on the kernel
	 * to fixup the jump target addres previously.
	 */
	ljmpl	$0x8, $acpi_protected_mode_trampoline

	.code32
	.align 16
_ACPI_TRMP_LABEL(acpi_protected_mode_trampoline)
_C_LABEL(acpi_protected_mode_resume):
	nop

	/*
	 * We're in protected mode now, without paging enabled.
	 *
	 * Set up segment selectors for protected mode.
	 * We've already set up our cs via the intersegment jump earlier,
	 * but we need to set ds,es,fs,gs,ss to all point to the
	 * 4GB flat data segment we defined earlier.
	 */
	movw	$GSEL(GDATA_SEL,SEL_KPL),%ax
	movw	%ax,%ds
	movw	%ax,%es
	movw	%ax,%gs
	movw	%ax,%ss
	movw	%ax,%fs

	/*
	 * Reset ESP based on protected mode. We can do this here
	 * because we haven't put anything on the stack via a
	 * call or push that we haven't cleaned up already.
	 */
	addl    $(ACPI_TRAMP_DATA), %esp

	/* Set CR4 to something sane for entry into long mode */
	mov	$(CR4_PAE|CR4_OSFXSR|CR4_OSXMMEXCPT|CR4_PSE),%eax
	mov	%eax,%cr4

	/*
	 * Set up a temporary long mode GDT describing 2
	 * segments, one for code and one for data.
	 */
	lgdt 	tmp_gdt64

	/* Restore saved EFER (LME, NXE, etc) */
	movl	$MSR_EFER, %ecx
	rdmsr
	movl	acpi_saved_efer, %eax
	andl 	$(EFER_LME | EFER_NXE | EFER_SCE), %eax
	wrmsr

	/* Reenable paging using temporary cr3 */
	movl	$acpi_pdirpa, %eax
	movl	(%eax), %eax
	movl	%eax, %cr3

	/* Flush the prefetch queue again */
	jmp 	1f
1:	jmp	1f
1:

	/* Reenable paging by setting the appropriate bits in CR0 */
	movl    %cr0,%eax
	orl     $(CR0_PE|CR0_PG|CR0_NE|CR0_TS|CR0_MP|CR0_WP),%eax
	movl    %eax,%cr0

	/* Flush the prefetch queue again */
	jmp 	1f
1:	jmp	1f
1:

	/* Enter long mode by making another intersegment jump */
	ljmp 	$0x8, $acpi_long_mode_trampoline

	.code64
	.align 16
_ACPI_TRMP_LABEL(acpi_long_mode_trampoline)
_C_LABEL(acpi_long_mode_resume):

	/* Reset stack */
	movq	$(ACPI_TRAMP_DATA + 0x0FF8), %rsp

	/* Load GDT based on our saved copy */
	lgdt	acpi_saved_gdt

	/* Reset segment registers */
	movw    $GSEL(GDATA_SEL, SEL_KPL),%ax
	movw    %ax,%ds
	movw    %ax,%es
	movw    %ax,%ss

	xorw	%ax, %ax
	movw	%ax, %fs
	movw	%ax, %gs

	/* Restore registers - start with the MSRs */
	movl	$MSR_STAR, %ecx
	movl	acpi_saved_star, %eax
	movl	acpi_saved_star+4, %edx
	wrmsr

	movl	$MSR_LSTAR, %ecx
	movl	acpi_saved_lstar, %eax
	movl	acpi_saved_lstar+4, %edx
	wrmsr

	movl	$MSR_CSTAR, %ecx
	movl	acpi_saved_cstar, %eax
	movl	acpi_saved_cstar+4, %edx
	wrmsr

	movl	$MSR_SFMASK, %ecx
	movl	acpi_saved_sfmask, %eax
	movl	acpi_saved_sfmask+4, %edx
	wrmsr

	movl	$MSR_FSBASE, %ecx
	movl	acpi_saved_fsbase, %eax
	movl	acpi_saved_fsbase+4, %edx
	wrmsr

	movl	$MSR_GSBASE, %ecx
	movl	acpi_saved_gsbase, %eax
	movl	acpi_saved_gsbase+4, %edx
	wrmsr

	movl	$MSR_KERNELGSBASE, %ecx
	movl	acpi_saved_kgs, %eax
	movl	acpi_saved_kgs+4, %edx
	wrmsr

	/* Restore control registers */
	movq	acpi_saved_cr8, %rax
	movq	%rax, %cr8
	movq	acpi_saved_cr4, %rax
	movq	%rax, %cr4
	movq	acpi_saved_cr3, %rax
	movq	%rax, %cr3

	/* Flush the prefetch queue again */
	jmp 	1f
1:	jmp	1f
1:

	movq	acpi_saved_cr2, %rax
	movq	%rax, %cr2
	movq	acpi_saved_cr0,	%rax
	movq	%rax, %cr0

	/* Flush the prefetch queue again */
	jmp 	1f
1:	jmp	1f
1:

	lldt	acpi_saved_ldt
	lidt	acpi_saved_idt

	/* Restore the saved task register */
	xorq	%rcx, %rcx
	movw	acpi_saved_tr, %cx
	movq	acpi_saved_gdt+2, %rax
	andb	$0xF9, 5(%rax,%rcx)
	ltr	%cx

	pushq 	acpi_saved_fl
	popfq

	movq    acpi_saved_rbx, %rbx
	movq    acpi_saved_rcx, %rcx
	movq    acpi_saved_rdx, %rdx
	movq    acpi_saved_rbp, %rbp
	movq    acpi_saved_rsi, %rsi
	movq    acpi_saved_rdi, %rdi
	movq    acpi_saved_rsp, %rsp

	movq    acpi_saved_r8, %r8
	movq    acpi_saved_r9, %r9
	movq    acpi_saved_r10, %r10
	movq    acpi_saved_r11, %r11
	movq    acpi_saved_r12, %r12
	movq    acpi_saved_r13, %r13
	movq    acpi_saved_r14, %r14
	movq    acpi_saved_r15, %r15

	/* Poke CR3 one more time. Might not be necessary */
	movq	acpi_saved_cr3, %rax
	movq	%rax, %cr3

	xorq	%rax, %rax
	jmp   	*acpi_saved_ret

#ifdef HIBERNATE
	/*
	 * hibernate_resume_machdep drops to real mode and
	 * restarts the OS using the saved S3 resume vector
	 */
	.code64
NENTRY(hibernate_resume_machdep)
	cli
	/* Jump to the identity mapped version of ourself */
	mov	$hibernate_resume_vector_2, %rax
	jmp	*%rax
_ACPI_TRMP_LABEL(hibernate_resume_vector_2)

	/* Get out of 64 bit CS */
	lgdtq	tmp_gdt6416

	/* Jump out of 64 bit mode, to hibernate_resume_vector_3 below */
	ljmp	*(hibernate_indirect_16)

_ACPI_TRMP_OFFSET(hibernate_resume_vector_3)
	.code16

	movl	%cr0, %eax
	/* Disable CR0.PG - no paging */
	andl	$(~CR0_PG), %eax
	/* Disable CR0.PE - real mode */
	andl	$(~CR0_PE), %eax
	movl	%eax, %cr0

	/* Set up real mode segment selectors */
	movw	$(_ACPI_RM_DATA_SEG), %ax
	movw	%ax, %ds
	movw	%ax, %ss
	movw	%ax, %es
	movw	%ax, %fs
	movw	%ax, %gs
	movl	$0x0FFE, %esp
	lidtl	clean_idt

	/* Jump to the S3 resume vector */
	ljmp	$(_ACPI_RM_CODE_SEG), $acpi_s3_vector_real

NENTRY(hibernate_drop_to_real_mode)
	.code64
	cli
	/* Jump to the identity mapped version of ourself */
	mov	$hibernate_resume_vector_2b, %rax
	jmp	*%rax
_ACPI_TRMP_LABEL(hibernate_resume_vector_2b)

	/* Get out of 64 bit CS */
	lgdtq	tmp_gdt6416

	/* Jump out of 64 bit mode, to hibernate_resume_vector_3b below */
	ljmp	*(hibernate_indirect_16b)

_ACPI_TRMP_OFFSET(hibernate_resume_vector_3b)
	.code16

	movl	%cr0, %eax
	/* Disable CR0.PG - no paging */
	andl	$(~CR0_PG), %eax
	/* Disable CR0.PE - real mode */
	andl	$(~CR0_PE), %eax
	movl	%eax, %cr0

	/* Set up real mode segment selectors */
	movw	$(_ACPI_RM_DATA_SEG), %ax
	movw	%ax, %ds
	movw	%ax, %es
	movw	%ax, %fs
	movw	%ax, %gs
	movw	%ax, %ss
	movl	$0x0FFE, %esp
	lidtl	clean_idt

_ACPI_TRMP_OFFSET(hib_hlt_real)
	hlt
	ljmp	$(_ACPI_RM_CODE_SEG), $hib_hlt_real

	.code64
	/* Switch to hibernate resume pagetable */
NENTRY(hibernate_activate_resume_pt_machdep)
	/* Enable large pages */
	movq	%cr4, %rax
	orq	$(CR4_PSE), %rax

	/* Disable global pages */
	andq	$(~CR4_PGE), %rax
	movq	%rax, %cr4

	wbinvd
	movq	$HIBERNATE_PML4T, %rax
	movq	%rax,	%cr3
	jmp	1f

1:	nop
	ret

	/*
	 * Switch to the private resume-time hibernate stack
	 */
NENTRY(hibernate_switch_stack_machdep)
	movq	(%rsp), %rax
	movq    %rax, HIBERNATE_STACK_PAGE + HIBERNATE_STACK_OFFSET
	movq    $(HIBERNATE_STACK_PAGE + HIBERNATE_STACK_OFFSET), %rax
	movq    %rax, %rsp

	/* On our own stack from here onward */
	ret

NENTRY(hibernate_flush)
	invlpg	HIBERNATE_INFLATE_PAGE
	ret
#endif /* HIBERNATE */

	/*
	 * End of resume code (code copied to ACPI_TRAMPOLINE)
	 */
_C_LABEL(acpi_resume_end):

	/*
	 * Initial copy of this data gets placed in .rodata, kernel makes
	 * RW copy of it in the tramp data page.
	 */
	.section .rodata
_C_LABEL(acpi_tramp_data_start):
_ACPI_TRMP_DATA_OFFSET(tmp_gdt)
	.word	tmp_gdt_end - tmp_gdtable
	.long	tmp_gdtable

	.align 8
_ACPI_TRMP_DATA_LABEL(tmp_gdtable)
	/*
	 * null
	 */
	.word	0, 0
	.byte	0, 0, 0, 0
	/*
	 * Code
	 * Limit: 0xffffffff
	 * Base: 0x00000000
	 * Descriptor Type: Code
	 * Segment Type: CRA
	 * Present: True
	 * Priv: 0
	 * AVL: False
	 * 64-bit: False
	 * 32-bit: True
	 *
	 */
	.word	0xffff, 0
	.byte	0, 0x9f, 0xcf, 0

	/*
	 * Data
	 * Limit: 0xffffffff
	 * Base: 0x00000000
	 * Descriptor Type:
	 * Segment Type: W
	 * Present: True
	 * Priv: 0
	 * AVL: False
	 * 64-bit: False
	 * 32-bit: True
	 *
	 */
	.word	0xffff, 0
	.byte	0, 0x93, 0xcf, 0
_ACPI_TRMP_DATA_LABEL(tmp_gdt_end)

	.align 8
_ACPI_TRMP_DATA_OFFSET(clean_idt)
	.word	0xffff
	.long	0
	.word	0

	.align 8
_ACPI_TRMP_DATA_LABEL(tmp_gdt64)
	.word 	tmp_gdt64_end - tmp_gdtable64
	.long	tmp_gdtable64

	.align 8
_ACPI_TRMP_DATA_LABEL(tmp_gdtable64)
	.quad	0x0000000000000000
	.quad	0x00af9a000000ffff
	.quad	0x00cf92000000ffff
_ACPI_TRMP_DATA_LABEL(tmp_gdt64_end)

	.align 8
_ACPI_TRMP_DATA_LABEL(tmp_gdt6416)
	.word 	tmp_gdt6416_end - tmp_gdtable6416
	.quad	tmp_gdtable6416

	.align 8
_ACPI_TRMP_DATA_LABEL(tmp_gdtable6416)
	.quad	0x0000000000000000
	.quad	0x00af9a000000ffff
	.quad	0x00cf92000000ffff
	.word	0x0fff, (ACPI_TRAMPOLINE % 0x10000)
	.byte	(ACPI_TRAMPOLINE >> 16), 0x9a, 0, 0
_ACPI_TRMP_DATA_LABEL(tmp_gdt6416_end)

	.align 8
_ACPI_TRMP_DATA_LABEL(acpi_saved_rbx)
	.quad 0
_ACPI_TRMP_DATA_LABEL(acpi_saved_rcx)
	.quad 0
_ACPI_TRMP_DATA_LABEL(acpi_saved_rdx)
	.quad 0
_ACPI_TRMP_DATA_LABEL(acpi_saved_rbp)
	.quad 0
_ACPI_TRMP_DATA_LABEL(acpi_saved_rsi)
	.quad 0
_ACPI_TRMP_DATA_LABEL(acpi_saved_rdi)
	.quad 0
_ACPI_TRMP_DATA_LABEL(acpi_saved_rsp)
	.quad 0
_ACPI_TRMP_DATA_LABEL(acpi_saved_r8)
	.quad 0
_ACPI_TRMP_DATA_LABEL(acpi_saved_r9)
	.quad 0
_ACPI_TRMP_DATA_LABEL(acpi_saved_r10)
	.quad 0
_ACPI_TRMP_DATA_LABEL(acpi_saved_r11)
	.quad 0
_ACPI_TRMP_DATA_LABEL(acpi_saved_r12)
	.quad 0
_ACPI_TRMP_DATA_LABEL(acpi_saved_r13)
	.quad 0
_ACPI_TRMP_DATA_LABEL(acpi_saved_r14)
	.quad 0
_ACPI_TRMP_DATA_LABEL(acpi_saved_r15)
	.quad 0
_ACPI_TRMP_DATA_LABEL(acpi_saved_fl)
	.quad 0
_ACPI_TRMP_DATA_LABEL(acpi_saved_cr0)
	.quad 0
_ACPI_TRMP_DATA_LABEL(acpi_saved_cr2)
	.quad 0
_ACPI_TRMP_DATA_LABEL(acpi_saved_cr3)
	.quad 0
_ACPI_TRMP_DATA_LABEL(acpi_saved_cr4)
	.quad 0
_ACPI_TRMP_DATA_LABEL(acpi_saved_cr8)
	.quad 0
_ACPI_TRMP_DATA_LABEL(acpi_saved_ret)
	.quad 0

	.align 8
_ACPI_TRMP_DATA_LABEL(acpi_saved_idt)
	.space 10

	.align 8
_ACPI_TRMP_DATA_LABEL(acpi_saved_gdt)
	.space 10

	.align 8
_ACPI_TRMP_DATA_LABEL(acpi_saved_ldt)
	.space 10

_ACPI_TRMP_DATA_LABEL(acpi_saved_tr)
	.short 0

	.align 4
_ACPI_TRMP_DATA_LABEL(acpi_saved_efer)
	.long 0

	.align 8
_ACPI_TRMP_DATA_LABEL(acpi_saved_fsbase)
	.quad 0
_ACPI_TRMP_DATA_LABEL(acpi_saved_gsbase)
	.quad 0
_ACPI_TRMP_DATA_LABEL(acpi_saved_kgs)
	.quad 0
_ACPI_TRMP_DATA_LABEL(acpi_saved_star)
	.quad 0
_ACPI_TRMP_DATA_LABEL(acpi_saved_lstar)
	.quad 0
_ACPI_TRMP_DATA_LABEL(acpi_saved_cstar)
	.quad 0
_ACPI_TRMP_DATA_LABEL(acpi_saved_sfmask)
	.quad 0

	.align 4
_ACPI_TRMP_DATA_LABEL(acpi_pdirpa)
	.long 0
#ifdef HIBERNATE
_ACPI_TRMP_DATA_LABEL(hibernate_indirect_16)
	.long	hibernate_resume_vector_3
	.word	0x18
_ACPI_TRMP_DATA_LABEL(hibernate_indirect_16b)
	.long	hibernate_resume_vector_3b
	.word	0x18
#endif /* HIBERNATE */

_C_LABEL(acpi_tramp_data_end):

	/*
	 * acpi_savecpu saves the processor's registers and flags
	 * for use during the ACPI suspend/resume process.
	 */

	.code64
NENTRY(acpi_savecpu)
	movq	(%rsp), %rax
	movq	%rax, acpi_saved_ret

	movq	%rbx, acpi_saved_rbx
	movq	%rcx, acpi_saved_rcx
	movq	%rdx, acpi_saved_rdx
	movq	%rbp, acpi_saved_rbp
	movq	%rsi, acpi_saved_rsi
	movq	%rdi, acpi_saved_rdi
	movq	%rsp, acpi_saved_rsp
	addq	$0x8, acpi_saved_rsp

	movq	%r8, acpi_saved_r8
	movq	%r9, acpi_saved_r9
	movq	%r10, acpi_saved_r10
	movq	%r11, acpi_saved_r11
	movq	%r12, acpi_saved_r12
	movq	%r13, acpi_saved_r13
	movq	%r14, acpi_saved_r14
	movq	%r15, acpi_saved_r15

	pushfq
	popq	acpi_saved_fl

	movq	%cr0, %rax
	movq	%rax, acpi_saved_cr0
	movq	%cr2, %rax
	movq	%rax, acpi_saved_cr2
	movq	%cr3, %rax
	movq	%rax, acpi_saved_cr3
	movq	%cr4, %rax
	movq	%rax, acpi_saved_cr4
	movq	%cr8, %rax
	movq	%rax, acpi_saved_cr8

	pushq	%rcx
	pushq	%rdx
	movl	$MSR_STAR, %ecx
	rdmsr
	movl	%eax, acpi_saved_star
	movl	%edx, acpi_saved_star+4

	movl	$MSR_CSTAR, %ecx
	rdmsr
	movl	%eax, acpi_saved_cstar
	movl	%edx, acpi_saved_cstar+4

	movl	$MSR_LSTAR, %ecx
	rdmsr
	movl	%eax, acpi_saved_lstar
	movl	%edx, acpi_saved_lstar+4

	movl	$MSR_SFMASK, %ecx
	rdmsr
	movl	%eax, acpi_saved_sfmask
	movl	%edx, acpi_saved_sfmask+4

	movl	$MSR_FSBASE, %ecx
	rdmsr
	movl	%eax, acpi_saved_fsbase
	movl	%edx, acpi_saved_fsbase+4

	movl	$MSR_GSBASE, %ecx
	rdmsr
	movl	%eax, acpi_saved_gsbase
	movl	%edx, acpi_saved_gsbase+4

	movl	$MSR_KERNELGSBASE, %ecx
	rdmsr
	movl	%eax, acpi_saved_kgs
	movl	%edx, acpi_saved_kgs+4

	movl	$MSR_EFER, %ecx
	rdmsr
	movl	%eax, acpi_saved_efer
	popq 	%rdx
	popq	%rcx

	sgdt	acpi_saved_gdt
	sidt	acpi_saved_idt
	sldt	acpi_saved_ldt
	str	acpi_saved_tr

	movl	$1, %eax
	ret


code:
/*	$OpenBSD: aes_intel.S,v 1.9 2013/03/26 15:47:01 jsing Exp $	*/

/*
 * Implement AES algorithm in Intel AES-NI instructions.
 *
 * The white paper of AES-NI instructions can be downloaded from:
 *   http://softwarecommunity.intel.com/isn/downloads/intelavx/AES-Instructions-Set_WP.pdf
 *
 * Copyright (C) 2008-2010, Intel Corporation
 *    Author: Huang Ying <ying.huang@intel.com>
 *            Vinodh Gopal <vinodh.gopal@intel.com>
 *            Kahraman Akdemir
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following
 * conditions are met:
 *
 * - Redistributions of source code must retain the above copyright
 *   notice, this list of conditions and the following disclaimer.
 *
 * - Redistributions in binary form must reproduce the above copyright
 *   notice, this list of conditions and the following disclaimer in the
 *   documentation and/or other materials provided with the
 *   distribution.
 *
 * - Neither the name of Intel Corporation nor the names of its
 *   contributors may be used to endorse or promote products
 *   derived from this software without specific prior written
 *   permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

/*
 * Changes to the original source code released by Intel:
 *
 * - assembler macros were converted to the actual instructions;
 * - aesni_ctr_enc was changed to be RFC 3686 compliant;
 * - aes-gcm mode added;
 * - aes-xts implementation added;
 *
 * Copyright (c) 2010,2011 Mike Belopuhov
 * Copyright (c) 2013 Joel Sing <jsing@openbsd.org>
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

#include <machine/param.h>
#include <machine/asm.h>

#define STATE1		%xmm0
#define STATE2		%xmm4
#define STATE3		%xmm5
#define STATE4		%xmm6
#define STATE		STATE1
#define IN1		%xmm1
#define IN2		%xmm7
#define IN3		%xmm8
#define IN4		%xmm9
#define IN		IN1
#define KEY		%xmm2
#define IV		%xmm3
#define BSWAP_MASK	%xmm10
#define CTR		%xmm11
#define INC		%xmm12

#define KEYP		%rdi
#define OUTP		%rsi
#define INP		%rdx
#define LEN		%rcx
#define HSTATE		%rcx
#define IVP		%r8
#define ICBP		%r8
#define KLEN		%r9d
#define T1		%r10
#define TKEYP		T1
#define T2		%r11
#define TCTR_LOW	T2

	.data
.align 16
.Lbswap_mask:
	.byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0

	.text

_key_expansion_128:
_key_expansion_256a:
	pshufd	$0b11111111,%xmm1,%xmm1
	shufps	$0b00010000,%xmm0,%xmm4
	pxor	%xmm4,%xmm0
	shufps	$0b10001100,%xmm0,%xmm4
	pxor	%xmm4,%xmm0
	pxor	%xmm1,%xmm0
	movaps	%xmm0,(%rcx)
	add	$0x10,%rcx
	ret

_key_expansion_192a:
	pshufd	$0b01010101,%xmm1,%xmm1
	shufps	$0b00010000,%xmm0,%xmm4
	pxor	%xmm4,%xmm0
	shufps	$0b10001100,%xmm0,%xmm4
	pxor	%xmm4,%xmm0
	pxor	%xmm1,%xmm0

	movaps	%xmm2,%xmm5
	movaps	%xmm2,%xmm6
	pslldq	$4,%xmm5
	pshufd	$0b11111111,%xmm0,%xmm3
	pxor	%xmm3,%xmm2
	pxor	%xmm5,%xmm2

	movaps	%xmm0,%xmm1
	shufps	$0b01000100,%xmm0,%xmm6
	movaps	%xmm6,(%rcx)
	shufps	$0b01001110,%xmm2,%xmm1
	movaps	%xmm1,16(%rcx)
	add	$0x20,%rcx
	ret

_key_expansion_192b:
	pshufd	$0b01010101,%xmm1,%xmm1
	shufps	$0b00010000,%xmm0,%xmm4
	pxor	%xmm4,%xmm0
	shufps	$0b10001100,%xmm0,%xmm4
	pxor	%xmm4,%xmm0
	pxor	%xmm1,%xmm0

	movaps	%xmm2,%xmm5
	pslldq	$4,%xmm5
	pshufd	$0b11111111,%xmm0,%xmm3
	pxor	%xmm3,%xmm2
	pxor	%xmm5,%xmm2

	movaps	%xmm0,(%rcx)
	add	$0x10,%rcx
	ret

_key_expansion_256b:
	pshufd	$0b10101010,%xmm1,%xmm1
	shufps	$0b00010000,%xmm2,%xmm4
	pxor	%xmm4,%xmm2
	shufps	$0b10001100,%xmm2,%xmm4
	pxor	%xmm4,%xmm2
	pxor	%xmm1,%xmm2
	movaps	%xmm2,(%rcx)
	add	$0x10,%rcx
	ret

/*
 * void aesni_set_key(struct aesni_session *ses, uint8_t *key, size_t len)
 */
ENTRY(aesni_set_key)
	movups	(%rsi),%xmm0		# user key (first 16 bytes)
	movaps	%xmm0,(%rdi)
	lea	0x10(%rdi),%rcx		# key addr
	movl	%edx,480(%rdi)
	pxor	%xmm4,%xmm4		# xmm4 is assumed 0 in _key_expansion_x
	cmp	$24,%dl
	jb	2f
	je	1f
	movups	0x10(%rsi),%xmm2	# other user key
	movaps	%xmm2,(%rcx)
	add	$0x10,%rcx
	aeskeygenassist $0x1,%xmm2,%xmm1	# round 1
	call	_key_expansion_256a
	aeskeygenassist $0x1,%xmm0,%xmm1
	call	_key_expansion_256b
	aeskeygenassist $0x2,%xmm2,%xmm1	# round 2
	call	_key_expansion_256a
	aeskeygenassist $0x2,%xmm0,%xmm1
	call	_key_expansion_256b
	aeskeygenassist $0x4,%xmm2,%xmm1	# round 3
	call	_key_expansion_256a
	aeskeygenassist $0x4,%xmm0,%xmm1
	call	_key_expansion_256b
	aeskeygenassist $0x8,%xmm2,%xmm1	# round 4
	call	_key_expansion_256a
	aeskeygenassist $0x8,%xmm0,%xmm1
	call	_key_expansion_256b
	aeskeygenassist $0x10,%xmm2,%xmm1	# round 5
	call	_key_expansion_256a
	aeskeygenassist $0x10,%xmm0,%xmm1
	call	_key_expansion_256b
	aeskeygenassist $0x20,%xmm2,%xmm1	# round 6
	call	_key_expansion_256a
	aeskeygenassist $0x20,%xmm0,%xmm1
	call	_key_expansion_256b
	aeskeygenassist $0x40,%xmm2,%xmm1	# round 7
	call	_key_expansion_256a
	jmp	3f
1:	/* 192 bit key */
	movq	0x10(%rsi),%xmm2	# other user key
	aeskeygenassist $0x1,%xmm2,%xmm1	# round 1
	call	_key_expansion_192a
	aeskeygenassist $0x2,%xmm2,%xmm1	# round 2
	call	_key_expansion_192b
	aeskeygenassist $0x4,%xmm2,%xmm1	# round 3
	call	_key_expansion_192a
	aeskeygenassist $0x8,%xmm2,%xmm1	# round 4
	call	_key_expansion_192b
	aeskeygenassist $0x10,%xmm2,%xmm1	# round 5
	call	_key_expansion_192a
	aeskeygenassist $0x20,%xmm2,%xmm1	# round 6
	call	_key_expansion_192b
	aeskeygenassist $0x40,%xmm2,%xmm1	# round 7
	call	_key_expansion_192a
	aeskeygenassist $0x80,%xmm2,%xmm1	# round 8
	call	_key_expansion_192b
	jmp	3f
2:	/* 128 bit key */
	aeskeygenassist $0x1,%xmm0,%xmm1	# round 1
	call	_key_expansion_128
	aeskeygenassist $0x2,%xmm0,%xmm1	# round 2
	call	_key_expansion_128
	aeskeygenassist $0x4,%xmm0,%xmm1	# round 3
	call	_key_expansion_128
	aeskeygenassist $0x8,%xmm0,%xmm1	# round 4
	call	_key_expansion_128
	aeskeygenassist $0x10,%xmm0,%xmm1	# round 5
	call	_key_expansion_128
	aeskeygenassist $0x20,%xmm0,%xmm1	# round 6
	call	_key_expansion_128
	aeskeygenassist $0x40,%xmm0,%xmm1	# round 7
	call	_key_expansion_128
	aeskeygenassist $0x80,%xmm0,%xmm1	# round 8
	call	_key_expansion_128
	aeskeygenassist $0x1b,%xmm0,%xmm1	# round 9
	call	_key_expansion_128
	aeskeygenassist $0x36,%xmm0,%xmm1	# round 10
	call	_key_expansion_128
3:
	sub	$0x10,%rcx
	movaps	(%rdi),%xmm0
	movaps	(%rcx),%xmm1
	movaps	%xmm0,240(%rcx)
	movaps	%xmm1,240(%rdi)
	add	$0x10,%rdi
	lea	240-16(%rcx),%rsi
.align 4
4:
	movaps	(%rdi),%xmm0
	aesimc	%xmm0,%xmm1
	movaps	%xmm1,(%rsi)
	add	$0x10,%rdi
	sub	$0x10,%rsi
	cmp	%rcx,%rdi
	jb	4b
	ret

/*
 * void aesni_enc(struct aesni_session *ses, uint8_t *dst, uint8_t *src)
 */
ENTRY(aesni_enc)
	movl	480(KEYP),KLEN		# key length
	movups	(INP),STATE		# input
	call	_aesni_enc1
	movups	STATE,(OUTP)		# output
	ret

/*
 * _aesni_enc1:		internal ABI
 * input:
 *	KEYP:		key struct pointer
 *	KLEN:		round count
 *	STATE:		initial state (input)
 * output:
 *	STATE:		final state (output)
 * changed:
 *	KEY
 *	TKEYP (T1)
 */
_aesni_enc1:
	movaps	(KEYP),KEY		# key
	mov	KEYP,TKEYP
	pxor	KEY,STATE		# round 0
	add	$0x30,TKEYP
	cmp	$24,KLEN
	jb	2f
	lea	0x20(TKEYP),TKEYP
	je	1f
	add	$0x20,TKEYP
	movaps	-0x60(TKEYP),KEY
	aesenc	KEY,STATE
	movaps	-0x50(TKEYP),KEY
	aesenc	KEY,STATE
.align 4
1:	/* 192 bit key */
	movaps	-0x40(TKEYP),KEY
	aesenc	KEY,STATE
	movaps	-0x30(TKEYP),KEY
	aesenc	KEY,STATE
.align 4
2:	/* 128 bit key */
	movaps	-0x20(TKEYP),KEY
	aesenc	KEY,STATE
	movaps	-0x10(TKEYP),KEY
	aesenc	KEY,STATE
	movaps	(TKEYP),KEY
	aesenc	KEY,STATE
	movaps	0x10(TKEYP),KEY
	aesenc	KEY,STATE
	movaps	0x20(TKEYP),KEY
	aesenc	KEY,STATE
	movaps	0x30(TKEYP),KEY
	aesenc	KEY,STATE
	movaps	0x40(TKEYP),KEY
	aesenc	KEY,STATE
	movaps	0x50(TKEYP),KEY
	aesenc	KEY,STATE
	movaps	0x60(TKEYP),KEY
	aesenc	KEY,STATE
	movaps	0x70(TKEYP),KEY
	aesenclast KEY,STATE
	ret

/*
 * _aesni_enc4:	internal ABI
 * input:
 *	KEYP:		key struct pointer
 *	KLEN:		round count
 *	STATE1:		initial state (input)
 *	STATE2
 *	STATE3
 *	STATE4
 * output:
 *	STATE1:		final state (output)
 *	STATE2
 *	STATE3
 *	STATE4
 * changed:
 *	KEY
 *	TKEYP (T1)
 */
_aesni_enc4:
	movaps	(KEYP),KEY		# key
	mov	KEYP,TKEYP
	pxor	KEY,STATE1		# round 0
	pxor	KEY,STATE2
	pxor	KEY,STATE3
	pxor	KEY,STATE4
	add	$0x30,TKEYP
	cmp	$24,KLEN
	jb	2f
	lea	0x20(TKEYP),TKEYP
	je	1f
	add	$0x20,TKEYP
	movaps	-0x60(TKEYP),KEY
	aesenc	KEY,STATE1
	aesenc	KEY,STATE2
	aesenc	KEY,STATE3
	aesenc	KEY,STATE4
	movaps	-0x50(TKEYP),KEY
	aesenc	KEY,STATE1
	aesenc	KEY,STATE2
	aesenc	KEY,STATE3
	aesenc	KEY,STATE4
#.align 4
1:	/* 192 bit key */
	movaps	-0x40(TKEYP),KEY
	aesenc	KEY,STATE1
	aesenc	KEY,STATE2
	aesenc	KEY,STATE3
	aesenc	KEY,STATE4
	movaps	-0x30(TKEYP),KEY
	aesenc	KEY,STATE1
	aesenc	KEY,STATE2
	aesenc	KEY,STATE3
	aesenc	KEY,STATE4
#.align 4
2:	/* 128 bit key */
	movaps	-0x20(TKEYP),KEY
	aesenc	KEY,STATE1
	aesenc	KEY,STATE2
	aesenc	KEY,STATE3
	aesenc	KEY,STATE4
	movaps	-0x10(TKEYP),KEY
	aesenc	KEY,STATE1
	aesenc	KEY,STATE2
	aesenc	KEY,STATE3
	aesenc	KEY,STATE4
	movaps	(TKEYP),KEY
	aesenc	KEY,STATE1
	aesenc	KEY,STATE2
	aesenc	KEY,STATE3
	aesenc	KEY,STATE4
	movaps	0x10(TKEYP),KEY
	aesenc	KEY,STATE1
	aesenc	KEY,STATE2
	aesenc	KEY,STATE3
	aesenc	KEY,STATE4
	movaps	0x20(TKEYP),KEY
	aesenc	KEY,STATE1
	aesenc	KEY,STATE2
	aesenc	KEY,STATE3
	aesenc	KEY,STATE4
	movaps	0x30(TKEYP),KEY
	aesenc	KEY,STATE1
	aesenc	KEY,STATE2
	aesenc	KEY,STATE3
	aesenc	KEY,STATE4
	movaps	0x40(TKEYP),KEY
	aesenc	KEY,STATE1
	aesenc	KEY,STATE2
	aesenc	KEY,STATE3
	aesenc	KEY,STATE4
	movaps	0x50(TKEYP),KEY
	aesenc	KEY,STATE1
	aesenc	KEY,STATE2
	aesenc	KEY,STATE3
	aesenc	KEY,STATE4
	movaps	0x60(TKEYP),KEY
	aesenc	KEY,STATE1
	aesenc	KEY,STATE2
	aesenc	KEY,STATE3
	aesenc	KEY,STATE4
	movaps	0x70(TKEYP),KEY
	aesenclast KEY,STATE1		# last round
	aesenclast KEY,STATE2
	aesenclast KEY,STATE3
	aesenclast KEY,STATE4
	ret

/*
 * void aesni_dec(struct aesni_session *ses, uint8_t *dst, uint8_t *src)
 */
ENTRY(aesni_dec)
	mov	480(KEYP),KLEN		# key length
	add	$240,KEYP
	movups	(INP),STATE		# input
	call	_aesni_dec1
	movups	STATE,(OUTP)		# output
	ret

/*
 * _aesni_dec1:		internal ABI
 * input:
 *	KEYP:		key struct pointer
 *	KLEN:		key length
 *	STATE:		initial state (input)
 * output:
 *	STATE:		final state (output)
 * changed:
 *	KEY
 *	TKEYP (T1)
 */
_aesni_dec1:
	movaps	(KEYP),KEY		# key
	mov	KEYP,TKEYP
	pxor	KEY,STATE		# round 0
	add	$0x30,TKEYP
	cmp	$24,KLEN
	jb	2f
	lea	0x20(TKEYP),TKEYP
	je	1f
	add	$0x20,TKEYP
	movaps	-0x60(TKEYP),KEY
	aesdec	KEY,STATE
	movaps	-0x50(TKEYP),KEY
	aesdec	KEY,STATE
.align 4
1:	/* 192 bit key */
	movaps	-0x40(TKEYP),KEY
	aesdec	KEY,STATE
	movaps	-0x30(TKEYP),KEY
	aesdec	KEY,STATE
.align 4
2:	/* 128 bit key */
	movaps	-0x20(TKEYP),KEY
	aesdec	KEY,STATE
	movaps	-0x10(TKEYP),KEY
	aesdec	KEY,STATE
	movaps	(TKEYP),KEY
	aesdec	KEY,STATE
	movaps	0x10(TKEYP),KEY
	aesdec	KEY,STATE
	movaps	0x20(TKEYP),KEY
	aesdec	KEY,STATE
	movaps	0x30(TKEYP),KEY
	aesdec	KEY,STATE
	movaps	0x40(TKEYP),KEY
	aesdec	KEY,STATE
	movaps	0x50(TKEYP),KEY
	aesdec	KEY,STATE
	movaps	0x60(TKEYP),KEY
	aesdec	KEY,STATE
	movaps	0x70(TKEYP),KEY
	aesdeclast KEY,STATE
	ret

/*
 * _aesni_dec4:	internal ABI
 * input:
 *	KEYP:		key struct pointer
 *	KLEN:		key length
 *	STATE1:		initial state (input)
 *	STATE2
 *	STATE3
 *	STATE4
 * output:
 *	STATE1:		final state (output)
 *	STATE2
 *	STATE3
 *	STATE4
 * changed:
 *	KEY
 *	TKEYP (T1)
 */
_aesni_dec4:
	movaps	(KEYP),KEY		# key
	mov	KEYP,TKEYP
	pxor	KEY,STATE1		# round 0
	pxor	KEY,STATE2
	pxor	KEY,STATE3
	pxor	KEY,STATE4
	add	$0x30,TKEYP
	cmp	$24,KLEN
	jb	2f
	lea	0x20(TKEYP),TKEYP
	je	1f
	add 	$0x20,TKEYP
	movaps	-0x60(TKEYP),KEY
	aesdec	KEY,STATE1
	aesdec	KEY,STATE2
	aesdec	KEY,STATE3
	aesdec	KEY,STATE4
	movaps	-0x50(TKEYP),KEY
	aesdec	KEY,STATE1
	aesdec	KEY,STATE2
	aesdec	KEY,STATE3
	aesdec	KEY,STATE4
.align 4
1:	/* 192 bit key */
	movaps	-0x40(TKEYP),KEY
	aesdec	KEY,STATE1
	aesdec	KEY,STATE2
	aesdec	KEY,STATE3
	aesdec	KEY,STATE4
	movaps	-0x30(TKEYP),KEY
	aesdec	KEY,STATE1
	aesdec	KEY,STATE2
	aesdec	KEY,STATE3
	aesdec	KEY,STATE4
.align 4
2:	/* 128 bit key */
	movaps	-0x20(TKEYP),KEY
	aesdec	KEY,STATE1
	aesdec	KEY,STATE2
	aesdec	KEY,STATE3
	aesdec	KEY,STATE4
	movaps	-0x10(TKEYP),KEY
	aesdec	KEY,STATE1
	aesdec	KEY,STATE2
	aesdec	KEY,STATE3
	aesdec	KEY,STATE4
	movaps	(TKEYP),KEY
	aesdec	KEY,STATE1
	aesdec	KEY,STATE2
	aesdec	KEY,STATE3
	aesdec	KEY,STATE4
	movaps	0x10(TKEYP),KEY
	aesdec	KEY,STATE1
	aesdec	KEY,STATE2
	aesdec	KEY,STATE3
	aesdec	KEY,STATE4
	movaps	0x20(TKEYP),KEY
	aesdec	KEY,STATE1
	aesdec	KEY,STATE2
	aesdec	KEY,STATE3
	aesdec	KEY,STATE4
	movaps	0x30(TKEYP),KEY
	aesdec	KEY,STATE1
	aesdec	KEY,STATE2
	aesdec	KEY,STATE3
	aesdec	KEY,STATE4
	movaps	0x40(TKEYP),KEY
	aesdec	KEY,STATE1
	aesdec	KEY,STATE2
	aesdec	KEY,STATE3
	aesdec	KEY,STATE4
	movaps	0x50(TKEYP),KEY
	aesdec	KEY,STATE1
	aesdec	KEY,STATE2
	aesdec	KEY,STATE3
	aesdec	KEY,STATE4
	movaps	0x60(TKEYP),KEY
	aesdec	KEY,STATE1
	aesdec	KEY,STATE2
	aesdec	KEY,STATE3
	aesdec	KEY,STATE4
	movaps	0x70(TKEYP),KEY
	aesdeclast KEY,STATE1		# last round
	aesdeclast KEY,STATE2
	aesdeclast KEY,STATE3
	aesdeclast KEY,STATE4
	ret

#if 0
/*
 * void aesni_ecb_enc(struct aesni_session *ses, uint8_t *dst, uint8_t *src,
 *     size_t len)
 */
ENTRY(aesni_ecb_enc)
	test	LEN,LEN			# check length
	jz	3f
	mov	480(KEYP),KLEN
	cmp	$16,LEN
	jb	3f
	cmp	$64,LEN
	jb	2f
.align 4
1:
	movups	(INP),STATE1
	movups	0x10(INP),STATE2
	movups	0x20(INP),STATE3
	movups	0x30(INP),STATE4
	call	_aesni_enc4
	movups	STATE1,(OUTP)
	movups	STATE2,0x10(OUTP)
	movups	STATE3,0x20(OUTP)
	movups	STATE4,0x30(OUTP)
	sub	$64,LEN
	add	$64,INP
	add	$64,OUTP
	cmp	$64,LEN
	jge	1b
	cmp	$16,LEN
	jb	3f
.align 4
2:
	movups	(INP),STATE1
	call	_aesni_enc1
	movups	STATE1,(OUTP)
	sub	$16,LEN
	add	$16,INP
	add	$16,OUTP
	cmp	$16,LEN
	jge	2b
3:
	ret

/*
 * void aesni_ecb_dec(struct aesni_session *ses, uint8_t *dst, uint8_t *src,
 *     size_t len);
 */
ENTRY(aesni_ecb_dec)
	test	LEN,LEN
	jz	3f
	mov	480(KEYP),KLEN
	add	$240,KEYP
	cmp	$16,LEN
	jb	3f
	cmp	$64,LEN
	jb	2f
.align 4
1:
	movups	(INP),STATE1
	movups	0x10(INP),STATE2
	movups	0x20(INP),STATE3
	movups	0x30(INP),STATE4
	call	_aesni_dec4
	movups	STATE1,(OUTP)
	movups	STATE2,0x10(OUTP)
	movups	STATE3,0x20(OUTP)
	movups	STATE4,0x30(OUTP)
	sub	$64,LEN
	add	$64,INP
	add	$64,OUTP
	cmp	$64,LEN
	jge	1b
	cmp	$16,LEN
	jb	3f
.align 4
2:
	movups	(INP),STATE1
	call	_aesni_dec1
	movups	STATE1,(OUTP)
	sub	$16,LEN
	add	$16,INP
	add	$16,OUTP
	cmp	$16,LEN
	jge	2b
3:
	ret
#endif

/*
 * void aesni_cbc_enc(struct aesni_session *ses, uint8_t *dst, uint8_t *src,
 *     size_t len, uint8_t *iv)
 */
ENTRY(aesni_cbc_enc)
	cmp	$16,LEN
	jb	2f
	mov	480(KEYP),KLEN
	movups	(IVP),STATE	# load iv as initial state
.align 4
1:
	movups	(INP),IN	# load input
	pxor	IN,STATE
	call	_aesni_enc1
	movups	STATE,(OUTP)	# store output
	sub	$16,LEN
	add	$16,INP
	add	$16,OUTP
	cmp	$16,LEN
	jge	1b
	movups	STATE,(IVP)
2:
	ret

/*
 * void aesni_cbc_dec(struct aesni_session *ses, uint8_t *dst, uint8_t *src,
 *     size_t len, uint8_t *iv)
 */
ENTRY(aesni_cbc_dec)
	cmp	$16,LEN
	jb	4f
	mov	480(KEYP),KLEN
	add	$240,KEYP
	movups	(IVP),IV
	cmp	$64,LEN
	jb	2f
.align 4
1:	/* pipeline 4 instructions when possible */
	movups	(INP),IN1
	movaps	IN1,STATE1
	movups	0x10(INP),IN2
	movaps	IN2,STATE2
	movups	0x20(INP),IN3
	movaps	IN3,STATE3
	movups	0x30(INP),IN4
	movaps	IN4,STATE4
	call	_aesni_dec4
	pxor	IV,STATE1
	pxor	IN1,STATE2
	pxor	IN2,STATE3
	pxor	IN3,STATE4
	movaps	IN4,IV
	movups	STATE1,(OUTP)
	movups	STATE2,0x10(OUTP)
	movups	STATE3,0x20(OUTP)
	movups	STATE4,0x30(OUTP)
	sub	$64,LEN
	add	$64,INP
	add	$64,OUTP
	cmp	$64,LEN
	jge	1b
	cmp	$16,LEN
	jb	3f
.align 4
2:
	movups	(INP),IN
	movaps	IN,STATE
	call	_aesni_dec1
	pxor	IV,STATE
	movups	STATE,(OUTP)
	movaps	IN,IV
	sub	$16,LEN
	add	$16,INP
	add	$16,OUTP
	cmp	$16,LEN
	jge	2b
3:
	movups	IV,(IVP)
4:
	ret

/*
 * _aesni_inc_init:	internal ABI
 *	setup registers used by _aesni_inc
 * input:
 *	ICB
 * output:
 *	CTR:		== CTR, in little endian
 *	IV:		== IV, in big endian
 *	TCTR_LOW:	== lower dword of CTR
 *	INC:		== 1, in little endian
 *	BSWAP_MASK	== endian swapping mask
 */
_aesni_inc_init:
	movdqa	CTR,IV
	pslldq	$8,IV
	movdqu	.Lbswap_mask,BSWAP_MASK
	pshufb	BSWAP_MASK,CTR
	mov	$1,TCTR_LOW
	movd	TCTR_LOW,INC
	movd	CTR,TCTR_LOW
	ret

/*
 * _aesni_inc:		internal ABI
 *	Increase IV by 1, IV is in big endian
 * input:
 *	IV
 *	CTR:		== IV, in little endian
 *	TCTR_LOW:	== lower dword of CTR
 *	INC:		== 1, in little endian
 *	BSWAP_MASK	== endian swapping mask
 * output:
 *	IV:		Increase by 1
 * changed:
 *	CTR:		== output IV, in little endian
 *	TCTR_LOW:	== lower dword of CTR
 */
_aesni_inc:
	paddq	INC,CTR
	add	$1,TCTR_LOW
	jnc	1f
	pslldq	$8,INC
	paddq	INC,CTR
	psrldq	$8,INC
1:
	movaps	CTR,IV
	pshufb	BSWAP_MASK,IV
	ret

/*
 * void aesni_ctr_enc(struct aesni_session *ses, uint8_t *dst, uint8_t *src,
 *     size_t len, uint8_t *icb)
 */
ENTRY(aesni_ctr_enc)
	cmp	$16,LEN
	jb	4f
	mov	480(KEYP),KLEN
	movdqu	(ICBP),CTR
	call	_aesni_inc_init
	cmp	$64,LEN
	jb	2f
.align 4
1:	/* pipeline 4 instructions when possible */
	call	_aesni_inc
	movaps	IV,STATE1
	movups	(INP),IN1
	call	_aesni_inc
	movaps	IV,STATE2
	movups	0x10(INP),IN2
	call	_aesni_inc
	movaps	IV,STATE3
	movups	0x20(INP),IN3
	call	_aesni_inc
	movaps	IV,STATE4
	movups	0x30(INP),IN4
	call	_aesni_enc4
	pxor	IN1,STATE1
	movups	STATE1,(OUTP)
	pxor	IN2,STATE2
	movups	STATE2,0x10(OUTP)
	pxor	IN3,STATE3
	movups	STATE3,0x20(OUTP)
	pxor	IN4,STATE4
	movups	STATE4,0x30(OUTP)
	sub	$64,LEN
	add	$64,INP
	add	$64,OUTP
	cmp	$64,LEN
	jge	1b
	cmp	$16,LEN
	jb	3f
.align 4
2:
	call	_aesni_inc
	movaps	IV,STATE
	movups	(INP),IN
	call	_aesni_enc1
	pxor	IN,STATE
	movups	STATE,(OUTP)
	sub	$16,LEN
	add	$16,INP
	add	$16,OUTP
	cmp	$16,LEN
	jge	2b
3:
	movq	IV,(IVP)
4:
	ret

_aesni_gmac_gfmul:
	movdqa	%xmm0,%xmm3
	pclmulqdq $0x00,%xmm1,%xmm3	# xmm3 holds a0*b0
	movdqa	%xmm0,%xmm4
	pclmulqdq $0x10,%xmm1,%xmm4	# xmm4 holds a0*b1
	movdqa	%xmm0,%xmm5
	pclmulqdq $0x01,%xmm1,%xmm5	# xmm5 holds a1*b0
	movdqa	%xmm0,%xmm6
	pclmulqdq $0x11,%xmm1,%xmm6	# xmm6 holds a1*b1

	pxor	%xmm5,%xmm4		# xmm4 holds a0*b1 + a1*b0
	movdqa	%xmm4,%xmm5
	psrldq	$8,%xmm4
	pslldq	$8,%xmm5
	pxor	%xmm5,%xmm3
	pxor	%xmm4,%xmm6

	/*
	 * <xmm6:xmm3> holds the result of the carry-less
	 * multiplication of xmm0 by xmm1
	 *
	 * shift the result by one bit position to the left
	 * cope for the fact that bits are reversed
	 */
	movdqa	%xmm3,%xmm7
	movdqa	%xmm6,%xmm8
	pslld	$1,%xmm3
	pslld	$1,%xmm6
	psrld	$31,%xmm7
	psrld	$31,%xmm8
	movdqa	%xmm7,%xmm9
	pslldq	$4,%xmm8
	pslldq	$4,%xmm7
	psrldq	$12,%xmm9
	por	%xmm7,%xmm3
	por	%xmm8,%xmm6
	por	%xmm9,%xmm6

	/* first phase of the reduction */
	movdqa	%xmm3,%xmm7
	movdqa	%xmm3,%xmm8
	movdqa	%xmm3,%xmm9
	pslld	$31,%xmm7		# packed right shifting << 31
	pslld	$30,%xmm8		# packed right shifting shift << 30
	pslld	$25,%xmm9		# packed right shifting shift << 25
	pxor	%xmm8,%xmm7		# xor the shifted versions
	pxor	%xmm9,%xmm7
	movdqa	%xmm7,%xmm8
	pslldq	$12,%xmm7
	psrldq	$4,%xmm8
	pxor	%xmm7,%xmm3

	/* second phase of the reduction */
	movdqa	%xmm3,%xmm2
	movdqa	%xmm3,%xmm4
	movdqa	%xmm3,%xmm5
	psrld	$1,%xmm2		# packed left shifting >> 1
	psrld	$2,%xmm4		# packed left shifting >> 2
	psrld	$7,%xmm5		# packed left shifting >> 7
	pxor	%xmm4,%xmm2		# xor the shifted versions
	pxor	%xmm5,%xmm2
	pxor	%xmm8,%xmm2
	pxor	%xmm2,%xmm3
	pxor	%xmm3,%xmm6		# the result is in xmm6
	ret

/*
 * void aesni_gmac_update(GHASH_CTX *ghash, uint8_t *src, size_t len)
 */
ENTRY(aesni_gmac_update)
	cmp	$16,%rdx
	jb	2f

	movdqu	.Lbswap_mask,BSWAP_MASK	# endianness swap mask

	movdqu	(%rdi),%xmm1		# hash subkey
	movdqu	32(%rdi),%xmm6		# initial state
	pshufb	BSWAP_MASK,%xmm1
	pshufb	BSWAP_MASK,%xmm6
1:
	movdqu	(%rsi),%xmm2
	pshufb	BSWAP_MASK,%xmm2
	movdqa	%xmm6,%xmm0
	pxor	%xmm2,%xmm0
	call	_aesni_gmac_gfmul

	sub	$16,%rdx
	add	$16,%rsi
	cmp	$16,%rdx
	jge	1b

	pshufb	BSWAP_MASK,%xmm6
	movdqu	%xmm6,16(%rdi)
	movdqu	%xmm6,32(%rdi)
2:
	ret

/*
 * void aesni_gmac_final(struct aesni_sess *ses, uint8_t *tag,
 *     uint8_t *icb, uint8_t *hashstate)
 */
ENTRY(aesni_gmac_final)
	movl	480(KEYP),KLEN		# key length
	movdqu	(INP),STATE		# icb
	call	_aesni_enc1
	movdqu	(HSTATE),IN
	pxor	IN,STATE
	movdqu	STATE,(OUTP)		# output
	ret

/*
 * void aesni_xts_enc(struct aesni_xts_ctx *xts, uint8_t *dst, uint8_t *src,
 *    size_t len, uint8_t *iv)
 */
ENTRY(aesni_xts_enc)
	cmp	$16,%rcx
	jb	2f

	call	_aesni_xts_tweak

	movl	480(KEYP),KLEN		# key length
1:
	movups	(%rdx),%xmm0		# src
	pxor	%xmm3,%xmm0		# xor block with tweak
	call	_aesni_enc1
	pxor	%xmm3,%xmm0		# xor block with tweak
	movups	%xmm0,(%rsi)		# dst

	call	_aesni_xts_tweak_exp

	add	$16,%rsi
	add	$16,%rdx
	sub	$16,%rcx
	cmp	$16,%rcx
	jge	1b
2:
	ret

/*
 * void aesni_xts_dec(struct aesni_xts_ctx *xts, uint8_t *dst, uint8_t *src,
 *    size_t len, uint8_t *iv)
 */
ENTRY(aesni_xts_dec)
	cmp	$16,%rcx
	jb	2f

	call	_aesni_xts_tweak

	movl	480(KEYP),KLEN		# key length
	add	$240,KEYP		# decryption key
1:
	movups	(%rdx),%xmm0		# src
	pxor	%xmm3,%xmm0		# xor block with tweak
	call	_aesni_dec1
	pxor	%xmm3,%xmm0		# xor block with tweak
	movups	%xmm0,(%rsi)		# dst

	call	_aesni_xts_tweak_exp

	add	$16,%rsi
	add	$16,%rdx
	sub	$16,%rcx
	cmp	$16,%rcx
	jge	1b
2:
	ret

/*
 * Prepare tweak as E_k2(IV). IV is specified as LE representation of a
 * 64-bit block number which we allow to be passed in directly. Since
 * we're on a 64-bit LE host the representation is already correct.
 *
 * xts is in %rdi, iv is in %r8 and we return the tweak in %xmm3.
 */
_aesni_xts_tweak:
	mov	(%r8),%r10
	movd	%r10,%xmm0		# Last 64-bits of IV are always zero.
	mov	KEYP,%r11
	lea	496(%rdi),KEYP
	movl	480(KEYP),KLEN
	call	_aesni_enc1
	movdqa	%xmm0,%xmm3
	mov	%r11,KEYP
	ret

/*
 * Exponentiate AES XTS tweak (in %xmm3).
 */
_aesni_xts_tweak_exp:
	pextrw	$7,%xmm3,%r10
	pextrw	$3,%xmm3,%r11
	psllq	$1,%xmm3		# Left shift.

	and	$0x8000,%r11		# Carry between quads.
	jz	1f
	mov	$1,%r11
	pxor	%xmm0,%xmm0
	pinsrw	$4,%r11,%xmm0
	por	%xmm0,%xmm3
1:
	and	$0x8000,%r10
	jz	2f
	pextrw	$0,%xmm3,%r11
	xor	$0x87,%r11		# AES XTS alpha - GF(2^128).
	pinsrw	$0,%r11,%xmm3
2:
	ret


code:
/*	$OpenBSD: aesni.c,v 1.32 2015/03/14 03:38:46 jsg Exp $	*/
/*-
 * Copyright (c) 2003 Jason Wright
 * Copyright (c) 2003, 2004 Theo de Raadt
 * Copyright (c) 2010, Thordur I. Bjornsson
 * Copyright (c) 2010, Mike Belopuhov
 * All rights reserved.
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/queue.h>
#include <sys/types.h>
#include <sys/malloc.h>
#include <sys/pool.h>
#include <sys/mbuf.h>

#include <crypto/cryptodev.h>
#include <crypto/rijndael.h>
#include <crypto/gmac.h>
#include <crypto/xform.h>
#include <crypto/cryptosoft.h>

#include <machine/fpu.h>

/* defines from crypto/xform.c */
#define AESCTR_NONCESIZE	4
#define AESCTR_IVSIZE		8
#define AESCTR_BLOCKSIZE	16

#define AES_XTS_BLOCKSIZE	16
#define AES_XTS_IVSIZE		8
#define AES_XTS_ALPHA		0x87	/* GF(2^128) generator polynomial */

struct aesni_aes_ctx {
	uint32_t		 aes_ekey[4 * (AES_MAXROUNDS + 1)];
	uint32_t		 aes_dkey[4 * (AES_MAXROUNDS + 1)];
	uint32_t		 aes_klen;
	uint32_t		 aes_pad[3];
};

struct aesni_xts_ctx {
	struct aesni_aes_ctx	 xts_keys[2];
};

struct aesni_session {
	uint32_t		 ses_ekey[4 * (AES_MAXROUNDS + 1)];
	uint32_t		 ses_dkey[4 * (AES_MAXROUNDS + 1)];
	uint32_t		 ses_klen;
	uint8_t			 ses_nonce[AESCTR_NONCESIZE];
	int			 ses_sid;
	GHASH_CTX		*ses_ghash;
	struct aesni_xts_ctx	*ses_xts;
	struct swcr_data	*ses_swd;
	LIST_ENTRY(aesni_session)
				 ses_entries;
};

struct aesni_softc {
	uint8_t			*sc_buf;
	size_t			 sc_buflen;
	int32_t			 sc_cid;
	uint32_t		 sc_sid;
	LIST_HEAD(, aesni_session)
				 sc_sessions;
} *aesni_sc;

struct pool aesnipl;

uint32_t aesni_ops;

/* assembler-assisted key setup */
extern void aesni_set_key(struct aesni_session *ses, uint8_t *key, size_t len);

/* aes encryption/decryption */
extern void aesni_enc(struct aesni_session *ses, uint8_t *dst, uint8_t *src);
extern void aesni_dec(struct aesni_session *ses, uint8_t *dst, uint8_t *src);

/* assembler-assisted CBC mode */
extern void aesni_cbc_enc(struct aesni_session *ses, uint8_t *dst,
	    uint8_t *src, size_t len, uint8_t *iv);
extern void aesni_cbc_dec(struct aesni_session *ses, uint8_t *dst,
	    uint8_t *src, size_t len, uint8_t *iv);

/* assembler-assisted CTR mode */
extern void aesni_ctr_enc(struct aesni_session *ses, uint8_t *dst,
	    uint8_t *src, size_t len, uint8_t *icb);

/* assembler-assisted XTS mode */
extern void aesni_xts_enc(struct aesni_xts_ctx *xts, uint8_t *dst,
	    uint8_t *src, size_t len, uint8_t *tweak);
extern void aesni_xts_dec(struct aesni_xts_ctx *xts, uint8_t *dst,
	    uint8_t *src, size_t len, uint8_t *tweak);

/* assembler-assisted GMAC */
extern void aesni_gmac_update(GHASH_CTX *ghash, uint8_t *src, size_t len);
extern void aesni_gmac_final(struct aesni_session *ses, uint8_t *tag,
    uint8_t *icb, uint8_t *hashstate);

void	aesni_setup(void);
int	aesni_newsession(u_int32_t *, struct cryptoini *);
int	aesni_freesession(u_int64_t);
int	aesni_process(struct cryptop *);

int	aesni_swauth(struct cryptop *, struct cryptodesc *, struct swcr_data *,
	    caddr_t);

int	aesni_encdec(struct cryptop *, struct cryptodesc *,
	    struct cryptodesc *, struct aesni_session *);

void
aesni_setup(void)
{
	int algs[CRYPTO_ALGORITHM_MAX + 1];

	aesni_sc = malloc(sizeof(*aesni_sc), M_DEVBUF, M_NOWAIT|M_ZERO);
	if (aesni_sc == NULL)
		return;

	aesni_sc->sc_buf = malloc(PAGE_SIZE, M_DEVBUF, M_NOWAIT|M_ZERO);
	if (aesni_sc->sc_buf != NULL)
		aesni_sc->sc_buflen = PAGE_SIZE;

	bzero(algs, sizeof(algs));

	/* Encryption algorithms. */
	algs[CRYPTO_AES_CBC] = CRYPTO_ALG_FLAG_SUPPORTED;
	algs[CRYPTO_AES_CTR] = CRYPTO_ALG_FLAG_SUPPORTED;
	algs[CRYPTO_AES_GCM_16] = CRYPTO_ALG_FLAG_SUPPORTED;
	algs[CRYPTO_AES_XTS] = CRYPTO_ALG_FLAG_SUPPORTED;

	/* Authenticated encryption algorithms. */
	algs[CRYPTO_AES_GMAC] = CRYPTO_ALG_FLAG_SUPPORTED;
	algs[CRYPTO_AES_128_GMAC] = CRYPTO_ALG_FLAG_SUPPORTED;
	algs[CRYPTO_AES_192_GMAC] = CRYPTO_ALG_FLAG_SUPPORTED;
	algs[CRYPTO_AES_256_GMAC] = CRYPTO_ALG_FLAG_SUPPORTED;

	/* HMACs needed for IPsec, uses software crypto. */
	algs[CRYPTO_MD5_HMAC] = CRYPTO_ALG_FLAG_SUPPORTED;
	algs[CRYPTO_SHA1_HMAC] = CRYPTO_ALG_FLAG_SUPPORTED;
	algs[CRYPTO_RIPEMD160_HMAC] = CRYPTO_ALG_FLAG_SUPPORTED;
	algs[CRYPTO_SHA2_256_HMAC] = CRYPTO_ALG_FLAG_SUPPORTED;
	algs[CRYPTO_SHA2_384_HMAC] = CRYPTO_ALG_FLAG_SUPPORTED;
	algs[CRYPTO_SHA2_512_HMAC] = CRYPTO_ALG_FLAG_SUPPORTED;

	/* IPsec Extended Sequence Numbers. */
	algs[CRYPTO_ESN] = CRYPTO_ALG_FLAG_SUPPORTED;

	aesni_sc->sc_cid = crypto_get_driverid(0);
	if (aesni_sc->sc_cid < 0) {
		free(aesni_sc, M_DEVBUF, 0);
		return;
	}

	pool_init(&aesnipl, sizeof(struct aesni_session), 16, 0, 0,
	    "aesni", NULL);
	pool_setlowat(&aesnipl, 2);

	crypto_register(aesni_sc->sc_cid, algs, aesni_newsession,
	    aesni_freesession, aesni_process);
}

int
aesni_newsession(u_int32_t *sidp, struct cryptoini *cri)
{
	struct aesni_session *ses = NULL;
	struct aesni_aes_ctx *aes1, *aes2;
	struct cryptoini *c;
	struct auth_hash *axf;
	struct swcr_data *swd;
	int i;

	if (sidp == NULL || cri == NULL)
		return (EINVAL);

	ses = pool_get(&aesnipl, PR_NOWAIT | PR_ZERO);
	if (!ses)
		return (ENOMEM);
	LIST_INSERT_HEAD(&aesni_sc->sc_sessions, ses, ses_entries);
	ses->ses_sid = ++aesni_sc->sc_sid;

	for (c = cri; c != NULL; c = c->cri_next) {
		switch (c->cri_alg) {
		case CRYPTO_AES_CBC:
			ses->ses_klen = c->cri_klen / 8;
			fpu_kernel_enter();
			aesni_set_key(ses, c->cri_key, ses->ses_klen);
			fpu_kernel_exit();
			break;

		case CRYPTO_AES_CTR:
		case CRYPTO_AES_GCM_16:
		case CRYPTO_AES_GMAC:
			ses->ses_klen = c->cri_klen / 8 - AESCTR_NONCESIZE;
			memcpy(ses->ses_nonce, c->cri_key + ses->ses_klen,
			    AESCTR_NONCESIZE);
			fpu_kernel_enter();
			aesni_set_key(ses, c->cri_key, ses->ses_klen);
			fpu_kernel_exit();
			break;

		case CRYPTO_AES_XTS:
			ses->ses_xts = malloc(sizeof(struct aesni_xts_ctx),
			    M_CRYPTO_DATA, M_NOWAIT | M_ZERO);
			if (ses->ses_xts == NULL) {
				aesni_freesession(ses->ses_sid);
				return (ENOMEM);
			}

			ses->ses_klen = c->cri_klen / 16;
			aes1 = &ses->ses_xts->xts_keys[0];
			aes1->aes_klen = ses->ses_klen;
			aes2 = &ses->ses_xts->xts_keys[1];
			aes2->aes_klen = ses->ses_klen;

			fpu_kernel_enter();
			aesni_set_key((struct aesni_session *)aes1,
			    c->cri_key, aes1->aes_klen);
			aesni_set_key((struct aesni_session *)aes2,
			    c->cri_key + ses->ses_klen, aes2->aes_klen);
			fpu_kernel_exit();
			break;

		case CRYPTO_AES_128_GMAC:
		case CRYPTO_AES_192_GMAC:
		case CRYPTO_AES_256_GMAC:
			ses->ses_ghash = malloc(sizeof(GHASH_CTX),
			    M_CRYPTO_DATA, M_NOWAIT | M_ZERO);
			if (ses->ses_ghash == NULL) {
				aesni_freesession(ses->ses_sid);
				return (ENOMEM);
			}

			bzero(ses->ses_ghash->H, GMAC_BLOCK_LEN);
			bzero(ses->ses_ghash->S, GMAC_BLOCK_LEN);
			bzero(ses->ses_ghash->Z, GMAC_BLOCK_LEN);

			/* prepare a hash subkey */
			aesni_enc(ses, ses->ses_ghash->H, ses->ses_ghash->H);
			break;

		case CRYPTO_MD5_HMAC:
			axf = &auth_hash_hmac_md5_96;
			goto authcommon;
		case CRYPTO_SHA1_HMAC:
			axf = &auth_hash_hmac_sha1_96;
			goto authcommon;
		case CRYPTO_RIPEMD160_HMAC:
			axf = &auth_hash_hmac_ripemd_160_96;
			goto authcommon;
		case CRYPTO_SHA2_256_HMAC:
			axf = &auth_hash_hmac_sha2_256_128;
			goto authcommon;
		case CRYPTO_SHA2_384_HMAC:
			axf = &auth_hash_hmac_sha2_384_192;
			goto authcommon;
		case CRYPTO_SHA2_512_HMAC:
			axf = &auth_hash_hmac_sha2_512_256;
		authcommon:
			swd = malloc(sizeof(struct swcr_data), M_CRYPTO_DATA,
			    M_NOWAIT|M_ZERO);
			if (swd == NULL) {
				aesni_freesession(ses->ses_sid);
				return (ENOMEM);
			}
			ses->ses_swd = swd;

			swd->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA,
			    M_NOWAIT);
			if (swd->sw_ictx == NULL) {
				aesni_freesession(ses->ses_sid);
				return (ENOMEM);
			}

			swd->sw_octx = malloc(axf->ctxsize, M_CRYPTO_DATA,
			    M_NOWAIT);
			if (swd->sw_octx == NULL) {
				aesni_freesession(ses->ses_sid);
				return (ENOMEM);
			}

			for (i = 0; i < c->cri_klen / 8; i++)
				c->cri_key[i] ^= HMAC_IPAD_VAL;

			axf->Init(swd->sw_ictx);
			axf->Update(swd->sw_ictx, c->cri_key, c->cri_klen / 8);
			axf->Update(swd->sw_ictx, hmac_ipad_buffer,
			    axf->blocksize - (c->cri_klen / 8));

			for (i = 0; i < c->cri_klen / 8; i++)
				c->cri_key[i] ^= (HMAC_IPAD_VAL ^
				    HMAC_OPAD_VAL);

			axf->Init(swd->sw_octx);
			axf->Update(swd->sw_octx, c->cri_key, c->cri_klen / 8);
			axf->Update(swd->sw_octx, hmac_opad_buffer,
			    axf->blocksize - (c->cri_klen / 8));

			for (i = 0; i < c->cri_klen / 8; i++)
				c->cri_key[i] ^= HMAC_OPAD_VAL;

			swd->sw_axf = axf;
			swd->sw_alg = c->cri_alg;

			break;

		case CRYPTO_ESN:
			/* nothing to do */
			break;

		default:
			aesni_freesession(ses->ses_sid);
			return (EINVAL);
		}
	}

	*sidp = ses->ses_sid;
	return (0);
}

int
aesni_freesession(u_int64_t tid)
{
	struct aesni_session *ses;
	struct swcr_data *swd;
	struct auth_hash *axf;
	u_int32_t sid = (u_int32_t)tid;

	LIST_FOREACH(ses, &aesni_sc->sc_sessions, ses_entries) {
		if (ses->ses_sid == sid)
			break;
	}

	if (ses == NULL)
		return (EINVAL);

	LIST_REMOVE(ses, ses_entries);

	if (ses->ses_ghash) {
		explicit_bzero(ses->ses_ghash, sizeof(GHASH_CTX));
		free(ses->ses_ghash, M_CRYPTO_DATA, 0);
	}

	if (ses->ses_xts) {
		explicit_bzero(ses->ses_xts, sizeof(struct aesni_xts_ctx));
		free(ses->ses_xts, M_CRYPTO_DATA, 0);
	}

	if (ses->ses_swd) {
		swd = ses->ses_swd;
		axf = swd->sw_axf;

		if (swd->sw_ictx) {
			explicit_bzero(swd->sw_ictx, axf->ctxsize);
			free(swd->sw_ictx, M_CRYPTO_DATA, 0);
		}
		if (swd->sw_octx) {
			explicit_bzero(swd->sw_octx, axf->ctxsize);
			free(swd->sw_octx, M_CRYPTO_DATA, 0);
		}
		free(swd, M_CRYPTO_DATA, 0);
	}

	explicit_bzero(ses, sizeof (*ses));
	pool_put(&aesnipl, ses);

	return (0);
}

int
aesni_swauth(struct cryptop *crp, struct cryptodesc *crd,
    struct swcr_data *sw, caddr_t buf)
{
	int type;

	if (crp->crp_flags & CRYPTO_F_IMBUF)
		type = CRYPTO_BUF_MBUF;
	else
		type = CRYPTO_BUF_IOV;

	return (swcr_authcompute(crp, crd, sw, buf, type));
}

int
aesni_encdec(struct cryptop *crp, struct cryptodesc *crd,
    struct cryptodesc *crda, struct aesni_session *ses)
{
	int aadlen, err, ivlen, iskip, oskip, rlen;
	uint8_t iv[EALG_MAX_BLOCK_LEN];
	uint8_t icb[AESCTR_BLOCKSIZE];
	uint8_t tag[GMAC_DIGEST_LEN];
	uint8_t *buf = aesni_sc->sc_buf;
	uint32_t *dw;

	aadlen = rlen = err = iskip = oskip = 0;

	if (crd->crd_len > aesni_sc->sc_buflen) {
		if (buf != NULL) {
			explicit_bzero(buf, aesni_sc->sc_buflen);
			free(buf, M_DEVBUF, 0);
		}

		aesni_sc->sc_buflen = 0;
		rlen = roundup(crd->crd_len, EALG_MAX_BLOCK_LEN);
		aesni_sc->sc_buf = buf = malloc(rlen, M_DEVBUF, M_NOWAIT |
		    M_ZERO);
		if (buf == NULL)
			return (ENOMEM);
		aesni_sc->sc_buflen = rlen;
	}

	/* CBC uses 16, CTR/XTS only 8. */
	ivlen = (crd->crd_alg == CRYPTO_AES_CBC) ? 16 : 8;

	/* Initialize the IV */
	if (crd->crd_flags & CRD_F_ENCRYPT) {
		if (crd->crd_flags & CRD_F_IV_EXPLICIT)
			memcpy(iv, crd->crd_iv, ivlen);
		else
			arc4random_buf(iv, ivlen);

		/* Do we need to write the IV */
		if ((crd->crd_flags & CRD_F_IV_PRESENT) == 0) {
			if (crp->crp_flags & CRYPTO_F_IMBUF) {
				if (m_copyback((struct mbuf *)crp->crp_buf,
				    crd->crd_inject, ivlen, iv, M_NOWAIT)) {
					err = ENOMEM;
					goto out;
				}
			} else
				cuio_copyback((struct uio *)crp->crp_buf,
				    crd->crd_inject, ivlen, iv);
		}
	} else {
		if (crd->crd_flags & CRD_F_IV_EXPLICIT)
			memcpy(iv, crd->crd_iv, ivlen);
		else {
			if (crp->crp_flags & CRYPTO_F_IMBUF)
				m_copydata((struct mbuf *)crp->crp_buf,
				    crd->crd_inject, ivlen, iv);
			else
				cuio_copydata((struct uio *)crp->crp_buf,
				    crd->crd_inject, ivlen, iv);
		}
	}

	if (crda) {
		/* Supply GMAC with AAD */
		aadlen = crda->crd_len;
		if (crda->crd_flags & CRD_F_ESN) {
			aadlen += 4;
			/* SPI */
			if (crp->crp_flags & CRYPTO_F_IMBUF)
				m_copydata((struct mbuf *)crp->crp_buf,
				    crda->crd_skip, 4, buf);
			else
				cuio_copydata((struct uio *)crp->crp_buf,
				    crda->crd_skip, 4, buf);
			iskip = 4; /* additional input offset */
			/* ESN */
			memcpy(buf + 4, crda->crd_esn, 4);
			oskip = iskip + 4; /* offset output buffer by 8 */
		}
		rlen = roundup(aadlen, GMAC_BLOCK_LEN);
		if (crp->crp_flags & CRYPTO_F_IMBUF)
			m_copydata((struct mbuf *)crp->crp_buf,
			    crda->crd_skip + iskip, crda->crd_len - iskip,
			    buf + oskip);
		else
			cuio_copydata((struct uio *)crp->crp_buf,
			    crda->crd_skip + iskip, crda->crd_len - iskip,
			    buf + oskip);
		fpu_kernel_enter();
		aesni_gmac_update(ses->ses_ghash, buf, rlen);
		fpu_kernel_exit();
		bzero(buf, aadlen);
	}

	/* Copy data to be processed to the buffer */
	if (crp->crp_flags & CRYPTO_F_IMBUF)
		m_copydata((struct mbuf *)crp->crp_buf, crd->crd_skip,
		    crd->crd_len, buf);
	else
		cuio_copydata((struct uio *)crp->crp_buf, crd->crd_skip,
		    crd->crd_len, buf);

	if (crd->crd_alg == CRYPTO_AES_CTR ||
	    crd->crd_alg == CRYPTO_AES_GCM_16 ||
	    crd->crd_alg == CRYPTO_AES_GMAC) {
		bzero(icb, AESCTR_BLOCKSIZE);
		memcpy(icb, ses->ses_nonce, AESCTR_NONCESIZE);
		memcpy(icb + AESCTR_NONCESIZE, iv, AESCTR_IVSIZE);
		/* rlen is for gcm and gmac only */
		rlen = roundup(crd->crd_len, AESCTR_BLOCKSIZE);
	}

	/* Apply cipher */
	fpu_kernel_enter();
	switch (crd->crd_alg) {
	case CRYPTO_AES_CBC:
		if (crd->crd_flags & CRD_F_ENCRYPT)
			aesni_cbc_enc(ses, buf, buf, crd->crd_len, iv);
		else
			aesni_cbc_dec(ses, buf, buf, crd->crd_len, iv);
		break;
	case CRYPTO_AES_CTR:
		aesni_ctr_enc(ses, buf, buf, crd->crd_len, icb);
		break;
	case CRYPTO_AES_GCM_16:
		icb[AESCTR_BLOCKSIZE - 1] = 1;
		if (crd->crd_flags & CRD_F_ENCRYPT) {
			/* encrypt padded data */
			aesni_ctr_enc(ses, buf, buf, rlen, icb);
			/* zero out padding bytes */
			bzero(buf + crd->crd_len, rlen - crd->crd_len);
			/* hash encrypted data padded with zeroes */
			aesni_gmac_update(ses->ses_ghash, buf, rlen);
		} else {
			aesni_gmac_update(ses->ses_ghash, buf, rlen);
			aesni_ctr_enc(ses, buf, buf, rlen, icb);
		}
		goto gcmcommon;
	case CRYPTO_AES_GMAC:
		icb[AESCTR_BLOCKSIZE - 1] = 1;
		aesni_gmac_update(ses->ses_ghash, buf, rlen);
	gcmcommon:
		/* lengths block */
		bzero(tag, GMAC_BLOCK_LEN);
		dw = (uint32_t *)tag + 1;
		*dw = htobe32(aadlen * 8);
		dw = (uint32_t *)tag + 3;
		*dw = htobe32(crd->crd_len * 8);
		aesni_gmac_update(ses->ses_ghash, tag, GMAC_BLOCK_LEN);
		/* finalization */
		aesni_gmac_final(ses, tag, icb, ses->ses_ghash->S);
		break;
	case CRYPTO_AES_XTS:
		if (crd->crd_flags & CRD_F_ENCRYPT)
			aesni_xts_enc(ses->ses_xts, buf, buf, crd->crd_len, iv);
		else
			aesni_xts_dec(ses->ses_xts, buf, buf, crd->crd_len, iv);
		break;
	}
	fpu_kernel_exit();

	aesni_ops++;

	/* Copy back the result */
	if (crp->crp_flags & CRYPTO_F_IMBUF) {
		if (m_copyback((struct mbuf *)crp->crp_buf, crd->crd_skip,
		    crd->crd_len, buf, M_NOWAIT)) {
			err = ENOMEM;
			goto out;
		}
	} else
		cuio_copyback((struct uio *)crp->crp_buf, crd->crd_skip,
		    crd->crd_len, buf);

	/* Copy back the authentication tag */
	if (crda) {
		if (crp->crp_flags & CRYPTO_F_IMBUF) {
			if (m_copyback((struct mbuf *)crp->crp_buf,
			    crda->crd_inject, GMAC_DIGEST_LEN, tag,
			    M_NOWAIT)) {
				err = ENOMEM;
				goto out;
			}
		} else
			memcpy(crp->crp_mac, tag, GMAC_BLOCK_LEN);

		/* clean up GHASH state */
		bzero(ses->ses_ghash->S, GMAC_BLOCK_LEN);
		bzero(ses->ses_ghash->Z, GMAC_BLOCK_LEN);
	}

out:
	explicit_bzero(buf, roundup(crd->crd_len, EALG_MAX_BLOCK_LEN));
	return (err);
}

int
aesni_process(struct cryptop *crp)
{
	struct aesni_session *ses;
	struct cryptodesc *crd, *crda, *crde;
	int err = 0;

	if (crp == NULL || crp->crp_callback == NULL)
		return (EINVAL);

	LIST_FOREACH(ses, &aesni_sc->sc_sessions, ses_entries) {
		if (ses->ses_sid == (crp->crp_sid & 0xffffffff))
			break;
	}

	if (!ses) {
		err = EINVAL;
		goto out;
	}

	crda = crde = NULL;
	for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
		switch (crd->crd_alg) {
		case CRYPTO_AES_CBC:
		case CRYPTO_AES_CTR:
		case CRYPTO_AES_XTS:
			err = aesni_encdec(crp, crd, NULL, ses);
			if (err != 0)
				goto out;
			break;

		case CRYPTO_AES_GCM_16:
		case CRYPTO_AES_GMAC:
			crde = crd;
			if (!crda)
				continue;
			goto gcmcommon;
		case CRYPTO_AES_128_GMAC:
		case CRYPTO_AES_192_GMAC:
		case CRYPTO_AES_256_GMAC:
			crda = crd;
			if (!crde)
				continue;
		gcmcommon:
			err = aesni_encdec(crp, crde, crda, ses);
			if (err != 0)
				goto out;
			break;

		case CRYPTO_MD5_HMAC:
		case CRYPTO_SHA1_HMAC:
		case CRYPTO_RIPEMD160_HMAC:
		case CRYPTO_SHA2_256_HMAC:
		case CRYPTO_SHA2_384_HMAC:
		case CRYPTO_SHA2_512_HMAC:
			err = aesni_swauth(crp, crd, ses->ses_swd,
			    crp->crp_buf);
			if (err != 0)
				goto out;
			break;

		default:
			err = EINVAL;
			goto out;
		}
	}

out:
	crp->crp_etype = err;
	crypto_done(crp);
	return (err);
}