/******************************************************************************
Hardware Programming Interface (HPI) for AudioScience
 ASI67xx series adapters.
 These PCIe bus adapters are based on a
 TMS320DM8147 ARM+DSP with PCIe bus mastering capability,

 Entry point:
   int HPI6700_adapter_init(struct hpi_adapter_obj *pao);

    Copyright (C) 1997-2017  AudioScience Inc. <support@audioscience.com>

    This program is free software; you can redistribute it and/or modify
    it under the terms of version 2 of the GNU General Public License as
    published by the Free Software Foundation;

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

*******************************************************************************/
#define SOURCEFILE_NAME "hpi6700.c"

#include "hpi_internal.h"
#include "hpimsginit.h"
#include "hpimsgx.h"
#include "hpidebug.h"
#include "hpi6700.h"
#include "hpidspcd.h"
#include "hpicmn.h"

/***********************************************************
	Defines used for basic messaging
************************************************************/
#define H670_HIF_RESET		0
#define H670_HIF_IDLE		1
#define H670_HIF_GET_RESP	2
#define H670_HIF_DATA_DONE	3
#define H670_HIF_DATA_MASK	0x10
#define H670_HIF_SEND_DATA	0x14
#define H670_HIF_GET_DATA	0x15
#define H670_HIF_UNKNOWN	0x0000ffff

#define HPI6700_TIMEOUT (100000) /* one hundred milliseconds */
#define HPI6700_WAIT_LOOP_DELAY	(10) /* 10 microseconds */

#define UBOOT_STATE_MAX_RETRIES (1000)
#define DSP_ACK_USEC_PER_RETRY	(1000)
#define LINE_ALIGNMENT		(0x10)

#ifndef HPI_OS_LINUX_KERNEL
#define HIF_BUFFER_ALIGNMENT		(0x100000)
#define STREAM_BUFFER_ALIGNMENT	(0x100000)
#else
#define HIF_BUFFER_ALIGNMENT	(0)
#define STREAM_BUFFER_ALIGNMENT	(0)

#define HPI6700_POLL_DSP_ACK
#endif

#define H670_CONTROLCACHEESTIMATE (HPI_NMIXER_CONTROLS * sizeof(struct hpi_control_cache_meter))
#define H670_HIF_BUFFER_SIZE \
		(sizeof(struct bus_master_interface_67) + \
		H670_CONTROLCACHEESTIMATE)

#ifndef HPI_BUILD_SANITISE
typedef volatile struct bus_master_interface_67 tBusMasteringInterfaceBuffer;
typedef volatile struct async_event_buffer_6700 H670_ASYNC_EVENT_BUFFER;
typedef volatile struct controlcache_6700 H670_CONTROLCACHE;
typedef volatile struct hpi_hostbuffer_status H670_HOSTBUFFER_STATUS;
#endif

/*****************************************************************************/

struct bar_status {
	uint32_t __iomem *host_address;
	uint32_t size;
	uint32_t mapped_device_address;
};

struct mapped_device_struct {
	uint32_t bar;
	uint32_t device_address;
	uint32_t size;
};

struct debug_values {
	uint32_t info_magic_val;
	uint32_t GPR0_first;
	uint32_t GPR2;
	uint32_t GPR3;
	uint32_t GPR0_second;
	uint32_t DSP_dspinfomagicok;
	uint32_t DSP_busmastermagicok;
	uint32_t dspinfo_magic_val;
	uint32_t dspinfo_dsp_ready;
};

struct adapter_properties {
	uint32_t max_sample_rate;
	uint32_t chans_per_player;
	uint32_t chans_per_recorder;
};

struct hpi_hw_stream_obj_67 {
	void 		*HostBufferVirt;
	uint32_t	HostBufferPhys;
	uint32_t	HostBufferSize;
	uint32_t	HostBufferActive;
#ifndef HPI_BUILD_NO_STREAM_WAIT
	/* non-zero size means a client thread is waiting */
	uint32_t	Threshold;
	hpios_event	Event;
#endif
};

struct hpi_hw_obj_67 {
	HpiOs_LockedMem_Handle hLockedMem;
	struct bus_master_interface_67 *pInterfaceBuffer;
	uint32_t dwInterfaceBufferPhysAddr;
	uint32_t dwMessageInterruptCount;
	uint32_t dwStreamInterruptCount;
#ifndef HPI6700_POLL_DSP_ACK
	hpios_event dsp_ack_event;
#endif
	struct bar_status bar[3];

	struct mapped_device_struct uboot_info;
	struct mapped_device_struct dsp_info;
	struct mapped_device_struct soc_phymem;

	struct debug_values dbg;
	struct adapter_properties props;

	uint16_t flagOStreamJustReset[HPI_MAX_STREAMS];

	HpiOs_LockedMem_Handle hStreamMem;

	struct hpi_hw_stream_obj_67 InStreams[HPI_MAX_STREAMS];
	struct hpi_hw_stream_obj_67 OutStreams[HPI_MAX_STREAMS];

	struct hpi_control_cache *pCache;
};

/*****************************************************************************/
/* local prototypes */

#ifndef HPI_OS_LINUX_KERNEL
/* Helper functions for power of 2 buffering */
/*Linux already defines these functions */
#ifndef min
#define min(a, b) ((a) < (b) ? (a) : (b))
#endif

static unsigned int roundup_pow_of_two(unsigned int size)
{
	unsigned int n = 0;
	if (size)
		size = size - 1;

	while (size) {
		size >>= 1;
		n++;
	}
	return 1 << n;

}
#endif

static hpi_err_t AdapterBootLoadDsp(
	struct hpi_adapter_obj *pao
);

static void HwMessage(
	struct hpi_adapter_obj *pao,
	struct hpi_message *phm,
	struct hpi_response *phr
);

static void AdapterDelete(
	struct hpi_adapter_obj *pao,
	struct hpi_message *phm,
	struct hpi_response *phr
);

static hpi_err_t AllocStreamMem(struct hpi_adapter_obj *pao);

static void AdapterShutdown(struct hpi_adapter_obj *pao);

static int AdapterIrqQueryAndClear(struct hpi_adapter_obj *pao, uint32_t message);

#ifndef HPI_BUILD_NO_STREAM_WAIT
static void AdapterMsgIrqQueryAndClear(
	struct hpi_adapter_obj *pao,
	struct hpi_message *phm,
	struct hpi_response *phr
);

static void AdapterIrqCallback(
	struct hpi_adapter_obj *pao,
	struct hpi_message *phm,
	struct hpi_response *phr
);

static uint32_t OutStreamThresholdReached(
	struct hpi_hostbuffer_status *status,
	uint32_t threshold
);

static uint32_t InStreamThresholdReached(
	struct hpi_hostbuffer_status *status,
	uint32_t threshold
);
#endif

/*****************************************************************************/
static void AdapterMessage(
	struct hpi_adapter_obj *pao,
	struct hpi_message *phm,
	struct hpi_response *phr
)
{
	switch (phm->wFunction) {
	case HPI_ADAPTER_DELETE:
		HPI_DEBUG_LOG0(DEBUG, "HPI_ADAPTER_DELETE\n");
		AdapterDelete(pao, phm, phr);
		break;
#ifndef HPI_BUILD_NO_STREAM_WAIT
	case HPI_ADAPTER_IRQ_QUERY_AND_CLEAR:
		AdapterMsgIrqQueryAndClear(pao, phm, phr);
		break;
	case HPI_ADAPTER_IRQ_CALLBACK:
		AdapterIrqCallback(pao, phm, phr);
		break;
#else
	case HPI_ADAPTER_IRQ_QUERY_AND_CLEAR:
	case HPI_ADAPTER_IRQ_CALLBACK:
		phr->wError = HPI_ERROR_UNIMPLEMENTED;
		break;
#endif
	default:
		switch (phm->wFunction) {
		case HPI_ADAPTER_CLOSE:
			HPI_DEBUG_LOG0(DEBUG, "HPI_ADAPTER_CLOSE\n");
			break;
		default:
			break;
		}
		HwMessage(pao, phm, phr);
		break;
	}
}

static void ControlMessage(
	struct hpi_adapter_obj *pao,
	struct hpi_message *phm,
	struct hpi_response *phr
)
{
	struct hpi_hw_obj_67 *phw = pao->priv;

	switch (phm->wFunction) {
	case HPI_CONTROL_GET_STATE:
		if (pao->wHasControlCache) {
			RMB(); /* make sure we see updates DMAed from DSP */
			if (HpiCheckControlCache(phw->pCache, phm, phr)) {
				break;
			}
			/* control cache must work for peak meters so report an error */
			if (phm->u.c.wAttribute == HPI_METER_PEAK) {
				phr->wError = HPI_ERROR_CONTROL_CACHING;
				break;
			}
		}
		HwMessage(pao, phm, phr);
		break;
	case HPI_CONTROL_GET_INFO:
		HwMessage(pao, phm, phr);
		break;
	case HPI_CONTROL_SET_STATE:
		HwMessage(pao, phm, phr);
		if (pao->wHasControlCache)
			HpiCmn_ControlCache_SyncToMsg(phw->pCache, phm, phr);
		break;
	default:
		phr->wError = HPI_ERROR_INVALID_FUNC;
		break;
	}
}

static int AdapterIrqQueryAndClear(struct hpi_adapter_obj *pao, uint32_t message)
{
	struct hpi_hw_obj_67 *phw = pao->priv;
	struct bus_master_interface_67 *interface = NULL;

	if (phw)
		interface = phw->pInterfaceBuffer;

	if (interface && interface->dwDSPStreamInterruptCount != phw->dwStreamInterruptCount)
		return HPI_IRQ_MIXER;

	return HPI_IRQ_MESSAGE;
}

#ifndef HPI_BUILD_NO_STREAM_WAIT
static void AdapterMsgIrqQueryAndClear(
	struct hpi_adapter_obj *pao,
	struct hpi_message *phm,
	struct hpi_response *phr
)
{
	int found = AdapterIrqQueryAndClear(pao, phm->u.ax.irq.message);

	HPI_InitResponse(phr, phm->wObject, phm->wFunction, 0);

	phr->u.ax.irq_query.yes = found;
}

static void OutStreamSignalEvent(
	struct hpi_hw_obj_67 *phw,
	int nStream
)
{
	struct bus_master_interface_67 *iface = phw->pInterfaceBuffer;

	if ( ! phw->OutStreams[nStream].HostBufferActive )
		return;

	if ( ! phw->OutStreams[nStream].Threshold )
		return;

	if ( iface->aOutStreamHostBufferStatus[nStream].dwStreamState
			== HPI_STATE_STOPPED ) {
		HpiOs_Event_Set(&phw->OutStreams[nStream].Event);
		return;
	}

	if (OutStreamThresholdReached(&iface->aOutStreamHostBufferStatus[nStream],
				      phw->OutStreams[nStream].Threshold))
		HpiOs_Event_Set(&phw->OutStreams[nStream].Event);
}

static void InStreamSignalEvent(
	struct hpi_hw_obj_67 *phw,
	int nStream
)
{
	struct bus_master_interface_67 *iface = phw->pInterfaceBuffer;

	if ( ! phw->InStreams[nStream].HostBufferActive )
		return ;

	if ( ! phw->InStreams[nStream].Threshold )
		return ;

	if (iface->aInStreamHostBufferStatus[nStream].dwStreamState
			== HPI_STATE_STOPPED ) {
		HpiOs_Event_Set(&phw->InStreams[nStream].Event);
		return;
	}

	if (InStreamThresholdReached(&iface->aInStreamHostBufferStatus[nStream],
				      phw->InStreams[nStream].Threshold))
		HpiOs_Event_Set(&phw->InStreams[nStream].Event);
}

static void AdapterIrqCallback(
	struct hpi_adapter_obj *pao,
	struct hpi_message *phm,
	struct hpi_response *phr
)
{
	struct hpi_hw_obj_67 *phw = pao->priv;
	int i;
	struct bus_master_interface_67 *iface = NULL;

	if (phw)
		iface = phw->pInterfaceBuffer;

	if(iface){
		uint32_t dwDspAckCount = iface->dwDspAckCount;
		uint32_t dwStreamInterruptCount = iface->dwDSPStreamInterruptCount;

		if (dwDspAckCount != phw->dwMessageInterruptCount) {
			phw->dwMessageInterruptCount = dwDspAckCount;
			HpiOs_Event_Set(&phw->dsp_ack_event);
		}
		if (dwStreamInterruptCount != phw->dwStreamInterruptCount) {
			phw->dwStreamInterruptCount = dwStreamInterruptCount;
			for (i = 0; i < pao->outstreams; i++)
				OutStreamSignalEvent(phw, i);
			for (i = 0; i < pao->instreams; i++)
				InStreamSignalEvent(phw, i);
		}
	}

	phr->wError = 0;
}
#endif

/** Allocate or attach buffer for busmastering
*/
static void StreamHostBufferAllocate(
	struct hpi_adapter_obj *pao,
	struct hpi_message *phm,
	struct hpi_response *phr,
	struct hpi_hw_stream_obj_67 *pstream,
	struct hpi_hostbuffer_status *phb_status
)
{
	uint32_t dwCommand = phm->u.d.u.buffer.dwCommand;

	HPI_InitResponse(phr, phm->wObject, phm->wFunction, 0);

	if (dwCommand == HPI_BUFFER_CMD_EXTERNAL
		|| dwCommand == HPI_BUFFER_CMD_INTERNAL_ALLOC) {
		/* ALLOC phase, allocate a buffer with power of 2 size,
		   get its bus address for PCI bus mastering
		 */
		phm->u.d.u.buffer.dwBufferSize =
			roundup_pow_of_two(phm->u.d.u.buffer.dwBufferSize);

		if (pstream->HostBufferSize <
			phm->u.d.u.buffer.dwBufferSize) {
			/* Invalid size */
			phr->wError = HPI_ERROR_INVALID_DATASIZE;
			HPI_DEBUG_LOG2(ERROR,
				"Buffer size requested is too big %d (must be %d or less)\n",
				phm->u.d.u.buffer.dwBufferSize,
				pstream->HostBufferSize);
			return;
		}

		phm->u.d.u.buffer.dwBufferSize =
			pstream->HostBufferSize;
		/* return old size and allocated size,
		   so caller can detect change */
		phr->u.d.u.stream_info.dwDataAvailable =
			pstream->HostBufferSize;
		phr->u.d.u.stream_info.dwBufferSize =
			phm->u.d.u.buffer.dwBufferSize;

		if (pstream->HostBufferActive) {
			/* Already active, no action required */
			return;
		}

		phm->u.d.u.buffer.dwPciAddress =
			pstream->HostBufferPhys;
		//phm->u.d.u.buffer.dwPciAddress64_hi32 = 0;

		/* get the phys addr into msg for single call alloc caller
		 * needs to do this for split alloc (or use the same message)
		 * return the phy address for split alloc in the respose too
		 */
		phr->u.d.u.stream_info.dwAuxiliaryDataAvailable =
			pstream->HostBufferPhys;
	}

	if (dwCommand == HPI_BUFFER_CMD_EXTERNAL
		|| dwCommand == HPI_BUFFER_CMD_INTERNAL_GRANTADAPTER) {
		/* GRANT phase.  Set up the BBM status, tell the DSP about
		   the buffer so it can start using BBM.
		 */

		phb_status->dwSamplesProcessed = 0;
		phb_status->dwStreamState = HPI_STATE_STOPPED;
		phb_status->dwDspIndex = 0;
		phb_status->dwHostIndex = phb_status->dwDspIndex;
		phb_status->dwSizeInBytes = phm->u.d.u.buffer.dwBufferSize;
		phb_status->dwAuxiliaryDataAvailable = 0;

		HwMessage(pao, phm, phr);

		if (phr->wError) {
			pstream->HostBufferActive = 0;
			HPI_DEBUG_LOG1(ERROR,
				"HPI_BUFFER_CMD_INTERNAL_GRANTADAPTER failed in DSP; returned error %d\n",
				phr->wError);
		} else {
			pstream->HostBufferActive = 1;
		}
	}
}

static void StreamHostBufferGetInfo(
	struct hpi_adapter_obj *pao,
	struct hpi_message *phm,
	struct hpi_response *phr,
	struct hpi_hw_stream_obj_67 *pstream,
	struct hpi_hostbuffer_status *phb_status
)
{
	//struct hpi_hw_obj_67 *phw = pao->priv;

	HPI_InitResponse(phr, phm->wObject, phm->wFunction, 0);

	if (pstream->HostBufferActive) {
		phr->u.d.u.hostbuffer_info.pBuffer = pstream->HostBufferVirt;
		phr->u.d.u.hostbuffer_info.pStatus = phb_status;
	} else {
		phr->wError = HPI_ERROR_INVALID_OPERATION;
	}
}

static void StreamHostBufferFree(
	struct hpi_adapter_obj *pao,
	struct hpi_message *phm,
	struct hpi_response *phr,
	struct hpi_hw_stream_obj_67 *pstream
)
{
	//struct hpi_hw_obj_67 *phw = pao->priv;
	uint32_t dwCommand = phm->u.d.u.buffer.dwCommand;

	HPI_InitResponse(phr, phm->wObject, phm->wFunction, 0);

	if (pstream->HostBufferActive) {
		if (dwCommand == HPI_BUFFER_CMD_EXTERNAL ||
			dwCommand == HPI_BUFFER_CMD_INTERNAL_REVOKEADAPTER) {
			pstream->HostBufferActive = 0;
			HwMessage(pao, phm, phr);
			// Tell adapter to stop using the host buffer.
		}
	}
}

/*****************************************************************************/
/* OutStream Host buffer functions */

/** Allocate or attach buffer for busmastering
*/
static void OutStreamHostBufferAllocate(
	struct hpi_adapter_obj *pao,
	struct hpi_message *phm,
	struct hpi_response *phr
)
{
	//hpi_err_t err = 0;
	//uint32_t dwCommand = phm->u.d.u.buffer.dwCommand;
	struct hpi_hw_obj_67 *phw = pao->priv;
	struct bus_master_interface_67 *interface = phw->pInterfaceBuffer;
	struct hpi_hw_stream_obj_67 *pstream;
	struct hpi_hostbuffer_status *phb_status;

	pstream = &phw->OutStreams[phm->wObjIndex];
	phb_status = &interface->aOutStreamHostBufferStatus[phm->wObjIndex];

	StreamHostBufferAllocate(pao, phm, phr, pstream, phb_status);
}

static void OutStreamHostBufferGetInfo(
	struct hpi_adapter_obj *pao,
	struct hpi_message *phm,
	struct hpi_response *phr
)
{
	struct hpi_hw_obj_67 *phw = pao->priv;
	struct bus_master_interface_67 *interface = phw->pInterfaceBuffer;
	struct hpi_hw_stream_obj_67 *pstream;
	struct hpi_hostbuffer_status *phb_status;

	pstream = &phw->OutStreams[phm->wObjIndex];
	phb_status = &interface->aOutStreamHostBufferStatus[phm->wObjIndex];

	StreamHostBufferGetInfo(pao, phm, phr, pstream, phb_status);
}

static void OutStreamHostBufferFree(
	struct hpi_adapter_obj *pao,
	struct hpi_message *phm,
	struct hpi_response *phr
)
{
	struct hpi_hw_obj_67 *phw = pao->priv;
	struct hpi_hw_stream_obj_67 *pstream;

	pstream = &phw->OutStreams[phm->wObjIndex];

	StreamHostBufferFree(pao, phm, phr, pstream);
}

static uint32_t OutStreamGetSpaceAvailable(
	struct hpi_hostbuffer_status *status
)
{
	return status->dwSizeInBytes - (status->dwHostIndex - status->dwDspIndex);
}

#ifndef HPI_BUILD_NO_STREAM_WAIT
static uint32_t OutStreamThresholdReached(
	struct hpi_hostbuffer_status *status,
	uint32_t threshold
)
{
	return (status->dwSizeInBytes - OutStreamGetSpaceAvailable(status) +
		 status->dwAuxiliaryDataAvailable) <= threshold;
}
#endif

static void OutStreamWrite(
	struct hpi_adapter_obj *pao,
	struct hpi_message *phm,
	struct hpi_response *phr
)
{
	struct hpi_hw_obj_67 *phw = pao->priv;
	struct bus_master_interface_67 *interface = phw->pInterfaceBuffer;
	struct hpi_hw_stream_obj_67 *pstream;
	struct hpi_hostbuffer_status *phb_status;
	uint32_t dwSpaceAvailable;

	pstream = &phw->OutStreams[phm->wObjIndex];
	phb_status = &interface->aOutStreamHostBufferStatus[phm->wObjIndex];

	HPI_InitResponse(phr, phm->wObject, phm->wFunction, 0);

	if (!pstream->HostBufferActive) {
		/* there is no BBM buffer, fail */
		phr->wError = HPI_ERROR_HOST_BUFFER_NOT_ALLOCATED;
		return;
	}

	dwSpaceAvailable = OutStreamGetSpaceAvailable(phb_status);
	if (dwSpaceAvailable < phm->u.d.u.data.dwDataSize) {
		phr->wError = HPI_ERROR_INVALID_DATASIZE;
		return;
	}

	/* HostBuffers is used to indicate host buffer is internally allocated.
	   otherwise, assumed external, data written externally */
	if (phm->u.d.u.data.pbData) {
		uint8_t *pBbmData;
		uint32_t lFirstWrite;
		uint8_t *pAppData =
			(uint8_t *)phm->u.d.u.data.pbData;

		pBbmData = (uint8_t *)pstream->HostBufferVirt;

		/* either all data,
		   or enough to fit from current to end of BBM buffer */
		lFirstWrite = min(phm->u.d.u.data.dwDataSize,
			phb_status->dwSizeInBytes -
			(phb_status->dwHostIndex & (phb_status->dwSizeInBytes - 1)));

		memcpy(pBbmData +
			(phb_status->dwHostIndex & (phb_status->dwSizeInBytes
					- 1)), pAppData, lFirstWrite);
		/* remaining data if any */
		memcpy(pBbmData, pAppData + lFirstWrite,
			phm->u.d.u.data.dwDataSize - lFirstWrite);
	}

	/*
	 * This version relies on the DSP code triggering an OStream buffer
	 * update immediately following a SET_FORMAT call. The host has
	 * already written data into the BBM buffer, but the DSP won't know
	 * about it until dwHostIndex is adjusted.
	 */
	if (phw->flagOStreamJustReset[phm->wObjIndex]) {
		// Format can only change after reset. Must tell DSP.
		uint16_t wFunction = phm->wFunction;

		phw->flagOStreamJustReset[phm->wObjIndex] = 0;
		phm->wFunction = HPI_OSTREAM_SET_FORMAT;
		HwMessage(pao, phm, phr); 	// send the format to the DSP
		phm->wFunction = wFunction;
		if (phr->wError)
			return;
	}

	phb_status->dwHostIndex += phm->u.d.u.data.dwDataSize;
}

static void OutStreamGetInfo(
	struct hpi_adapter_obj *pao,
	struct hpi_message *phm,
	struct hpi_response *phr
)
{
	struct hpi_hw_obj_67 *phw = pao->priv;
	struct bus_master_interface_67 *interface = phw->pInterfaceBuffer;
	struct hpi_hw_stream_obj_67 *pstream;
	struct hpi_hostbuffer_status *phb_status;

	pstream = &phw->OutStreams[phm->wObjIndex];
	phb_status = &interface->aOutStreamHostBufferStatus[phm->wObjIndex];

	HPI_InitResponse(phr, phm->wObject, phm->wFunction, 0);

	if (!pstream->HostBufferActive) {
		/* there is no BBM buffer, fail */
		phr->wError = HPI_ERROR_HOST_BUFFER_NOT_ALLOCATED;
		return;
	}

	phr->u.d.u.stream_info.wState = (uint16_t)phb_status->dwStreamState;
	phr->u.d.u.stream_info.dwSamplesTransferred =
		phb_status->dwSamplesProcessed;
	phr->u.d.u.stream_info.dwBufferSize = phb_status->dwSizeInBytes;
	phr->u.d.u.stream_info.dwDataAvailable =
		phb_status->dwSizeInBytes - OutStreamGetSpaceAvailable(phb_status);
	phr->u.d.u.stream_info.dwAuxiliaryDataAvailable =
		phb_status->dwAuxiliaryDataAvailable;
}

static void OutStreamStart(
	struct hpi_adapter_obj *pao,
	struct hpi_message *phm,
	struct hpi_response *phr
)
{
	HwMessage(pao, phm, phr);
	HPI_DEBUG_LOG0(DEBUG, "OutStreamStart()\n");
}

static void OutStreamReset(
	struct hpi_adapter_obj *pao,
	struct hpi_message *phm,
	struct hpi_response *phr
)
{
	struct hpi_hw_obj_67 *phw = pao->priv;

#ifndef HPI_BUILD_NO_STREAM_WAIT
	HpiOs_Event_Set(&phw->OutStreams[phm->wObjIndex].Event);
#endif

	phw->flagOStreamJustReset[phm->wObjIndex] = 1;
	HwMessage(pao, phm, phr);
}

static void OutStreamOpen(
	struct hpi_adapter_obj *pao,
	struct hpi_message *phm,
	struct hpi_response *phr
)
{
	OutStreamReset(pao, phm, phr);
}

#ifndef HPI_BUILD_NO_STREAM_WAIT
static void OutStreamWait(
	struct hpi_adapter_obj *pao,
	struct hpi_message *phm,
	struct hpi_response *phr
)
{
	struct hpi_hw_obj_67 *phw = pao->priv;
	struct bus_master_interface_67 *interface = phw->pInterfaceBuffer;
	struct hpi_hw_stream_obj_67 *pstream;
	struct hpi_hostbuffer_status *phb_status;

	pstream = &phw->OutStreams[phm->wObjIndex];
	phb_status = &interface->aOutStreamHostBufferStatus[phm->wObjIndex];

	HPI_InitResponse(phr, phm->wObject, phm->wFunction, 0);

	if (!pstream->HostBufferActive) {
		/* there is no BBM buffer, fail */
		phr->wError = HPI_ERROR_HOST_BUFFER_NOT_ALLOCATED;
		return;
	}

	pstream->Threshold = phm->u.d.u.dwThresholdBytes;
	if (!OutStreamThresholdReached(phb_status,
		pstream->Threshold)){
			HpiOs_Event_Wait(&pstream->Event);
			HpiOs_Event_Clear(&pstream->Event);
			if (!OutStreamThresholdReached(phb_status,
				pstream->Threshold)){
					phr->wError = HPI_ERROR_WAIT_CANCELLED;
			}
	}
	pstream->Threshold = 0;
}
#endif

static void OStreamMessage(
	struct hpi_adapter_obj *pao,
	struct hpi_message *phm,
	struct hpi_response *phr
)
{
	if (phm->wObjIndex >= HPI_MAX_STREAMS) {
		phr->wError = HPI_ERROR_INVALID_OBJ_INDEX;
		HPI_DEBUG_LOG2(WARNING,
			"Message referencing invalid stream %d on adapter index %d\n",
			phm->wObjIndex, phm->wAdapterIndex);
		return;
	}

	switch (phm->wFunction) {
	case HPI_OSTREAM_WRITE:
		OutStreamWrite(pao, phm, phr);
		break;
	case HPI_OSTREAM_GET_INFO:
		OutStreamGetInfo(pao, phm, phr);
		break;
	case HPI_OSTREAM_HOSTBUFFER_ALLOC:
		OutStreamHostBufferAllocate(pao, phm, phr);
		break;
	case HPI_OSTREAM_HOSTBUFFER_GET_INFO:
		OutStreamHostBufferGetInfo(pao, phm, phr);
		break;
	case HPI_OSTREAM_HOSTBUFFER_FREE:
		OutStreamHostBufferFree(pao, phm, phr);
		break;
	case HPI_OSTREAM_START:
		OutStreamStart(pao, phm, phr);
		break;
	case HPI_OSTREAM_OPEN:
		OutStreamOpen(pao, phm, phr);
		break;
	case HPI_OSTREAM_RESET:
		OutStreamReset(pao, phm, phr);
		break;
#ifndef HPI_BUILD_NO_STREAM_WAIT
	case HPI_OSTREAM_WAIT:
		OutStreamWait(pao, phm, phr);
		break;
#endif
	default:
		HwMessage(pao, phm, phr);
		break;
	}
}

/*****************************************************************************/
/* InStream Host buffer functions */

/** Allocate or attach buffer for busmastering
*/
static void InStreamHostBufferAllocate(
	struct hpi_adapter_obj *pao,
	struct hpi_message *phm,
	struct hpi_response *phr
)
{
	//hpi_err_t err = 0;
	//uint32_t dwCommand = phm->u.d.u.buffer.dwCommand;
	struct hpi_hw_obj_67 *phw = pao->priv;
	struct bus_master_interface_67 *interface = phw->pInterfaceBuffer;
	struct hpi_hw_stream_obj_67 *pstream;
	struct hpi_hostbuffer_status *phb_status;

	pstream = &phw->InStreams[phm->wObjIndex];
	phb_status = &interface->aInStreamHostBufferStatus[phm->wObjIndex];

	StreamHostBufferAllocate(pao, phm, phr, pstream, phb_status);
}

static void InStreamHostBufferGetInfo(
	struct hpi_adapter_obj *pao,
	struct hpi_message *phm,
	struct hpi_response *phr
)
{
	struct hpi_hw_obj_67 *phw = pao->priv;
	struct bus_master_interface_67 *interface = phw->pInterfaceBuffer;
	struct hpi_hw_stream_obj_67 *pstream;
	struct hpi_hostbuffer_status *phb_status;

	pstream = &phw->InStreams[phm->wObjIndex];
	phb_status = &interface->aInStreamHostBufferStatus[phm->wObjIndex];

	StreamHostBufferGetInfo(pao, phm, phr, pstream, phb_status);
}

static void InStreamHostBufferFree(
	struct hpi_adapter_obj *pao,
	struct hpi_message *phm,
	struct hpi_response *phr
)
{
	struct hpi_hw_obj_67 *phw = pao->priv;
	struct hpi_hw_stream_obj_67 *pstream;

	pstream = &phw->InStreams[phm->wObjIndex];

	StreamHostBufferFree(pao, phm, phr, pstream);
}

static uint32_t InStreamGetBytesAvailable(
	struct hpi_hostbuffer_status *status
)
{
	return status->dwDspIndex - status->dwHostIndex;
}

#ifndef HPI_BUILD_NO_STREAM_WAIT
static uint32_t InStreamThresholdReached(
	struct hpi_hostbuffer_status *status,
	uint32_t threshold
)
{
	return InStreamGetBytesAvailable(status) >= threshold;
}
#endif

static void InStreamRead(
	struct hpi_adapter_obj *pao,
	struct hpi_message *phm,
	struct hpi_response *phr
)
{
	struct hpi_hw_obj_67 *phw = pao->priv;
	struct bus_master_interface_67 *interface = phw->pInterfaceBuffer;
	struct hpi_hw_stream_obj_67 *pstream;
	struct hpi_hostbuffer_status *phb_status;
	uint32_t dwDataAvailable;
	uint8_t *pBbmData;
	uint32_t lFirstRead;
	uint8_t *pAppData = (uint8_t *)phm->u.d.u.data.pbData;

	pstream = &phw->InStreams[phm->wObjIndex];
	phb_status = &interface->aInStreamHostBufferStatus[phm->wObjIndex];

	HPI_InitResponse(phr, phm->wObject, phm->wFunction, 0);

	if (!pstream->HostBufferActive) {
		/* there is no BBM buffer, fail */
		phr->wError = HPI_ERROR_HOST_BUFFER_NOT_ALLOCATED;
		return;
	}

	dwDataAvailable = InStreamGetBytesAvailable(phb_status);
	if (dwDataAvailable < phm->u.d.u.data.dwDataSize) {
		phr->wError = HPI_ERROR_INVALID_DATASIZE;
		return;
	}

	pBbmData = (uint8_t *)pstream->HostBufferVirt;

	/* either all data,
	   or enough to fit from current to end of BBM buffer */
	lFirstRead = min(phm->u.d.u.data.dwDataSize,
		phb_status->dwSizeInBytes -
		(phb_status->dwHostIndex & (phb_status->dwSizeInBytes - 1)));

	memcpy(pAppData, pBbmData +
		(phb_status->dwHostIndex & (phb_status->dwSizeInBytes
		- 1)), lFirstRead);

	/* remaining data if any */
	memcpy(pAppData + lFirstRead,
		pBbmData, phm->u.d.u.data.dwDataSize - lFirstRead);

	phb_status->dwHostIndex += phm->u.d.u.data.dwDataSize;
}

static void InStreamGetInfo(
	struct hpi_adapter_obj *pao,
	struct hpi_message *phm,
	struct hpi_response *phr
)
{
	struct hpi_hw_obj_67 *phw = pao->priv;
	struct bus_master_interface_67 *interface = phw->pInterfaceBuffer;
	struct hpi_hw_stream_obj_67 *pstream;
	struct hpi_hostbuffer_status *phb_status;

	pstream = &phw->InStreams[phm->wObjIndex];
	phb_status = &interface->aInStreamHostBufferStatus[phm->wObjIndex];

	HPI_InitResponse(phr, phm->wObject, phm->wFunction, 0);

	if (!pstream->HostBufferActive) {
		/* there is no BBM buffer, fail */
		phr->wError = HPI_ERROR_HOST_BUFFER_NOT_ALLOCATED;
		return;
	}

	HPI_InitResponse(phr, phm->wObject, phm->wFunction, 0);

	phr->u.d.u.stream_info.wState = (uint16_t)phb_status->dwStreamState;
	phr->u.d.u.stream_info.dwSamplesTransferred =
		phb_status->dwSamplesProcessed;
	phr->u.d.u.stream_info.dwBufferSize = phb_status->dwSizeInBytes;
	phr->u.d.u.stream_info.dwDataAvailable =
		InStreamGetBytesAvailable(phb_status);
	phr->u.d.u.stream_info.dwAuxiliaryDataAvailable =
		phb_status->dwAuxiliaryDataAvailable;
}

static void InStreamStart(
	struct hpi_adapter_obj *pao,
	struct hpi_message *phm,
	struct hpi_response *phr
)
{
	HwMessage(pao, phm, phr);
	HPI_DEBUG_LOG0(DEBUG, "InStreamStart()\n");
}

static void InStreamReset(
	struct hpi_adapter_obj *pao,
	struct hpi_message *phm,
	struct hpi_response *phr
)
{
#ifndef HPI_BUILD_NO_STREAM_WAIT
	struct hpi_hw_obj_67 *phw = pao->priv;

	HpiOs_Event_Set(&phw->InStreams[phm->wObjIndex].Event);
#endif

	HwMessage(pao, phm, phr);
}

#ifndef HPI_BUILD_NO_STREAM_WAIT
static void InStreamWait(
	struct hpi_adapter_obj *pao,
	struct hpi_message *phm,
	struct hpi_response *phr
)
{
	struct hpi_hw_obj_67 *phw = pao->priv;
	struct bus_master_interface_67 *interface = phw->pInterfaceBuffer;
	struct hpi_hw_stream_obj_67 *pstream;
	struct hpi_hostbuffer_status *phb_status;

	pstream = &phw->InStreams[phm->wObjIndex];
	phb_status = &interface->aInStreamHostBufferStatus[phm->wObjIndex];

	HPI_InitResponse(phr, phm->wObject, phm->wFunction, 0);

	if (!pstream->HostBufferActive) {
		/* there is no BBM buffer, fail */
		phr->wError = HPI_ERROR_HOST_BUFFER_NOT_ALLOCATED;
		return;
	}

	pstream->Threshold = phm->u.d.u.dwThresholdBytes;
	if (!InStreamThresholdReached(phb_status,
		pstream->Threshold)){
			HpiOs_Event_Wait(&pstream->Event);
			HpiOs_Event_Clear(&pstream->Event);
			if (!InStreamThresholdReached(phb_status,
				pstream->Threshold)){
					phr->wError = HPI_ERROR_WAIT_CANCELLED;
			}
	}
	pstream->Threshold = 0;
}
#endif

static void IStreamMessage(
	struct hpi_adapter_obj *pao,
	struct hpi_message *phm,
	struct hpi_response *phr
)
{

	if (phm->wObjIndex >= HPI_MAX_STREAMS) {
		phr->wError = HPI_ERROR_INVALID_OBJ_INDEX;
		HPI_DEBUG_LOG2(WARNING,
			"Message referencing invalid stream %d on adapter index %d\n",
			phm->wObjIndex, phm->wAdapterIndex);
		return;
	}
	switch (phm->wFunction) {
	case HPI_ISTREAM_READ:
		InStreamRead(pao, phm, phr);
		break;
	case HPI_ISTREAM_GET_INFO:
		InStreamGetInfo(pao, phm, phr);
		break;
	case HPI_ISTREAM_HOSTBUFFER_ALLOC:
		InStreamHostBufferAllocate(pao, phm, phr);
		break;
	case HPI_ISTREAM_HOSTBUFFER_GET_INFO:
		InStreamHostBufferGetInfo(pao, phm, phr);
		break;
	case HPI_ISTREAM_HOSTBUFFER_FREE:
		InStreamHostBufferFree(pao, phm, phr);
		break;
	case HPI_ISTREAM_START:
		InStreamStart(pao, phm, phr);
		break;
	case HPI_ISTREAM_RESET:
		InStreamReset(pao, phm, phr);
		break;
#ifndef HPI_BUILD_NO_STREAM_WAIT
	case HPI_ISTREAM_WAIT:
		InStreamWait(pao, phm, phr);
		break;
#endif
	default:
		HwMessage(pao, phm, phr);
		break;
	}
}

/*****************************************************************************/
/** Entry point to this HPI backend
 * All calls to the HPI start here
 */
static void HPI_6700(
	struct hpi_adapter_obj *pao,
	struct hpi_message *phm,
	struct hpi_response *phr
)
{
	if ((pao->wDspCrashed >= 10) &&
		(phm->wFunction != HPI_ADAPTER_DEBUG_READ)){
			/* allow last resort debug read even after crash */
		HPI_InitResponse(phr, phm->wObject, phm->wFunction,
			HPI_ERROR_DSP_HARDWARE);
		HPI_DEBUG_LOG2(WARNING, " %d, %d dsp crashed.\n",
			phm->wObject, phm->wFunction);
		return;
	}

	/* Init default response  */
	phr->wError = HPI_ERROR_PROCESSING_MESSAGE;

	switch (phm->wType) {
	case HPI_TYPE_REQUEST:
		switch (phm->wObject) {
		case HPI_OBJ_SUBSYSTEM:
			phr->wError = HPI_ERROR_INVALID_FUNC;
			break;

		case HPI_OBJ_ADAPTER:
			AdapterMessage(pao, phm, phr);
			break;

		case HPI_OBJ_CONTROL:
			ControlMessage(pao, phm, phr);
			break;

		case HPI_OBJ_OSTREAM:
			OStreamMessage(pao, phm, phr);
			break;

		case HPI_OBJ_ISTREAM:
			IStreamMessage(pao, phm, phr);
			break;

		default:
			HwMessage(pao, phm, phr);
			break;
		}
		break;

	default:
		phr->wError = HPI_ERROR_INVALID_TYPE;
		break;
	}
}

static void write_app_reg(struct hpi_hw_obj_67 *phw,
			  uint32_t reg_offset, uint32_t value)
{
	uint32_t idx = reg_offset / sizeof(value);

	HPIOS_MEMWRITE32(&phw->bar[0].host_address[idx], value);
}

static uint32_t read_app_reg(struct hpi_hw_obj_67 *phw, uint32_t reg_offset)
{
	uint32_t value;
	uint32_t idx = reg_offset / sizeof(value);

	value = HPIOS_MEMREAD32(&phw->bar[0].host_address[idx]);
	return value;
}

static void write_dev_struct(struct hpi_hw_obj_67 *phw,
			     struct mapped_device_struct *mds,
			     uint32_t reg_offset, uint32_t value)
{
	uint32_t idx = (reg_offset +
			   (mds->device_address -
			    phw->bar[mds->bar].mapped_device_address)
			  ) / sizeof(value);
	HPIOS_MEMWRITE32(&phw->bar[mds->bar].host_address[idx], value);
}

static uint32_t read_dev_struct(struct hpi_hw_obj_67 *phw,
				struct mapped_device_struct *mds,
				uint32_t reg_offset)
{
	uint32_t value;
	uint32_t idx = (reg_offset +
			(mds->device_address -
			 phw->bar[mds->bar].mapped_device_address)
			) / sizeof(value);
	value = HPIOS_MEMREAD32(&phw->bar[mds->bar].host_address[idx]);
	return value;
}

static int setup_bar(struct hpi_adapter_obj *pao,
		     unsigned int bar_num, unsigned int addr)
{
	struct hpi_hw_obj_67 *phw = pao->priv;
        uint32_t ib_num;
#ifdef HPI_OS_LINUX_KERNEL
        resource_size_t bar_val;

	bar_val = pci_resource_start(pao->os.pci_dev, bar_num);
#else
        uint32_t bar_val;

	bar_val = (uint32_t)
		pao->os.wdm_bus_info.Pci.MemoryInfo[bar_num].Physical.QuadPart;
	//HpiOs_GetUnmappedBARAddress(bar_num, &, &bar_val);
#endif
        ib_num = bar_num - 1;
	/* Turn the IB section off */
	write_app_reg(phw, C6700_IB_BAR(ib_num), 0);

	write_app_reg(phw, C6700_IB_START_LO(ib_num), bar_val);
#ifdef HPI_OS_DELETE
	if (mode_64bit == 1)
		write_app_reg(phw, C6700_IB_START_HI(ib_num),
				(u32)(bar_val >> 16 >> 16));
	else if (mode_64bit == 0)
#endif
	write_app_reg(phw, C6700_IB_START_HI(ib_num), 0);
	write_app_reg(phw, C6700_IB_OFFSET(ib_num), addr);

	/* Turn the IB section on */
	write_app_reg(phw, C6700_IB_BAR(ib_num), bar_num);

	phw->bar[bar_num].mapped_device_address = addr;

        return 0;
}

static void map_dev_struct(struct hpi_adapter_obj *pao,
			   struct mapped_device_struct *mds,
			   uint32_t dev_addr, uint32_t struct_size, int bar)
{
	struct hpi_hw_obj_67 *phw = pao->priv;
	uint32_t mask;
	uint32_t dev_base;

	mask = phw->bar[bar].size - 1;
	dev_base = dev_addr & ~mask;

	mds->bar = bar;
	mds->size = struct_size;
	mds->device_address = dev_addr;

	setup_bar(pao, bar, dev_base);
}

/** If the device structure is larger than the BAR window
 * we may need to remap to access the whole range.
 */
static void remap_bar(struct hpi_adapter_obj *pao,
		      struct mapped_device_struct *mds,
		      uint32_t dev_addr)
{
	struct hpi_hw_obj_67 *phw = pao->priv;
	int bar = mds->bar;
	uint32_t mask;
	uint32_t new_base;

	mask = phw->bar[bar].size-1;
	new_base = dev_addr & ~mask;

	setup_bar(pao,bar,new_base);
}

/*****************************************************************************/
/** delete an adapter - required by WDM driver */
static void AdapterDelete(
	struct hpi_adapter_obj *pao,
	struct hpi_message *phm,
	struct hpi_response *phr
)
{
	// struct hpi_hw_obj_67 *phw;

	if (!pao) {
		phr->wError = HPI_ERROR_INVALID_OBJ_INDEX;
		return;
	}

	AdapterShutdown(pao);
	phr->wError = 0;
}

/*****************************************************************************/
/* INITIALIZATION */
#ifdef HPI_OS_LINUX_KERNEL

static irqreturn_t MessageIrqHandler(int irq, void *dev_id)
{
	struct hpi_adapter_obj *pao = dev_id;
	struct hpi_hw_obj_67 *phw = pao->priv;
	struct bus_master_interface_67 *iface = phw->pInterfaceBuffer;
	unsigned int stream_irqs;
	unsigned int msg_irqs;

#ifndef HPI6700_POLL_DSP_ACK
	msg_irqs = iface->dwDspAckCount - phw->dwMessageInterruptCount;
	if (msg_irqs) {
		if (msg_irqs != 1)
			pr_err("Missed %d message IRQs?\n", msg_irqs - 1);
		phw->dwMessageInterruptCount = iface->dwDspAckCount;
		HpiOs_Event_Set(&phw->dsp_ack_event);
	}
#else
	msg_irqs = 1;
#endif

	stream_irqs = iface->dwDSPStreamInterruptCount - phw->dwStreamInterruptCount;
	if (stream_irqs) {
		if (stream_irqs != 1)
			pr_warn("hpi6700 adapter %d missed %d stream IRQs?\n",
			        pao->index, stream_irqs - 1);
		phw->dwStreamInterruptCount = iface->dwDSPStreamInterruptCount;
		return IRQ_WAKE_THREAD;
	}

	if (msg_irqs)
		return IRQ_HANDLED;

	pr_err("hpi6700 adapter %d interface IRQ source counters unchanged\n",
	       pao->index);
	return IRQ_NONE;
}

static irqreturn_t StreamThreadedIrqHandler(int irq, void *dev_id)
{
	struct hpi_adapter_obj *pao = dev_id;
	struct hpi_hw_obj_67 *phw = pao->priv;

	HPI_DEBUG_LOG2(VERBOSE, "STREAM %d IRQ %d\n", phw->dwMessageInterruptCount, irq);
	if (pao->os.irq_thread)
		pao->os.irq_thread(pao);

	return IRQ_HANDLED;
}

/** Linux interrupt setup */
static int SetupInterrupts(struct hpi_adapter_obj *pao)
{
	/* MSI initialization */
	struct pci_dev *pci_dev = pao->os.pci_dev;
	struct device *dev = &pci_dev->dev;
	int result;
#ifdef HPI6700_POLL_DSP_ACK
	struct hpi_hw_obj_67 *phw = pao->priv;

	// Disable HPI messaging interrupts
	phw->pInterfaceBuffer->dwFlags = BMI67_FLAGS_NOMESSAGEIRQ;
#endif

	result = pci_enable_msi(pci_dev);
	// dev_info(dev, "pci_enable_msi returned %d\n", result);
	if (result < 0)
		return result;

	// dev_info(dev, "dev->irq is now %d\n", pci_dev->irq);

	result = devm_request_threaded_irq(dev, pci_dev->irq,
					MessageIrqHandler,
					StreamThreadedIrqHandler,
					0, dev_name(dev), pao);
	if (result < 0) {
		dev_info(dev, "devm_request_threaded_irq error %d\n", result);
		return result;
	}

	pao->os.irq = pci_dev->irq;
	return 0;
}

static void FreeInterrupts(struct hpi_adapter_obj *pao)
{
	/* IRQ and MSI will be disabled by pcim_release */
}

/* Map first 3 PCI memory resources
 * Also set DMA masks to 32 bit
 */
static hpi_err_t SetupBars(struct hpi_adapter_obj *pao)
{
	struct pci_dev *pci_dev = pao->os.pci_dev;
	struct hpi_hw_obj_67 *phw = pao->priv;
	int i;

	i = pci_set_dma_mask(pci_dev, DMA_BIT_MASK(32));
	if (!i)
		i = pci_set_consistent_dma_mask(pci_dev, DMA_BIT_MASK(32));
	if (i) {
		dev_err(&pci_dev->dev, "DMA set 32 bit mask error %d\n", i);
		return HPI_ERROR_MEMORY_ALLOC;
	}

	for (i = 0; i < 3; i++) {
		phw->bar[i].host_address = pcim_iomap(pci_dev, i, 0);
		if (!phw->bar[i].host_address) {
			dev_err(&pci_dev->dev, "Failed to map BAR %d\n", i);
			return HPI_ERROR_MEMORY_ALLOC;
		}
		phw->bar[i].size = pci_resource_len(pci_dev, i);
	}

	return 0;
}

#elif (defined HPI_OS_WDM) || (defined HPI_OS_OSX)
static hpi_err_t SetupInterrupts(struct hpi_adapter_obj *pao)
{
	return 0;
}

static void FreeInterrupts(struct hpi_adapter_obj *pao)
{
}

static hpi_err_t SetupBars(struct hpi_adapter_obj *pao)
{
	int i;
	struct hpi_hw_obj_67 *phw = pao->priv;

	for (i = 0; i < HPI_MAX_ADAPTER_MEM_SPACES; i++) {
		phw->bar[i].host_address = (uint32_t *)pao->os.wdm_bus_info.Pci.MemoryInfo[i].Virtual;
		phw->bar[i].size = pao->os.wdm_bus_info.Pci.MemoryInfo[i].Length;
	}
	return 0;
}
#endif

int HPI6700_adapter_init(struct hpi_adapter_obj *pao);

static const struct hpi_endpoint HPI6700_endpoint = {
	C99(.name =)			"HPI6700",
	C99(.adapter_init =)		HPI6700_adapter_init,
	C99(.shutdown =)		AdapterShutdown,
	C99(.irq_query_and_clear =)	AdapterIrqQueryAndClear,
	C99(.transact =)		HPI_6700
};

static hpi_err_t get_adapter_info( struct hpi_adapter_obj *pao, struct hpi_hw_obj_67 *phw)
{
	struct hpi_message hm;
	struct hpi_response hr;

	HPI_InitMessageResponse(&hm, &hr, HPI_OBJ_ADAPTER, HPI_ADAPTER_GET_INFO);
	HwMessage(pao, &hm, &hr);
	if (!hr.wError){
		pao->type = hr.u.ax.info.wAdapterType;
		pao->index = hr.u.ax.info.wAdapterIndex;
		pao->instreams = (uint8_t)hr.u.ax.info.wNumIStreams;
		pao->outstreams = (uint8_t)hr.u.ax.info.wNumOStreams;

		HPI_DEBUG_LOG3(VERBOSE,
			"Got adapter info Type %x Index %d Serial %d\n",
			hr.u.ax.info.wAdapterType,
			hr.u.ax.info.wAdapterIndex,
			hr.u.ax.info.dwSerialNumber);
	}

	return hr.wError;
}

static hpi_err_t get_channel_counts( struct hpi_adapter_obj *pao, struct hpi_hw_obj_67 *phw)
{
	struct hpi_message hm;
	struct hpi_response hr;

	HPI_InitMessageResponse(&hm, &hr, HPI_OBJ_ADAPTER, HPI_ADAPTER_GET_PROPERTY);
	hm.wAdapterIndex = pao->index;
	hm.u.ax.property_set.wProperty = HPI_ADAPTER_PROPERTY_CURCHANNELS;

	HwMessage(pao, &hm, &hr);
	if (!hr.wError) {
		phw->props.chans_per_player =
			hr.u.ax.property_get.wParameter2;
		phw->props.chans_per_recorder =
			hr.u.ax.property_get.wParameter1;
	} else {
		phw->props.chans_per_player = 2;
		phw->props.chans_per_recorder = 2;
		HPI_DEBUG_LOG1(ERROR,
			"error %d reading HPI_ADAPTER_PROPERTY_CURCHANNELS\n",
			hr.wError);
	}
	return hr.wError;
}

static hpi_err_t get_max_samplerate( struct hpi_adapter_obj *pao, struct hpi_hw_obj_67 *phw)
{
	struct hpi_message hm;
	struct hpi_response hr;
	uint16_t wControlIndex;
	uint32_t dwSamplerateIndex;

	phw->props.max_sample_rate = 48000;

	HPI_InitMessageResponse(&hm, &hr, HPI_OBJ_ADAPTER, HPI_ADAPTER_OPEN);
	hm.wAdapterIndex = pao->index;

	HwMessage(pao, &hm, &hr);
	if (hr.wError) {
		HPI_DEBUG_LOG1(ERROR,"error %d opening adapter\n",hr.wError);
		return hr.wError;
	}

	HPI_InitMessageResponse(&hm, &hr, HPI_OBJ_MIXER, HPI_MIXER_OPEN);
	hm.wAdapterIndex = pao->index;

	HwMessage(pao, &hm, &hr);
	if (hr.wError) {
		HPI_DEBUG_LOG1(ERROR,"error %d opening mixer\n",hr.wError);
		return hr.wError;
	}

	HPI_InitMessageResponse(&hm, &hr, HPI_OBJ_MIXER, HPI_MIXER_GET_CONTROL);
	hm.wAdapterIndex = pao->index;
	hm.u.m.wNodeType1 = HPI_SOURCENODE_CLOCK_SOURCE;
	hm.u.m.wNodeIndex1 = 0;
	hm.u.m.wNodeType2 = 0;
	hm.u.m.wNodeIndex2 = 0;
	hm.u.m.wControlType = HPI_CONTROL_SAMPLECLOCK;

	HwMessage(pao, &hm, &hr);
	if (hr.wError) {
		HPI_DEBUG_LOG1(ERROR,"error %d getting sampleclock control\n",hr.wError);
		return hr.wError;
	}
	wControlIndex = hr.u.m.wControlIndex;

	for(dwSamplerateIndex = 0;hr.wError==0; dwSamplerateIndex++){
		HPI_InitMessageResponse(&hm, &hr, HPI_OBJ_CONTROL, HPI_CONTROL_GET_INFO);
		hm.wAdapterIndex = pao->index;
		hm.wObjIndex = wControlIndex;
		hm.u.c.wAttribute = HPI_SAMPLECLOCK_LOCAL_SAMPLERATE;
		hm.u.c.dwParam1 = dwSamplerateIndex;
		hm.u.c.dwParam2 = 0;

		HwMessage(pao, &hm, &hr);
		if ( !hr.wError &&
			( !dwSamplerateIndex ||
				hr.u.c.dwParam1 > phw->props.max_sample_rate
			)
		)
		{
			phw->props.max_sample_rate = hr.u.c.dwParam1;
		}
	}
	return (dwSamplerateIndex != 0)?0:hr.wError;
}

/** Initialize adapter
 * allocate buffers, bootload DSPs, initialise control cache
 * expects pao to have initialized:
 * linux: os.pci_dev
 * other: os.apMemBase, ?
 */
static int _HPI6700_adapter_init(struct hpi_adapter_obj *pao)
{
	struct hpi_hw_obj_67 *phw;
	struct bus_master_interface_67 *interface;
#if HIF_BUFFER_ALIGNMENT
	uint32_t alignment_offset = 0;
	uint32_t misaligned_by;
#endif
	int i;
	int err;

	pao->ep = HPI6700_endpoint;

	phw = HpiOs_MemAllocZero(sizeof(*phw));
	if (!phw)
		return HPI_ERROR_MEMORY_ALLOC;
	pao->priv = phw;

	err = SetupBars(pao);
	if (err)
		return err;

	pao->wDspCrashed = 0;
	pao->wHasControlCache = 0;
	HpiOs_SpinLock_Init(&pao->dspLock);

	for (i = 0; i < HPI_MAX_STREAMS; i++){
		phw->flagOStreamJustReset[i] = 1;
#ifndef HPI_BUILD_NO_STREAM_WAIT
		HpiOs_Event_Init(&phw->OutStreams[i].Event);
		HpiOs_Event_Init(&phw->InStreams[i].Event);
#endif
	}

	phw->pInterfaceBuffer = HpiOs_LockedMem_Alloc(&phw->hLockedMem,
			H670_HIF_BUFFER_SIZE + HIF_BUFFER_ALIGNMENT, &pao->os,
			&phw->dwInterfaceBufferPhysAddr);
	if (!phw->pInterfaceBuffer)
		return HPI_ERROR_MEMORY_ALLOC;

	HPI_DEBUG_LOG3(DEBUG,
		"Interface buffer (unaligned) virtual addr: %p, phys addr: %x, size: %lu\n",
		phw->pInterfaceBuffer, phw->dwInterfaceBufferPhysAddr,
		(unsigned long)H670_HIF_BUFFER_SIZE);

#if HIF_BUFFER_ALIGNMENT
	misaligned_by = phw->dwInterfaceBufferPhysAddr % HIF_BUFFER_ALIGNMENT;
	if (misaligned_by)
		alignment_offset = HIF_BUFFER_ALIGNMENT - misaligned_by;

	HPI_DEBUG_LOG1(DEBUG,
		"Interface buffer alignment offset: %d\n",
		alignment_offset);

	phw->dwInterfaceBufferPhysAddr += alignment_offset;
	phw->pInterfaceBuffer = (struct bus_master_interface_67 *)(
		(char *)phw->pInterfaceBuffer + alignment_offset);

#endif
	interface = phw->pInterfaceBuffer;
	memset(interface, 0, sizeof(*interface));
	interface->dwMagic = FW6700_BUS_MASTER_INTERFACE_MAGIC_V1;

	err = SetupInterrupts(pao);
	if (err) {
		HPI_DEBUG_LOG0(ERROR, "IRQ setup failed\n");
		return err;
	}

	HPI_DEBUG_LOG2(DEBUG, "Interface buffer address %p, physical address %x\n",
		phw->pInterfaceBuffer, phw->dwInterfaceBufferPhysAddr);

	err = AdapterBootLoadDsp(pao);
	if (err) {
		HPI_DEBUG_LOG0(ERROR, "DSP code load failed\n");
		// no need to clean up as SubSysCreateAdapter
		// calls DeleteAdapter on error.
		return err;
	}
	HPI_DEBUG_LOG0(DEBUG, "Bootload DSP code OK\n");

	pao->type = 0x6714;
	pao->index = 0;

	/* Note that *pao, *phw are zeroed after allocation,
	 * so pointers and flags are NULL by default.
	 * Allocate bus mastering control cache buffer and tell the DSP about it
	 */
	if (interface->aControlCache.dwNumberOfControls) {
		uint32_t misaligned_by;
		uint32_t alignment_offset;
		uint8_t *pControlCacheVirtual;
		uint8_t *pControlCacheLastByte;

		// next 16-byte boundary past end of interface buffer
		misaligned_by = (phw->dwInterfaceBufferPhysAddr +
			sizeof(struct bus_master_interface_67)) % LINE_ALIGNMENT;
		if (misaligned_by)
			alignment_offset = LINE_ALIGNMENT - misaligned_by;
		else
			alignment_offset = 0;

		pControlCacheVirtual = (uint8_t *)phw->pInterfaceBuffer +
			sizeof(struct bus_master_interface_67) +
			alignment_offset;

		// verify that it fits
		pControlCacheLastByte = pControlCacheVirtual +
			interface->aControlCache.dwSizeInBytes - 1;

		if (HpiOs_LockedMem_ValidAddr(&phw->hLockedMem, pControlCacheLastByte))
		{
			phw->pCache = HpiAllocControlCache(
				interface->aControlCache.dwNumberOfControls,
				interface->aControlCache.dwSizeInBytes,
				pControlCacheVirtual);
			if (phw->pCache){
				pao->wHasControlCache = 1;
				interface->aControlCache.dwPhysicalAddress32 =
					phw->dwInterfaceBufferPhysAddr +
					sizeof(struct bus_master_interface_67) +
					alignment_offset;
				HPI_DEBUG_LOG3(DEBUG,
					"Successfully allocated cache for %d controls %d bytes at %x\n",
					interface->aControlCache.dwNumberOfControls,
					interface->aControlCache.dwSizeInBytes,
					interface->aControlCache.dwPhysicalAddress32);
			} else {
				HPI_DEBUG_LOG0(ERROR,
					"HpiAllocControlCache() failed\n");
			}
		} else {
			HPI_DEBUG_LOG1(ERROR,
				"Allocated memory doesn't have room for %d byte control cache!\n",
				interface->aControlCache.dwSizeInBytes);
			return HPI_ERROR_MEMORY_ALLOC;
		}
	}

	{
#ifndef HPI6700_POLL_DSP_ACK
		HpiOs_Event_Init(&phw->dsp_ack_event);
#endif
		err = get_adapter_info(pao, phw);
		if(err)
			return err;

		if ((pao->type & 0xfff0) == 0x6790) {
			phw->props.max_sample_rate = 48000;
			phw->props.chans_per_player = 64;
			phw->props.chans_per_recorder = 64;
		} else {
			get_channel_counts(pao, phw);
			get_max_samplerate(pao, phw);
		}

		err = AllocStreamMem(pao);
		if(err)
			return err;
	}

	if (phw->pCache)
		phw->pCache->adap_idx = pao->index;


	pao->aInStreamHostBufferStatus = interface->aInStreamHostBufferStatus;
	pao->aOutStreamHostBufferStatus = interface->aOutStreamHostBufferStatus;
	
	HPI_DEBUG_LOG0(DEBUG, "Adapter Init OK\n");
	HPI_DEBUG_LOG0(DEBUG, ".\n");

	return 0;
}

int HPI6700_adapter_init(struct hpi_adapter_obj *pao)
{
	int err = _HPI6700_adapter_init(pao);

	if (err)
		AdapterShutdown(pao);
	return err;
}

static hpi_err_t GetMaxStreamBandwidth(
	struct hpi_adapter_obj *pao,
	uint32_t *pdwMaxOStreamBW,
	uint32_t *pdwMaxIStreamBW
)
{
	/** Hardcoded for now for the ASI6791
	 * it would be preferable to determine this
	 * from adapter properties
	 */
	struct hpi_hw_obj_67 *phw = pao->priv;

	// Max Chans Per Stream * Max Bytes Per Sample * Max Samplerate
	*pdwMaxOStreamBW =
		phw->props.chans_per_player *
		4 *
		phw->props.max_sample_rate;

	*pdwMaxIStreamBW =
		phw->props.chans_per_recorder *
		4 *
		phw->props.max_sample_rate;

	return 0;
}

static hpi_err_t AllocStreamMem(struct hpi_adapter_obj *pao)
{
	hpi_err_t err;
	int i;
	struct hpi_hw_obj_67 *phw = pao->priv;
	uint32_t dwMaxOStreamBW;
	uint32_t dwOStreamBufferSize;
	uint32_t dwMaxIStreamBW;
	uint32_t dwIStreamBufferSize;
	uint32_t dwTotalSize;
	uint8_t *pVirtAddr;
	uint32_t dwPhysAddr;
	uint32_t dwOffset = 0;

	err = GetMaxStreamBandwidth(pao, &dwMaxOStreamBW, &dwMaxIStreamBW);
	if (err) {
		HPI_DEBUG_LOG1(ERROR,
			"GetMaxStreamBandwidth() returned error: %d\n",
			err);
		return err;
	}

	// buffer size = banwidth / 10 increased to be a power of 2
	// this provides a minimum of 100 milliseconds
	dwOStreamBufferSize = roundup_pow_of_two(dwMaxOStreamBW / 10);
	dwIStreamBufferSize = roundup_pow_of_two(dwMaxIStreamBW / 10);

	dwTotalSize = dwOStreamBufferSize * pao->outstreams +
		dwIStreamBufferSize * pao->instreams +
		STREAM_BUFFER_ALIGNMENT;

	pVirtAddr = HpiOs_LockedMem_Alloc(&phw->hStreamMem,
			dwTotalSize, &pao->os, &dwPhysAddr);
	if (!pVirtAddr) {
		HPI_DEBUG_LOG0(ERROR,
			"HpiOs_LockedMem_Alloc() failed to get memory for stream buffers\n");
		// note failed alloc already sets handle to invalid value
		return HPI_ERROR_MEMORY_ALLOC;
	}

	HPI_DEBUG_LOG3(DEBUG,
		"Stream buffer (unaligned) virtual addr: %p, phys addr: %x, size: %d\n",
		pVirtAddr, dwPhysAddr, dwTotalSize);

#if STREAM_BUFFER_ALIGNMENT
	if (dwPhysAddr % STREAM_BUFFER_ALIGNMENT)
		dwOffset = STREAM_BUFFER_ALIGNMENT - dwPhysAddr % STREAM_BUFFER_ALIGNMENT;
#endif

	for (i = 0; i < pao->outstreams; i++)
	{
		phw->OutStreams[i].HostBufferVirt = pVirtAddr + dwOffset;
		phw->OutStreams[i].HostBufferPhys = dwPhysAddr + dwOffset;
		phw->OutStreams[i].HostBufferSize = dwOStreamBufferSize;
		dwOffset += dwOStreamBufferSize;
	}

	for (i = 0; i < pao->instreams; i++)
	{
		phw->InStreams[i].HostBufferVirt = pVirtAddr + dwOffset;
		phw->InStreams[i].HostBufferPhys = dwPhysAddr + dwOffset;
		phw->InStreams[i].HostBufferSize = dwIStreamBufferSize;
		dwOffset += dwIStreamBufferSize;
	}

	return err;
}

static hpi_err_t FreeStreamMem(
	struct hpi_adapter_obj *pao
)
{
	struct hpi_hw_obj_67 *phw = pao->priv;
	uint32_t i = 0;

	HpiOs_LockedMem_Free(&phw->hStreamMem);

	for (i = 0; i < HPI_MAX_STREAMS; i++)
	{
		phw->OutStreams[i].HostBufferVirt = NULL;
		phw->OutStreams[i].HostBufferPhys = 0;
		phw->OutStreams[i].HostBufferSize = 0;
		phw->OutStreams[i].HostBufferActive = 0;
	}
	for (i = 0; i < HPI_MAX_STREAMS; i++)
	{
		phw->InStreams[i].HostBufferVirt = NULL;
		phw->InStreams[i].HostBufferPhys = 0;
		phw->InStreams[i].HostBufferSize = 0;
		phw->InStreams[i].HostBufferActive = 0;
	}
	return 0;
}

/** Reset DSPs and free memory areas allocated for adapter
*/
static void AdapterShutdown(struct hpi_adapter_obj *pao)
{
	struct hpi_hw_obj_67 *phw = pao->priv;
	int timeout = UBOOT_STATE_MAX_RETRIES;
	uint32_t uboot_state;

	uboot_state = read_dev_struct(phw, &phw->uboot_info,
		C6700_INFO_STATE_OFFSET);
	HPI_DEBUG_LOG1(DEBUG,
		"AdapterShutdown: Setting u-boot state to FW6700_S_HOSTPROTORESTART (prev state=%d)\n", uboot_state);
	write_dev_struct(phw, &phw->uboot_info,
			C6700_INFO_STATE_OFFSET, FW6700_S_HOSTPROTORESTART);
	HPI_DEBUG_LOG0(DEBUG,
		"AdapterShutdown: waiting for u-boot state to be FW6700_S_EPREADY\n");
	do {
		if (timeout < UBOOT_STATE_MAX_RETRIES)
			HpiOs_DelayMicroSeconds(100);

		uboot_state = read_dev_struct(phw, &phw->uboot_info,
					C6700_INFO_STATE_OFFSET);
	} while (uboot_state != FW6700_S_EPREADY && --timeout);
	if (uboot_state != FW6700_S_EPREADY){
		HPI_DEBUG_LOG1(ERROR,
			"AdapterShutdown: Timeout waiting for FW6700_S_EPREADY  (state=%d)\n", uboot_state);
	}
	else{
		HPI_DEBUG_LOG0(DEBUG,
			"AdapterShutdown: u-boot state is FW6700_S_EPREADY\n");
	}

	HpiFreeControlCache(phw->pCache);
	HpiOs_LockedMem_Free(&phw->hLockedMem);
	FreeStreamMem(pao);
	FreeInterrupts(pao);
	HpiOs_MemFree(phw);
	HPI_DEBUG_LOG0(DEBUG, "AdapterShutdown: finished\n");
	HPI_DEBUG_LOG0(DEBUG, ".\n");
}

static uint32_t __iomem *Dev2Host(
	const struct bar_status *bar,
	const uint32_t device_address
)
{
	if ((device_address >= bar->mapped_device_address) &&
	    (device_address - bar->mapped_device_address < bar->size))
	{
		uint32_t idx = (device_address - bar->mapped_device_address) / sizeof(uint32_t);
		return &bar->host_address[idx];
	}

	return NULL;
}

static uint32_t BootLoader_ReadMem32(
	struct hpi_adapter_obj *pao,
	uint32_t dwAddress
)
{
	struct hpi_hw_obj_67 *phw = pao->priv;
	uint32_t dwResult;
	uint32_t __iomem *pHostAddr;

	pHostAddr = Dev2Host(&phw->bar[phw->soc_phymem.bar], dwAddress);

	if (!pHostAddr) {
		remap_bar(pao, &phw->soc_phymem, dwAddress);
		pHostAddr = Dev2Host(&phw->bar[phw->soc_phymem.bar], dwAddress);
	}

	HPI_DEBUG_ASSERT(pHostAddr);

	dwResult = HPIOS_MEMREAD32(pHostAddr);

	return dwResult;
}

static void BootLoader_WriteMem32(
	struct hpi_adapter_obj *pao,
	uint32_t dwAddress,
	uint32_t dwData
)
{
	struct hpi_hw_obj_67 *phw = pao->priv;
	uint32_t __iomem *pHostAddr;

	pHostAddr = Dev2Host(&phw->bar[phw->soc_phymem.bar], dwAddress);

	if (!pHostAddr) {
		remap_bar(pao, &phw->soc_phymem, dwAddress);
		pHostAddr = Dev2Host(&phw->bar[phw->soc_phymem.bar], dwAddress);
	}

	HPI_DEBUG_ASSERT(pHostAddr);

	HPIOS_MEMWRITE32(pHostAddr, dwData);
}

/* Currently, Linux builds define HPI6700_POLL_DSP_ACK and windows builds do not.
 *
 * Under linux the BMI67_FLAGS_NOMESSAGEIRQ flag is set in the interface structure
 * so the DSP does not signal the interrupt after completing a message and the #else
 * implementation of WaitDspAck below polls dwDspAckCount to see when the message
 * processing is done.
 *
 * In windows the message processing is also setup to avoid blocking but in this case
 * the BMI67_FLAGS_NOMESSAGEIRQ flag is not set so the DSP does signal the interrupt
 * when the message processing is complete.  The IRQ handler sets dsp_ack_event
 * which is polled along with dwDspAckCount.  Either condition will signal the
 * completion of message processing.
 */

#ifndef HPI6700_POLL_DSP_ACK
/* Wait for acknowledge from DSP
 * return 0 for success, or an HPI error number
 */
static hpi_err_t WaitDspAck(
	struct hpi_hw_obj_67 *phw,
	int *interrupt_timeout_count,
	int *count_mismatch_count
)
{
	int result = 0;
	int event_was_set = 0;
	volatile uint32_t *pdwDspAck = &phw->pInterfaceBuffer->dwDspAckCount;
	int timeout_us = HPI6700_TIMEOUT;
	hpi_err_t err = 0;

	do{
		if(!event_was_set && HpiOs_Event_IsSet(&phw->dsp_ack_event))
			event_was_set = 1;
		if(*pdwDspAck==phw->pInterfaceBuffer->dwHostCmdCount)
			break;
		HpiOs_DelayMicroSeconds(HPI6700_WAIT_LOOP_DELAY);
		timeout_us -= HPI6700_WAIT_LOOP_DELAY;
	}while(timeout_us > 0);

	if (timeout_us <= 0)
	{
		*interrupt_timeout_count += 1;
		err = HPI_ERROR_WAIT_TIMEOUT;
	}

	if (!err && (*pdwDspAck!=phw->pInterfaceBuffer->dwHostCmdCount))
	{
		*count_mismatch_count += 1;
		err = HPI_ERROR_DSP_COMMUNICATION;
	}

	return err;
}
#else

/* Wait for acknowledge from DSP by polling
 * return 0 for success, or an HPI error number
 */
static hpi_err_t WaitDspAck(
	struct hpi_hw_obj_67 *phw,
	int *interrupt_timeout_count,
	int *count_mismatch_count
)
{
	hpi_err_t err;
	uint32_t *pdwDspAck = &phw->pInterfaceBuffer->dwDspAckCount;
	int timeout = 1000;

	while (phw->pInterfaceBuffer->dwHostCmdCount != *pdwDspAck && --timeout)
		HpiOs_DelayMicroSeconds(50);

	if (timeout) {
		err = 0;
	} else {
		*interrupt_timeout_count += 1;
		err = HPI_ERROR_WAIT_TIMEOUT;
	}
	return err;
}
#endif

static void SendDspInterrupt(
	struct hpi_hw_obj_67 *phw,
	uint32_t val
)
{
	// interrupt the DSP
	write_app_reg(phw, C6700_BAR0_MSI_IRQ, val);
}

#ifndef HPI_OS_LINUX_KERNEL
static hpi_err_t TriggerDspAndWait(struct hpi_hw_obj_67 *phw)
{
	hpi_err_t err = HPI_ERROR_DSP_COMMUNICATION;
	int retries;
	int microseconds;
	int timeout_counter = 0;
	int msg_resp_mismatch_counter = 0;
	static const int max_retries = 3;
	uint32_t old_interrupt_count = phw->dwMessageInterruptCount;
	LARGE_INTEGER before;
	LARGE_INTEGER after;
	LARGE_INTEGER qpf;

	phw->pInterfaceBuffer->dwHostCmdCount++;

	for (retries = 0; retries < max_retries; retries++){
		HpiOs_Event_Clear(&phw->dsp_ack_event);
		before = KeQueryPerformanceCounter(&qpf);
		SendDspInterrupt(phw, 0);
		err = WaitDspAck(phw, &timeout_counter, &msg_resp_mismatch_counter);
		if ( !err ) { // success
			after = KeQueryPerformanceCounter(NULL);
			break;
		}
	}
	if (timeout_counter || msg_resp_mismatch_counter) {
		// Got interrupt, so failure condition is that dwHostCmd != dwDspAck
		HPI_DEBUG_LOG5(ERROR,
			"MsgResp, cmd_count %d, fn 0x%x, COUNTS timeout %d, no resp %d, retries %d\n",
			phw->pInterfaceBuffer->dwHostCmdCount,
			phw->pInterfaceBuffer->msg.MessageBuffer.message.wFunction,
			timeout_counter,
			msg_resp_mismatch_counter,
			retries);
	}
	microseconds = (int)((after.QuadPart - before.QuadPart)*1000000/qpf.QuadPart);
	if (microseconds > 20000)
		HPI_DEBUG_LOG1(ERROR, "Message took %d microseconds.\n", microseconds);
	else
		HPI_DEBUG_LOG1(DEBUG, "Message took %d microseconds.\n", microseconds);
	return err;
}
#else
static hpi_err_t TriggerDspAndWait(struct hpi_hw_obj_67 *phw)
{
	hpi_err_t err = HPI_ERROR_DSP_COMMUNICATION;
	int retries;
	int microseconds;
	int timeout_counter = 0;
	int msg_resp_mismatch_counter = 0;
	static const int max_retries = 3;
	ktime_t before, after;
	u64 elapsed_ns;

	phw->pInterfaceBuffer->dwHostCmdCount++;
	WMB(); /* DSP reads */

	before = ktime_get();
	for (retries = 0; retries < max_retries; retries++){
		HpiOs_Event_Clear(&phw->dsp_ack_event);
		SendDspInterrupt(phw, 0);

		err = WaitDspAck(phw, &timeout_counter, &msg_resp_mismatch_counter);

		if (!err)
			break;
	}
	if (timeout_counter || msg_resp_mismatch_counter) {
		HPI_DEBUG_LOG5(ERROR,
			"MsgResp, cmd_count %d, fn 0x%x, COUNTS timeout %d, no resp %d, retries %d\n",
			phw->pInterfaceBuffer->dwHostCmdCount,
			phw->pInterfaceBuffer->msg.MessageBuffer.message.wFunction,
			timeout_counter,
			msg_resp_mismatch_counter,
			retries);
	}
	after = ktime_get();
	elapsed_ns = ktime_to_ns(ktime_sub(after, before));
	microseconds = do_div(elapsed_ns, 1000);
	if (microseconds > 20000)
		HPI_DEBUG_LOG1(ERROR, "DSP transaction took %d microseconds.\n", microseconds);
	else
		HPI_DEBUG_LOG1(VERBOSE, "DSP transaction took %d microseconds.\n", microseconds);
	return err;
}
#endif

static hpi_err_t AdapterBootLoadDsp(struct hpi_adapter_obj *pao)
{
	struct hpi_hw_obj_67 *phw = pao->priv;
	struct dsp_code DspCode;
	uint16_t aBootCodeId;
	uint32_t i;
	uint32_t info_struct_location;
	uint32_t soc_phymem_location;
	uint32_t soc_phymem_size;
	hpi_err_t err=0;
	int timeout = UBOOT_STATE_MAX_RETRIES;
	uint32_t uboot_state;

	// First map uboot info structure
	phw->dbg.GPR0_first = read_app_reg(phw, C6700_BAR0_GPR0);
	phw->dbg.GPR2 = read_app_reg(phw, C6700_BAR0_GPR2);
	phw->dbg.GPR3 = read_app_reg(phw, C6700_BAR0_GPR3);
	info_struct_location = phw->dbg.GPR0_first;
	map_dev_struct(pao, &phw->uboot_info, info_struct_location,
		       sizeof(struct fw6700_info_v0), 1);
	HPI_DEBUG_LOG1(DEBUG,
		"Set bar 1 to fw6700_info_v0 struct (GPR0 address %8.8X)\n",
		info_struct_location);

	// Read magic value
	phw->dbg.info_magic_val = read_dev_struct(phw, &phw->uboot_info, C6700_INFO_MAGIC_OFFSET);
	HPI_DEBUG_LOG1(DEBUG, "Magic = %x\n", phw->dbg.info_magic_val);

	/* Detect early if the bootloader is suspended and trigger a PCIE boot protocol restart */
	uboot_state = read_dev_struct(phw, &phw->uboot_info, C6700_INFO_STATE_OFFSET);
	if (uboot_state == FW6700_S_EPIDLE || uboot_state == FW6700_S_EPHALTED) {
		HPI_DEBUG_LOG0(DEBUG, "Uboot idle or halted, set state to FW6700_S_HOSTPROTORESTART\n");
		write_dev_struct(phw, &phw->uboot_info, C6700_INFO_STATE_OFFSET, FW6700_S_HOSTPROTORESTART);
	}

	HPI_DEBUG_LOG0(DEBUG, "Wait for Uboot state to be FW6700_S_EPREADY\n");
	timeout = UBOOT_STATE_MAX_RETRIES;
	do {
		if (timeout < UBOOT_STATE_MAX_RETRIES)
			HpiOs_DelayMicroSeconds(100);
		uboot_state = read_dev_struct(phw, &phw->uboot_info, C6700_INFO_STATE_OFFSET);
	} while(uboot_state != FW6700_S_EPREADY && --timeout);

	if (uboot_state != FW6700_S_EPREADY){
		HPI_DEBUG_LOG0(ERROR, "Timeout waiting for FW6700_S_EPREADY; attempting restart\n");
		HPI_DEBUG_LOG0(DEBUG, "Set Uboot state to FW6700_S_HOSTPROTORESTART\n");
		write_dev_struct(phw, &phw->uboot_info, C6700_INFO_STATE_OFFSET, FW6700_S_HOSTPROTORESTART);
		HPI_DEBUG_LOG0(DEBUG, "Wait for Uboot state to be FW6700_S_EPREADY\n");
		timeout = UBOOT_STATE_MAX_RETRIES;
		do {
			if (timeout < UBOOT_STATE_MAX_RETRIES)
				HpiOs_DelayMicroSeconds(100);
			uboot_state = read_dev_struct(phw, &phw->uboot_info, C6700_INFO_STATE_OFFSET);
		} while(uboot_state != FW6700_S_EPREADY && --timeout);
	}

	if (uboot_state != FW6700_S_EPREADY){
		HPI_DEBUG_LOG0(ERROR, "Timeout waiting for FW6700_S_EPREADY\n");
		return HPI_ERROR_DSP_BOOTLOAD;
	}
	HPI_DEBUG_LOG0(DEBUG, "Uboot state is now FW6700_S_EPREADY\n");
	
	write_dev_struct(phw, &phw->uboot_info, C6700_INFO_DRVINFOSIZE_OFFSET,
			sizeof(struct fw6700_dsp_info_v0));
	write_dev_struct(phw, &phw->uboot_info, C6700_INFO_STATE_OFFSET, FW6700_S_HOSTREADY);
	HPI_DEBUG_LOG0(DEBUG, "Set Uboot state to FW6700_S_HOSTREADY\n");

	HPI_DEBUG_LOG0(DEBUG, "Wait for Uboot state to be FW6700_S_EPDONE\n");
	timeout = UBOOT_STATE_MAX_RETRIES;
	do {
		if (timeout < UBOOT_STATE_MAX_RETRIES)
			HpiOs_DelayMicroSeconds(100);
		uboot_state = read_dev_struct(phw, &phw->uboot_info, C6700_INFO_STATE_OFFSET);
	} while(uboot_state != FW6700_S_EPDONE && --timeout);

	if (uboot_state != FW6700_S_EPDONE){
		HPI_DEBUG_LOG0(ERROR, "Timeout waiting for FW6700_S_EPDONE\n");
		return HPI_ERROR_DSP_BOOTLOAD;
	}
	HPI_DEBUG_LOG0(DEBUG, "Uboot state is now FW6700_S_EPDONE\n");

	// map dsp info structure
	info_struct_location = read_dev_struct(phw, &phw->uboot_info, C6700_INFO_DRVINFOADDR_OFFSET);
	map_dev_struct(pao, &phw->dsp_info, info_struct_location, sizeof(struct fw6700_dsp_info_v0), 2);
	HPI_DEBUG_LOG1(DEBUG, "Set bar 2 to fw6700_dsp_info_v0 struct (address %8.8X)\n", info_struct_location);

	for (i = 0; i < sizeof(struct fw6700_dsp_info_v0); i+= sizeof(uint32_t))
		write_dev_struct(phw, &phw->dsp_info, i, 0);

	write_dev_struct(phw, &phw->dsp_info, C6700_DSP_INFO_MAGIC_OFFSET, FW6700_DSP_INFO_MAGIC_V0);
	write_dev_struct(phw, &phw->dsp_info, C6700_DSP_INFO_INTERFACEBUFFER_OFFSET,
			phw->dwInterfaceBufferPhysAddr);
	write_dev_struct(phw, &phw->dsp_info, C6700_DSP_INFO_INTERFACEBUFFERSIZE_OFFSET,
			sizeof(struct bus_master_interface_67));

	soc_phymem_location = read_dev_struct(phw, &phw->uboot_info, C6700_INFO_SOCPHYMEMSTARTADDR_OFFSET);
	soc_phymem_size = read_dev_struct(phw, &phw->uboot_info, C6700_INFO_SOCPHYMEMSIZE_OFFSET);

	map_dev_struct(pao, &phw->soc_phymem, soc_phymem_location, soc_phymem_size, 2);
	HPI_DEBUG_LOG1(DEBUG, "Set bar 2 to phymem location (address %8.8X)\n", soc_phymem_location);

	switch (HPIOS_PCI_SUBSYS_DEVICE(pao) & 0xfff0) {
	case 0x6790:
		aBootCodeId = HPI_ADAPTER_ASI(0x6790);
		break;
	case 0x6700:
	case 0x6780:
		aBootCodeId = HPI_ADAPTER_ASI(0x6780);
		break;
	default:
		HPI_DEBUG_LOG1(ERROR, "SubSys ID %04x not recognised\n",
			       HPIOS_PCI_SUBSYS_DEVICE(pao));
		return HPI_ERROR_DSP_BOOTLOAD;
	}

	err = HpiDspCode_Open(aBootCodeId, &pao->os, &DspCode);
	if (err)
		return err;

	while (1) {
		uint32_t dwLength;
		uint32_t dwAddress;
		uint32_t dwType;
		uint32_t *pdwCode;

		err = HpiDspCode_ReadWord(&DspCode, &dwLength);
		if (err)
			break;
		if (dwLength == 0xFFFFFFFF)
			break; 	// end of code

		err = HpiDspCode_ReadWord(&DspCode, &dwAddress);
		if (err)
			break;
		err = HpiDspCode_ReadWord(&DspCode, &dwType);
		if (err)
			break;
		err = HpiDspCode_ReadBlock(dwLength, &DspCode,
			&pdwCode);
		if (err)
			break;
		for (i = 0; i < (int)dwLength; i++) {
			BootLoader_WriteMem32(pao, dwAddress, *pdwCode);
			pdwCode++;
			dwAddress += 4;
		}

	}
	if (err) {
		HpiDspCode_Close(&DspCode);
		return err;
	}

	/* verify code */
	HpiDspCode_Rewind(&DspCode);
	while (1) {
		uint32_t dwLength = 0;
		uint32_t dwAddress = 0;
		uint32_t dwType = 0;
		uint32_t *pdwCode = NULL;
		uint32_t dwData = 0;

		HpiDspCode_ReadWord(&DspCode, &dwLength);
		if (dwLength == 0xFFFFFFFF)
			break; 	// end of code

		HpiDspCode_ReadWord(&DspCode, &dwAddress);
		HpiDspCode_ReadWord(&DspCode, &dwType);
		HpiDspCode_ReadBlock(dwLength, &DspCode, &pdwCode);

		for (i = 0; i < (int)dwLength; i++) {
			dwData = BootLoader_ReadMem32(pao, dwAddress);
			if (dwData != *pdwCode) {
				err = HPI_ERROR_DSP_BOOTLOAD;
				break;
			}
			pdwCode++;
			dwAddress += 4;
		}
		if (err)
			break;
	}
	HpiDspCode_Close(&DspCode);

	if (!err){
		uint32_t cmd_status;

		// enable DSP outbound address translation
		cmd_status = read_app_reg(phw, C6700_BAR0_CMD_STATUS);
		cmd_status |= OB_XLT_EN;
		write_app_reg(phw, C6700_BAR0_CMD_STATUS, cmd_status);

		write_dev_struct(phw, &phw->uboot_info, C6700_INFO_ENTRYPOINTADDR_OFFSET, 0x80000000);
		write_dev_struct(phw, &phw->uboot_info, C6700_INFO_HALTARM_OFFSET, 0);
		write_dev_struct(phw, &phw->uboot_info, C6700_INFO_STATE_OFFSET, FW6700_S_HOSTDONE);
		HPI_DEBUG_LOG0(DEBUG, "Set Uboot state to FW6700_S_HOSTDONE\n");

		phw->dbg.GPR0_second = read_app_reg(phw, C6700_BAR0_GPR0);

		map_dev_struct(pao, &phw->dsp_info, info_struct_location, sizeof(struct fw6700_dsp_info_v0), 2);

		timeout = UBOOT_STATE_MAX_RETRIES;
		do {
			if (timeout < UBOOT_STATE_MAX_RETRIES)
				HpiOs_DelayMicroSeconds(1000);
			phw->dbg.DSP_dspinfomagicok =
				read_dev_struct(phw, &phw->dsp_info, C6700_DSP_INFO_DSPINFOMAGICOK_OFFSET);
			phw->dbg.DSP_busmastermagicok =
				read_dev_struct(phw, &phw->dsp_info, C6700_DSP_INFO_BUSMASTERMAGICOK_OFFSET);
			phw->dbg.dspinfo_magic_val =
				read_dev_struct(phw, &phw->dsp_info, C6700_DSP_INFO_MAGIC_OFFSET);
			phw->dbg.dspinfo_dsp_ready =
				read_dev_struct(phw, &phw->dsp_info, C6700_DSP_INFO_DSPREADY_OFFSET);
		} while(phw->dbg.dspinfo_dsp_ready == 0 && timeout--);
		if (phw->dbg.dspinfo_dsp_ready == 0){
			HPI_DEBUG_LOG0(ERROR, "Timeout waiting for DSP ready.\n");
			return HPI_ERROR_DSP_BOOTLOAD;
		} else {
			HPI_DEBUG_LOG3(DEBUG,
				"Got DSP ready! after %d loops. DSPINFOMAGICOK = %d BUSMASTERMAGICOK = %d\n",
				UBOOT_STATE_MAX_RETRIES - timeout + 1,
				phw->dbg.DSP_dspinfomagicok,
				phw->dbg.DSP_busmastermagicok);
		}
		uboot_state = read_dev_struct(phw, &phw->uboot_info, C6700_INFO_STATE_OFFSET);
		HPI_DEBUG_LOG1(DEBUG, "Uboot state is %s\n", fw6700_state_str(uboot_state));
	}
	return err;
}

static hpi_err_t MessageResponseSequence(
	struct hpi_adapter_obj *pao,
	struct hpi_message *phm,
	struct hpi_response *phr
)
{
	hpi_err_t err;
	struct hpi_hw_obj_67 *phw = pao->priv;
	struct bus_master_interface_67 *interface = phw->pInterfaceBuffer;

	if (phm->wSize > sizeof(interface->msg.MessageBuffer)) {
		phr->wError = HPI_ERROR_MESSAGE_BUFFER_TOO_SMALL;
		phr->wSpecificError = sizeof(interface->msg.MessageBuffer);
		phr->wSize = sizeof(struct hpi_response_header);
#ifdef HPI_OS_LINUX_KERNEL
		HPI_DEBUG_LOG2(ERROR, "Message len %d too big for buffer %zd\n",
			phm->wSize, sizeof(interface->msg.MessageBuffer));
#else
		HPI_DEBUG_LOG2(ERROR, "Message len %d too big for buffer %Iu\n",
			phm->wSize, sizeof(interface->msg.MessageBuffer));
#endif
		return 0;
	}

	/* Assume buffer of type struct bus_master_interface_67
	   is allocated "noncacheable" */
	memcpy(&interface->msg.MessageBuffer, phm, phm->wSize);

	WMB(); /* DSP uses DMA to get the request message */

	// signal we want a response
	err = TriggerDspAndWait(phw);

	if (err) {
		HPI_DEBUG_LOG3(ERROR,
			"Error %d waiting for DSP response [%d], FN 0x%x\n",
			err, interface->dwDspAckCount, phm->wFunction);
		return err;
	}

	if (interface->rsp.ResponseBuffer.response.wSize <= phr->wSize)
		memcpy(phr, &interface->rsp.ResponseBuffer,
			interface->rsp.ResponseBuffer.response.wSize);
	else {
		HPI_DEBUG_LOG2(ERROR,
			"Response len %d too big for buffer %d\n",
			interface->rsp.ResponseBuffer.response.wSize,
			phr->wSize);
		memcpy(phr, &interface->rsp.ResponseBuffer, sizeof(struct hpi_response_header));
		phr->wError = HPI_ERROR_RESPONSE_BUFFER_TOO_SMALL;
		phr->wSpecificError = interface->rsp.ResponseBuffer.response.wSize;
		phr->wSize = sizeof(struct hpi_response_header);
	}

	return HpiValidateResponse(phm, phr);
}

static void HwMessage(
	struct hpi_adapter_obj *pao,
	struct hpi_message *phm,
	struct hpi_response *phr
)
{
	hpi_err_t err = 0;

	HpiOs_SpinLock_Lock(&pao->dspLock);
	err = MessageResponseSequence(pao, phm, phr);
	HpiOs_SpinLock_Unlock(&pao->dspLock);

	if (err) {
		phr->wError = err;
		pao->wDspCrashed++;
		// just the header of the response is valid
		phr->wSize = sizeof(struct hpi_response_header);
	} else {
		pao->wDspCrashed = 0;
	}
}
