path: root/sys/dev/qat/qat_api/common/include/lac_mem.h

/* SPDX-License-Identifier: BSD-3-Clause */
/* Copyright(c) 2007-2025 Intel Corporation */
/**
 ***************************************************************************
 * @file lac_mem.h
 *
 * @defgroup LacMem     Memory
 *
 * @ingroup LacCommon
 *
 * Memory re-sizing functions and memory accessor macros.
 *
 ***************************************************************************/

#ifndef LAC_MEM_H
#define LAC_MEM_H

/***************************************************************************
 * Include header files
 ***************************************************************************/
#include "cpa.h"
#include "qat_utils.h"
#include "lac_common.h"

/**
 *******************************************************************************
 * @ingroup LacMem
 *      These macros are used to Endian swap variables from IA to QAT.
 *
 * @param[out] x    The variable to be swapped.
 *
 * @retval none
 ******************************************************************************/
#if (BYTE_ORDER == LITTLE_ENDIAN)
#define LAC_MEM_WR_64(x) QAT_UTILS_HOST_TO_NW_64(x)
#define LAC_MEM_WR_32(x) QAT_UTILS_HOST_TO_NW_32(x)
#define LAC_MEM_WR_16(x) QAT_UTILS_HOST_TO_NW_16(x)
#define LAC_MEM_RD_64(x) QAT_UTILS_NW_TO_HOST_64(x)
#define LAC_MEM_RD_32(x) QAT_UTILS_NW_TO_HOST_32(x)
#define LAC_MEM_RD_16(x) QAT_UTILS_NW_TO_HOST_16(x)
#else
#define LAC_MEM_WR_64(x) (x)
#define LAC_MEM_WR_32(x) (x)
#define LAC_MEM_WR_16(x) (x)
#define LAC_MEM_RD_64(x) (x)
#define LAC_MEM_RD_32(x) (x)
#define LAC_MEM_RD_16(x) (x)
#endif

/*
*******************************************************************************
* Shared Memory Macros (memory accessible by Acceleration Engines, e.g. QAT)
*******************************************************************************
*/

/**
 *******************************************************************************
 * @ingroup LacMem
 *      This macro can be used to write to a variable that will be read by the
 * QAT. The macro will automatically detect the size of the target variable and
 * will select the correct method for performing the write. The data is cast to
 * the type of the field that it will be written to.
 * This macro swaps data if required.
 *
 * @param[out] var    The variable to be written. Can be a field of a struct.
 *
 * @param[in] data    The value to be written.  Will be cast to the size of the
 *                    target.
 *
 * @retval none
 ******************************************************************************/
#define LAC_MEM_SHARED_WRITE_SWAP(var, data)                                   \
	do {                                                                   \
		switch (sizeof(var)) {                                         \
		case 1:                                                        \
			(var) = (Cpa8U)(data);                                 \
			break;                                                 \
		case 2:                                                        \
			(var) = (Cpa16U)(data);                                \
			(var) = LAC_MEM_WR_16(((Cpa16U)var));                  \
			break;                                                 \
		case 4:                                                        \
			(var) = (Cpa32U)(data);                                \
			(var) = LAC_MEM_WR_32(((Cpa32U)var));                  \
			break;                                                 \
		case 8:                                                        \
			(var) = (Cpa64U)(data);                                \
			(var) = LAC_MEM_WR_64(((Cpa64U)var));                  \
			break;                                                 \
		default:                                                       \
			break;                                                 \
		}                                                              \
	} while (0)

/**
 *******************************************************************************
 * @ingroup LacMem
 *      This macro can be used to read a variable that was written by the QAT.
 * The macro will automatically detect the size of the data to be read and will
 * select the correct method for performing the read. The value read from the
 * variable is cast to the size of the data type it will be stored in.
 * This macro swaps data if required.
 *
 * @param[in] var     The variable to be read. Can be a field of a struct.
 *
 * @param[out] data   The variable to hold the result of the read. Data read
 *                    will be cast to the size of this variable
 *
 * @retval none
 ******************************************************************************/
#define LAC_MEM_SHARED_READ_SWAP(var, data)                                    \
	do {                                                                   \
		switch (sizeof(var)) {                                         \
		case 1:                                                        \
			(data) = (var);                                        \
			break;                                                 \
		case 2:                                                        \
			(data) = LAC_MEM_RD_16(((Cpa16U)var));                 \
			break;                                                 \
		case 4:                                                        \
			(data) = LAC_MEM_RD_32(((Cpa32U)var));                 \
			break;                                                 \
		case 8:                                                        \
			(data) = LAC_MEM_RD_64(((Cpa64U)var));                 \
			break;                                                 \
		default:                                                       \
			break;                                                 \
		}                                                              \
	} while (0)

/**
 *******************************************************************************
 * @ingroup LacMem
 *      This macro can be used to write a pointer to a QAT request. The fields
 *      for pointers in the QAT request and response messages are always 64 bits
 *
 * @param[out] var    The variable to be written to. Can be a field of a struct.
 *
 * @param[in] data    The value to be written.  Will be cast to size of target
 *                    variable
 *
 * @retval none
 ******************************************************************************/
/* cast pointer to scalar of same size of the native pointer */
#define LAC_MEM_SHARED_WRITE_FROM_PTR(var, data)                               \
	((var) = (Cpa64U)(LAC_ARCH_UINT)(data))

/* Note: any changes to this macro implementation should also be made to the
 * similar LAC_MEM_CAST_PTR_TO_UINT64 macro
 */

/**
 *******************************************************************************
 * @ingroup LacMem
 *      This macro can be used to read a pointer from a QAT response. The fields
 *      for pointers in the QAT request and response messages are always 64 bits
 *
 * @param[in] var     The variable to be read. Can be a field of a struct.
 *
 * @param[out] data   The variable to hold the result of the read. Data read
 *                    will be cast to the size of this variable
 *
 * @retval none
 ******************************************************************************/
/* Cast back to native pointer */
#define LAC_MEM_SHARED_READ_TO_PTR(var, data)                                  \
	((data) = (void *)(LAC_ARCH_UINT)(var))

/**
 *******************************************************************************
 * @ingroup LacMem
 *      This macro safely casts a pointer to a Cpa64U type.
 *
 * @param[in] pPtr   The pointer to be cast.
 *
 * @retval pointer cast to Cpa64U
 ******************************************************************************/
#define LAC_MEM_CAST_PTR_TO_UINT64(pPtr) ((Cpa64U)(pPtr))

/**
 *******************************************************************************
 * @ingroup LacMem
 *      This macro uses an QAT Utils macro to convert from a virtual address to
 *a
 *      physical address for internally allocated memory.
 *
 * @param[in] pVirtAddr   The address to be converted.
 *
 * @retval The converted physical address
 ******************************************************************************/
#define LAC_OS_VIRT_TO_PHYS_INTERNAL(pVirtAddr)                                \
	(QAT_UTILS_MMU_VIRT_TO_PHYS(pVirtAddr))

/**
 *******************************************************************************
 * @ingroup LacMem
 *      This macro should be called on all externally allocated memory it calls
 *      SalMem_virt2PhysExternal function which allows a user
 *      to set the virt2phys function used by an instance.
 *      Defaults to virt to phys for kernel.
 *
 * @param[in] genService  Generic sal_service_t structure.
 * @param[in] pVirtAddr   The address to be converted.
 *
 * @retval The converted physical address
 ******************************************************************************/
#define LAC_OS_VIRT_TO_PHYS_EXTERNAL(genService, pVirtAddr)                    \
	((SalMem_virt2PhysExternal(pVirtAddr, &(genService))))

/**
 *******************************************************************************
 * @ingroup LacMem
 *      This macro can be used to write an address variable that will be read by
 * the QAT.  The macro will perform the necessary virt2phys address translation
 * This macro is only to be called on memory allocated internally by the driver.
 *
 * @param[out] var  The address variable to write. Can be a field of a struct.
 *
 * @param[in] pPtr  The pointer variable to containing the address to be
 *                  written
 *
 * @retval none
 ******************************************************************************/
#define LAC_MEM_SHARED_WRITE_VIRT_TO_PHYS_PTR_INTERNAL(var, pPtr)              \
	do {                                                                   \
		Cpa64U physAddr = 0;                                           \
		physAddr = LAC_MEM_CAST_PTR_TO_UINT64(                         \
		    LAC_OS_VIRT_TO_PHYS_INTERNAL(pPtr));                       \
		var = physAddr;                                                \
	} while (0)

/**
 *******************************************************************************
 * @ingroup LacMem
 *      This macro can be used to write an address variable that will be read by
 * the QAT.  The macro will perform the necessary virt2phys address translation
 * This macro is to be used on memory allocated externally by the user. It calls
 * the user supplied virt2phys address translation.
 *
 * @param[in] pService The pointer to the service
 * @param[out] var     The address variable to write. Can be a field of a struct
 * @param[in] pPtr     The pointer variable to containing the address to be
 *                     written
 *
 * @retval none
 ******************************************************************************/
#define LAC_MEM_SHARED_WRITE_VIRT_TO_PHYS_PTR_EXTERNAL(pService, var, pPtr)    \
	do {                                                                   \
		Cpa64U physAddr = 0;                                           \
		physAddr = LAC_MEM_CAST_PTR_TO_UINT64(                         \
		    LAC_OS_VIRT_TO_PHYS_EXTERNAL(pService, pPtr));             \
		var = physAddr;                                                \
	} while (0)

/*
*******************************************************************************
* OS Memory Macros
*******************************************************************************
*/

/**
 *******************************************************************************
 * @ingroup LacMem
 *      This function and associated macro allocates the memory for the given
 *      size and stores the address of the memory allocated in the pointer.
 *
 * @param[out] ppMemAddr    address of pointer where address will be stored
 * @param[in] sizeBytes     the size of the memory to be allocated.
 *
 * @retval CPA_STATUS_RESOURCE  Macro failed to allocate Memory
 * @retval CPA_STATUS_SUCCESS   Macro executed successfully
 *
 ******************************************************************************/
static __inline CpaStatus
LacMem_OsMemAlloc(void **ppMemAddr, Cpa32U sizeBytes)
{
	*ppMemAddr = malloc(sizeBytes, M_QAT, M_WAITOK);

	return CPA_STATUS_SUCCESS;
}

/**
 *******************************************************************************
 * @ingroup LacMem
 *      This function and associated macro allocates the contiguous
 *       memory for the given
 *      size and stores the address of the memory allocated in the pointer.
 *
 * @param[out] ppMemAddr     address of pointer where address will be stored
 * @param[in] sizeBytes      the size of the memory to be allocated.
 * @param[in] alignmentBytes the alignment
 * @param[in] node           node to allocate from
 *
 * @retval CPA_STATUS_RESOURCE  Macro failed to allocate Memory
 * @retval CPA_STATUS_SUCCESS   Macro executed successfully
 *
 ******************************************************************************/
static __inline CpaStatus
LacMem_OsContigAlignMemAlloc(void **ppMemAddr,
			     Cpa32U sizeBytes,
			     Cpa32U alignmentBytes,
			     Cpa32U node)
{
	if ((alignmentBytes & (alignmentBytes - 1)) !=
	    0) /* if is not power of 2 */
	{
		*ppMemAddr = NULL;
		QAT_UTILS_LOG("alignmentBytes MUST be the power of 2\n");
		return CPA_STATUS_INVALID_PARAM;
	}

	*ppMemAddr =
	    qatUtilsMemAllocContiguousNUMA(sizeBytes, node, alignmentBytes);

	if (NULL == *ppMemAddr) {
		return CPA_STATUS_RESOURCE;
	}

	return CPA_STATUS_SUCCESS;
}

/**
 *******************************************************************************
 * @ingroup LacMem
 *      Macro from the malloc() function
 *
 ******************************************************************************/
#define LAC_OS_MALLOC(sizeBytes) malloc(sizeBytes, M_QAT, M_WAITOK)

/**
 *******************************************************************************
 * @ingroup LacMem
 *      Macro from the LacMem_OsContigAlignMemAlloc function
 *
 ******************************************************************************/
#define LAC_OS_CAMALLOC(ppMemAddr, sizeBytes, alignmentBytes, node)            \
	LacMem_OsContigAlignMemAlloc((void *)ppMemAddr,                        \
				     sizeBytes,                                \
				     alignmentBytes,                           \
				     node)

/**
 *******************************************************************************
 * @ingroup LacMem
 *      Macro for declaration static const unsigned int constant. One provides
 *   the compilation time computation with the highest bit set in the
 *   sizeof(TYPE) value. The constant is being put by the linker by default in
 *   .rodata section
 *
 *   E.g. Statement LAC_DECLARE_HIGHEST_BIT_OF(lac_mem_blk_t)
 *   results in following entry:
 *     static const unsigned int highest_bit_of_lac_mem_blk_t = 3
 *
 *   CAUTION!
 *      Macro is prepared only for type names NOT-containing ANY
 *  special characters. Types as amongst others:
 *  - void *
 *  - unsigned long
 *  - unsigned int
 *  are strictly forbidden and will result in compilation error.
 *  Use typedef to provide one-word type name for MACRO's usage.
 ******************************************************************************/
#define LAC_DECLARE_HIGHEST_BIT_OF(TYPE)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       \
	static const unsigned int highest_bit_of_##TYPE =                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      \
	    (sizeof(TYPE) & 0x80000000 ? 31 : (sizeof(TYPE) & 0x40000000 ? 30 : (sizeof(TYPE) & 0x20000000 ? 29 : (                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            \
														      sizeof(TYPE) & 0x10000000 ? 28 : (                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       \
																			   sizeof(TYPE) & 0x08000000 ? 27 : (                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  \
																								sizeof(TYPE) & 0x04000000 ? 26 : (                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             \
																												     sizeof(TYPE) & 0x02000000 ? 25 : (                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        \
																																	  sizeof(TYPE) & 0x01000000 ? 24 : (                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   \
																																					       sizeof(TYPE) & 0x00800000 ?                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     \
																																						   23 :                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        \
																																						   (sizeof(TYPE) & 0x00400000 ? 22 : (                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         \
																																											 sizeof(                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               \
																																											     TYPE) &                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           \
																																												 0x00200000 ?                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  \
																																											     21 :                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              \
																																											     (                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 \
																																												 sizeof(TYPE) & 0x00100000 ? 20 : (sizeof(TYPE) & 0x00080000 ? 19 : (                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          \
																																																					sizeof(                                                                                                                                                                                                                                                                                                                                                                                                                                                                                \
																																																					    TYPE) &                                                                                                                                                                                                                                                                                                                                                                                                                                                                            \
																																																						0x00040000 ?                                                                                                                                                                                                                                                                                                                                                                                                                                                                   \
																																																					    18 :                                                                                                                                                                                                                                                                                                                                                                                                                                                                               \
																																																					    (                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  \
																																																						sizeof(TYPE) & 0x00020000 ? 17 : (                                                                                                                                                                                                                                                                                                                                                                                                                                             \
																																																										     sizeof(TYPE) & 0x00010000 ? 16 : (sizeof(TYPE) &                                                                                                                                                                                                                                                                                                                                                                                          \
																																																															       0x00008000 ?                                                                                                                                                                                                                                                                                                                                                                                    \
																																																															   15 :                                                                                                                                                                                                                                                                                                                                                                                                \
																																																															   (sizeof(TYPE) & 0x00004000 ? 14 : (                                                                                                                                                                                                                                                                                                                                                                 \
																																																																				 sizeof(TYPE) & 0x00002000 ? 13 :                                                                                                                                                                                                                                                                                                                              \
																																																																							     (                                                                                                                                                                                                                                                                                                                                 \
																																																																								 sizeof(TYPE) & 0x00001000 ? 12 : (                                                                                                                                                                                                                                                                                            \
																																																																												      sizeof(TYPE) & 0x00000800 ? 11 : (                                                                                                                                                                                                                                                       \
																																																																																	   sizeof(TYPE) & 0x00000400 ? 10 :                                                                                                                                                                                                                    \
																																																																																				       (                                                                                                                                                                                                                       \
																																																																																					   sizeof(TYPE) &                                                                                                                                                                                                      \
																																																																																						   0x00000200 ?                                                                                                                                                                                                \
																																																																																					       9 :                                                                                                                                                                                                             \
																																																																																					       (sizeof(                                                                                                                                                                                                        \
																																																																																						    TYPE) &                                                                                                                                                                                                    \
																																																																																							0x00000100 ?                                                                                                                                                                                           \
																																																																																						    8 :                                                                                                                                                                                                        \
																																																																																						    (sizeof(TYPE) & 0x00000080 ? 7 :                                                                                                                                                                           \
																																																																																										 (                                                                                                                                                                             \
																																																																																										     sizeof(TYPE) & 0x00000040 ?                                                                                                                                               \
																																																																																											 6 :                                                                                                                                                                   \
																																																																																											 (                                                                                                                                                                     \
																																																																																											     sizeof(TYPE) & 0x00000020 ? 5 :                                                                                                                                   \
																																																																																															 (                                                                                                                                     \
																																																																																															     sizeof(TYPE) & 0x00000010 ? 4 :                                                                                                   \
																																																																																																			 (                                                                                                     \
																																																																																																			     sizeof(TYPE) & 0x00000008 ? 3 :                                                                   \
																																																																																																							 (                                                                     \
																																																																																																							     sizeof(TYPE) & 0x00000004 ? 2 :                                   \
																																																																																																											 (                                     \
																																																																																																											     sizeof(TYPE) & 0x00000002 ? 1 : ( \
																																																																																																																 sizeof(TYPE) & 0x00000001 ? 0 : 32))))))))))))))))) /*16*/))))))))))))))) /* 31 */

/**
 *******************************************************************************
 * @ingroup LacMem
 *      This function and associated macro frees the memory at the given address
 *      and resets the pointer to NULL
 *
 * @param[out] ppMemAddr    address of pointer where mem address is stored.
 *                          If pointer is NULL, the function will exit silently
 *
 * @retval void
 *
 ******************************************************************************/
static __inline void
LacMem_OsMemFree(void **ppMemAddr)
{
	free(*ppMemAddr, M_QAT);
	*ppMemAddr = NULL;
}

/**
 *******************************************************************************
 * @ingroup LacMem
 *      This function and associated macro frees the contiguous memory at the
 *      given address and resets the pointer to NULL
 *
 * @param[out] ppMemAddr    address of pointer where mem address is stored.
 *                          If pointer is NULL, the function will exit silently
 *
 * @retval void
 *
 ******************************************************************************/
static __inline void
LacMem_OsContigAlignMemFree(void **ppMemAddr)
{
	if (NULL != *ppMemAddr) {
		qatUtilsMemFreeNUMA(*ppMemAddr);
		*ppMemAddr = NULL;
	}
}

#define LAC_OS_FREE(pMemAddr) LacMem_OsMemFree((void *)&pMemAddr)

#define LAC_OS_CAFREE(pMemAddr) LacMem_OsContigAlignMemFree((void *)&pMemAddr)

/**
*******************************************************************************
 * @ingroup LacMem
 *     Copies user data to a working buffer of the correct size (required by
 *     PKE services)
 *
 * @description
 *      This function produces a correctly sized working buffer from the input
 *      user buffer. If the original buffer is too small a new buffer shall
 *      be allocated and memory is copied (left padded with zeros to the
*required
 *      length).
 *
 *      The returned working buffer is guaranteed to be of the desired size for
 *      QAT.
 *
 *      When this function is called pInternalMem describes the user_buffer and
 *      when the function returns pInternalMem describes the working buffer.
 *      This is because pInternalMem describes the memory that will be sent to
 *      QAT.
 *
 *      The caller must keep the original buffer pointer. The allocated buffer
*is
 *      freed (as necessary) using icp_LacBufferRestore().
 *
 * @param[in] instanceHandle Handle to crypto instance so pke_resize mem pool
*can
 *                           be located
 * @param[in] pUserBuffer Pointer on the user buffer
 * @param[in] userLen     length of the user buffer
 * @param[in] workingLen  length of the working (correctly sized) buffer
 * @param[in/out] pInternalMem    pointer to boolean if TRUE on input then
 *                                user_buffer is internally allocated memory
 *                                if false then it is externally allocated.
 *                                This value gets updated by the function
 *                                if the returned pointer references internally
 *                                allocated memory.
 *
 * @return a pointer to the working (correctly sized) buffer or NULL if the
 *      allocation failed
 *
 * @note the working length cannot be smaller than the user buffer length
 *
 * @warning the working buffer may be the same or different from the original
 * user buffer; the caller should make no assumptions in this regard
 *
 * @see icp_LacBufferRestore()
 *
 ******************************************************************************/
Cpa8U *icp_LacBufferResize(CpaInstanceHandle instanceHandle,
			   Cpa8U *pUserBuffer,
			   Cpa32U userLen,
			   Cpa32U workingLen,
			   CpaBoolean *pInternalMemory);

/**
*******************************************************************************
 * @ingroup LacMem
 *     Restores a user buffer
 *
 * @description
 *      This function restores a user buffer and releases its
 *      corresponding working buffer. The working buffer, assumed to be
 *      previously obtained using icp_LacBufferResize(), is freed as necessary.
 *
 *      The contents are copied in the process.
 *
 * @note the working length cannot be smaller than the user buffer length
 *
 * @param[out] pUserBuffer     Pointer on the user buffer
 * @param[in] userLen          length of the user buffer
 * @param[in] pWorkingBuffer   Pointer on the working buffer
 * @param[in] workingLen       working buffer length
 * @param[in] copyBuf          if set _TRUE the data in the workingBuffer
 *                             will be copied to the userBuffer before the
 *                             workingBuffer is freed.
 *
 * @return the status of the operation
 *
 * @see icp_LacBufferResize()
 *
 ******************************************************************************/
CpaStatus icp_LacBufferRestore(Cpa8U *pUserBuffer,
			       Cpa32U userLen,
			       Cpa8U *pWorkingBuffer,
			       Cpa32U workingLen,
			       CpaBoolean copyBuf);

/**
*******************************************************************************
 * @ingroup LacMem
 *    Uses an instance specific user supplied virt2phys function to convert a
 *    virtual address to a physical address.
 *
 * @description
 *    Uses an instance specific user supplied virt2phys function to convert a
 *    virtual address to a physical address. A client of QA API can set the
 *    virt2phys function for an instance by using the
 *    cpaXxSetAddressTranslation() function. If the client does not set the
 *    virt2phys function and the instance is in kernel space then OS specific
 *    virt2phys function will be used. In user space the virt2phys function
 *    MUST be set by the user.
 *
 * @param[in] pVirtAddr         the virtual addr to be converted
 * @param[in] pServiceGen       Pointer on the sal_service_t structure
 *                              so client supplied virt2phys function can be
 *                              called.
 *
 * @return the physical address
 *
 ******************************************************************************/
CpaPhysicalAddr SalMem_virt2PhysExternal(void *pVirtAddr, void *pServiceGen);

#endif /* LAC_MEM_H */