diff --git a/include/zephyr/drivers/usb/udc_buf.h b/include/zephyr/drivers/usb/udc_buf.h
index aae4cf28fba..69b85da94db 100644
--- a/include/zephyr/drivers/usb/udc_buf.h
+++ b/include/zephyr/drivers/usb/udc_buf.h
@@ -132,15 +132,17 @@ extern const struct net_buf_data_cb net_buf_dma_cb;
  */
 #define UDC_BUF_POOL_DEFINE(pname, count, size, ud_size, fdestroy)		\
 	_NET_BUF_ARRAY_DEFINE(pname, count, ud_size);				\
+	BUILD_ASSERT((UDC_BUF_GRANULARITY) % (UDC_BUF_ALIGN) == 0,		\
+		     "Code assumes granurality is multiple of alignment");	\
 	static uint8_t __nocache __aligned(UDC_BUF_ALIGN)			\
-		net_buf_data_##pname[count][size];				\
+		net_buf_data_##pname[count][ROUND_UP(size, UDC_BUF_GRANULARITY)];\
 	static const struct net_buf_pool_fixed net_buf_fixed_##pname = {	\
 		.data_pool = (uint8_t *)net_buf_data_##pname,			\
 	};									\
 	static const struct net_buf_data_alloc net_buf_fixed_alloc_##pname = {	\
 		.cb = &net_buf_fixed_cb,					\
 		.alloc_data = (void *)&net_buf_fixed_##pname,			\
-		.max_alloc_size = size,						\
+		.max_alloc_size = ROUND_UP(size, UDC_BUF_GRANULARITY),		\
 	};									\
 	static STRUCT_SECTION_ITERABLE(net_buf_pool, pname) =			\
 		NET_BUF_POOL_INITIALIZER(pname, &net_buf_fixed_alloc_##pname,	\