This patch series revisits the proposal for a GPU cgroup controller to
track and limit memory allocations by various device/allocator
subsystems. The patch series also contains a simple prototype to
illustrate how Android intends to implement DMA-BUF allocator
attribution using the GPU cgroup controller. The prototype does not
include resource limit enforcements.
Changelog:
v2:
See the previous revision of this change submitted by Hridya Valsaraju
at: https://lore.kernel.org/all/[email protected]/
Move dma-buf cgroup charge transfer from a dma_buf_op defined by every
heap to a single dma-buf function for all heaps per Daniel Vetter and
Christian König. Pointers to struct gpucg and struct gpucg_device
tracking the current associations were added to the dma_buf struct to
achieve this.
Fix incorrect Kconfig help section indentation per Randy Dunlap.
History of the GPU cgroup controller
====================================
The GPU/DRM cgroup controller came into being when a consensus[1]
was reached that the resources it tracked were unsuitable to be integrated
into memcg. Originally, the proposed controller was specific to the DRM
subsystem and was intended to track GEM buffers and GPU-specific
resources[2]. In order to help establish a unified memory accounting model
for all GPU and all related subsystems, Daniel Vetter put forth a
suggestion to move it out of the DRM subsystem so that it can be used by
other DMA-BUF exporters as well[3]. This RFC proposes an interface that
does the same.
[1]: https://patchwork.kernel.org/project/dri-devel/cover/[email protected]/#22624705
[2]: https://lore.kernel.org/amd-gfx/[email protected]/
[3]: https://lore.kernel.org/amd-gfx/YCVOl8%[email protected]/
T.J. Mercier (6):
gpu: rfc: Proposal for a GPU cgroup controller
cgroup: gpu: Add a cgroup controller for allocator attribution of GPU
memory
dmabuf: Use the GPU cgroup charge/uncharge APIs
dmabuf: heaps: export system_heap buffers with GPU cgroup charging
dmabuf: Add gpu cgroup charge transfer function
android: binder: Add a buffer flag to relinquish ownership of fds
Documentation/gpu/rfc/gpu-cgroup.rst | 195 +++++++++++++++++
Documentation/gpu/rfc/index.rst | 4 +
drivers/android/binder.c | 26 +++
drivers/dma-buf/dma-buf.c | 100 +++++++++
drivers/dma-buf/dma-heap.c | 27 +++
drivers/dma-buf/heaps/system_heap.c | 3 +
include/linux/cgroup_gpu.h | 127 +++++++++++
include/linux/cgroup_subsys.h | 4 +
include/linux/dma-buf.h | 22 +-
include/linux/dma-heap.h | 11 +
include/uapi/linux/android/binder.h | 1 +
init/Kconfig | 7 +
kernel/cgroup/Makefile | 1 +
kernel/cgroup/gpu.c | 304 +++++++++++++++++++++++++++
14 files changed, 830 insertions(+), 2 deletions(-)
create mode 100644 Documentation/gpu/rfc/gpu-cgroup.rst
create mode 100644 include/linux/cgroup_gpu.h
create mode 100644 kernel/cgroup/gpu.c
--
2.35.1.265.g69c8d7142f-goog
All DMA heaps now register a new GPU cgroup device upon creation, and the
system_heap now exports buffers associated with its GPU cgroup device for
tracking purposes.
From: Hridya Valsaraju <[email protected]>
Signed-off-by: Hridya Valsaraju <[email protected]>
Co-developed-by: T.J. Mercier <[email protected]>
Signed-off-by: T.J. Mercier <[email protected]>
---
changes in v2
- Move dma-buf cgroup charge transfer from a dma_buf_op defined by every
heap to a single dma-buf function for all heaps per Daniel Vetter and
Christian König.
drivers/dma-buf/dma-heap.c | 27 +++++++++++++++++++++++++++
drivers/dma-buf/heaps/system_heap.c | 3 +++
include/linux/dma-heap.h | 11 +++++++++++
3 files changed, 41 insertions(+)
diff --git a/drivers/dma-buf/dma-heap.c b/drivers/dma-buf/dma-heap.c
index 8f5848aa144f..885072427775 100644
--- a/drivers/dma-buf/dma-heap.c
+++ b/drivers/dma-buf/dma-heap.c
@@ -7,6 +7,7 @@
*/
#include <linux/cdev.h>
+#include <linux/cgroup_gpu.h>
#include <linux/debugfs.h>
#include <linux/device.h>
#include <linux/dma-buf.h>
@@ -31,6 +32,7 @@
* @heap_devt heap device node
* @list list head connecting to list of heaps
* @heap_cdev heap char device
+ * @gpucg_dev gpu cgroup device for memory accounting
*
* Represents a heap of memory from which buffers can be made.
*/
@@ -41,6 +43,9 @@ struct dma_heap {
dev_t heap_devt;
struct list_head list;
struct cdev heap_cdev;
+#ifdef CONFIG_CGROUP_GPU
+ struct gpucg_device gpucg_dev;
+#endif
};
static LIST_HEAD(heap_list);
@@ -216,6 +221,26 @@ const char *dma_heap_get_name(struct dma_heap *heap)
return heap->name;
}
+#ifdef CONFIG_CGROUP_GPU
+/**
+ * dma_heap_get_gpucg_dev() - get struct gpucg_device for the heap.
+ * @heap: DMA-Heap to get the gpucg_device struct for.
+ *
+ * Returns:
+ * The gpucg_device struct for the heap. NULL if the GPU cgroup controller is
+ * not enabled.
+ */
+struct gpucg_device *dma_heap_get_gpucg_dev(struct dma_heap *heap)
+{
+ return &heap->gpucg_dev;
+}
+#else /* CONFIG_CGROUP_GPU */
+struct gpucg_device *dma_heap_get_gpucg_dev(struct dma_heap *heap)
+{
+ return NULL;
+}
+#endif /* CONFIG_CGROUP_GPU */
+
struct dma_heap *dma_heap_add(const struct dma_heap_export_info *exp_info)
{
struct dma_heap *heap, *h, *err_ret;
@@ -288,6 +313,8 @@ struct dma_heap *dma_heap_add(const struct dma_heap_export_info *exp_info)
list_add(&heap->list, &heap_list);
mutex_unlock(&heap_list_lock);
+ gpucg_register_device(dma_heap_get_gpucg_dev(heap), exp_info->name);
+
return heap;
err2:
diff --git a/drivers/dma-buf/heaps/system_heap.c b/drivers/dma-buf/heaps/system_heap.c
index ab7fd896d2c4..752a05c3cfe2 100644
--- a/drivers/dma-buf/heaps/system_heap.c
+++ b/drivers/dma-buf/heaps/system_heap.c
@@ -395,6 +395,9 @@ static struct dma_buf *system_heap_allocate(struct dma_heap *heap,
exp_info.ops = &system_heap_buf_ops;
exp_info.size = buffer->len;
exp_info.flags = fd_flags;
+#ifdef CONFIG_CGROUP_GPU
+ exp_info.gpucg_dev = dma_heap_get_gpucg_dev(heap);
+#endif
exp_info.priv = buffer;
dmabuf = dma_buf_export(&exp_info);
if (IS_ERR(dmabuf)) {
diff --git a/include/linux/dma-heap.h b/include/linux/dma-heap.h
index 0c05561cad6e..e447a61d054e 100644
--- a/include/linux/dma-heap.h
+++ b/include/linux/dma-heap.h
@@ -10,6 +10,7 @@
#define _DMA_HEAPS_H
#include <linux/cdev.h>
+#include <linux/cgroup_gpu.h>
#include <linux/types.h>
struct dma_heap;
@@ -59,6 +60,16 @@ void *dma_heap_get_drvdata(struct dma_heap *heap);
*/
const char *dma_heap_get_name(struct dma_heap *heap);
+/**
+ * dma_heap_get_gpucg_dev() - get a pointer to the struct gpucg_device for the
+ * heap.
+ * @heap: DMA-Heap to retrieve gpucg_device for.
+ *
+ * Returns:
+ * The gpucg_device struct for the heap.
+ */
+struct gpucg_device *dma_heap_get_gpucg_dev(struct dma_heap *heap);
+
/**
* dma_heap_add - adds a heap to dmabuf heaps
* @exp_info: information needed to register this heap
--
2.35.1.265.g69c8d7142f-goog
Hello,
On Fri, Feb 11, 2022 at 04:18:23PM +0000, T.J. Mercier wrote:
> The GPU/DRM cgroup controller came into being when a consensus[1]
> was reached that the resources it tracked were unsuitable to be integrated
> into memcg. Originally, the proposed controller was specific to the DRM
> subsystem and was intended to track GEM buffers and GPU-specific
> resources[2]. In order to help establish a unified memory accounting model
> for all GPU and all related subsystems, Daniel Vetter put forth a
> suggestion to move it out of the DRM subsystem so that it can be used by
> other DMA-BUF exporters as well[3]. This RFC proposes an interface that
> does the same.
>
> [1]: https://patchwork.kernel.org/project/dri-devel/cover/[email protected]/#22624705
> [2]: https://lore.kernel.org/amd-gfx/[email protected]/
> [3]: https://lore.kernel.org/amd-gfx/YCVOl8%[email protected]/
IIRC, the only consensus was that it needs to be a separate controller and
folks had trouble agreeing on resource types, control mechanism and
interface. Imma keep an eye on how the discussion develops among GPU folks.
Please feel free to ping if there's an area my input may be useful.
Thanks.
--
tejun
On Fri, Feb 11, 2022 at 8:18 AM T.J. Mercier <[email protected]> wrote:
>
> This patch series revisits the proposal for a GPU cgroup controller to
> track and limit memory allocations by various device/allocator
> subsystems. The patch series also contains a simple prototype to
> illustrate how Android intends to implement DMA-BUF allocator
> attribution using the GPU cgroup controller. The prototype does not
> include resource limit enforcements.
>
> Changelog:
>
> v2:
> See the previous revision of this change submitted by Hridya Valsaraju
> at: https://lore.kernel.org/all/[email protected]/
>
> Move dma-buf cgroup charge transfer from a dma_buf_op defined by every
> heap to a single dma-buf function for all heaps per Daniel Vetter and
> Christian König. Pointers to struct gpucg and struct gpucg_device
> tracking the current associations were added to the dma_buf struct to
> achieve this.
>
> Fix incorrect Kconfig help section indentation per Randy Dunlap.
>
> History of the GPU cgroup controller
> ====================================
> The GPU/DRM cgroup controller came into being when a consensus[1]
> was reached that the resources it tracked were unsuitable to be integrated
> into memcg. Originally, the proposed controller was specific to the DRM
> subsystem and was intended to track GEM buffers and GPU-specific
> resources[2]. In order to help establish a unified memory accounting model
> for all GPU and all related subsystems, Daniel Vetter put forth a
> suggestion to move it out of the DRM subsystem so that it can be used by
> other DMA-BUF exporters as well[3]. This RFC proposes an interface that
> does the same.
>
> [1]: https://patchwork.kernel.org/project/dri-devel/cover/[email protected]/#22624705
> [2]: https://lore.kernel.org/amd-gfx/[email protected]/
> [3]: https://lore.kernel.org/amd-gfx/YCVOl8%[email protected]/
>
> T.J. Mercier (6):
> gpu: rfc: Proposal for a GPU cgroup controller
> cgroup: gpu: Add a cgroup controller for allocator attribution of GPU
> memory
> dmabuf: Use the GPU cgroup charge/uncharge APIs
> dmabuf: heaps: export system_heap buffers with GPU cgroup charging
> dmabuf: Add gpu cgroup charge transfer function
> android: binder: Add a buffer flag to relinquish ownership of fds
>
> Documentation/gpu/rfc/gpu-cgroup.rst | 195 +++++++++++++++++
> Documentation/gpu/rfc/index.rst | 4 +
> drivers/android/binder.c | 26 +++
> drivers/dma-buf/dma-buf.c | 100 +++++++++
> drivers/dma-buf/dma-heap.c | 27 +++
> drivers/dma-buf/heaps/system_heap.c | 3 +
> include/linux/cgroup_gpu.h | 127 +++++++++++
> include/linux/cgroup_subsys.h | 4 +
> include/linux/dma-buf.h | 22 +-
> include/linux/dma-heap.h | 11 +
> include/uapi/linux/android/binder.h | 1 +
> init/Kconfig | 7 +
> kernel/cgroup/Makefile | 1 +
> kernel/cgroup/gpu.c | 304 +++++++++++++++++++++++++++
> 14 files changed, 830 insertions(+), 2 deletions(-)
> create mode 100644 Documentation/gpu/rfc/gpu-cgroup.rst
> create mode 100644 include/linux/cgroup_gpu.h
> create mode 100644 kernel/cgroup/gpu.c
>
> --
> 2.35.1.265.g69c8d7142f-goog
>
Gentle nudge to GPU maintainers to please provide their feedback on
this RFC. Thanks!