Skip to content

Commit 3cacd85

Browse files
authored
ggml: new backend for Virglrenderer API Remoting acceleration (v2) (ggml-org#18718)
1 parent 9fa104f commit 3cacd85

47 files changed

Lines changed: 4711 additions & 0 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

CODEOWNERS

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@
6767
/ggml/src/ggml-rpc/ @rgerganov
6868
/ggml/src/ggml-threading.* @ggerganov
6969
/ggml/src/ggml-vulkan/ @0cc4m
70+
/ggml/src/ggml-virtgpu/ @kpouget
7071
/ggml/src/ggml-webgpu/ @reeselevine
7172
/ggml/src/ggml-zdnn/ @taronaeo @Andreas-Krebbel @AlekseiNikiforovIBM
7273
/ggml/src/ggml.c @ggerganov

ggml/CMakeLists.txt

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -228,6 +228,8 @@ option(GGML_WEBGPU_CPU_PROFILE "ggml: enable WebGPU profiling (CPU)
228228
option(GGML_WEBGPU_GPU_PROFILE "ggml: enable WebGPU profiling (GPU)" OFF)
229229
option(GGML_WEBGPU_JSPI "ggml: use JSPI for WebGPU" ON)
230230
option(GGML_ZDNN "ggml: use zDNN" OFF)
231+
option(GGML_VIRTGPU "ggml: use the VirtGPU/Virglrenderer API Remoting frontend" OFF)
232+
option(GGML_VIRTGPU_BACKEND "ggml: build the VirtGPU/Virglrenderer API Remoting backend" OFF)
231233
option(GGML_METAL "ggml: use Metal" ${GGML_METAL_DEFAULT})
232234
option(GGML_METAL_NDEBUG "ggml: disable Metal debugging" OFF)
233235
option(GGML_METAL_SHADER_DEBUG "ggml: compile Metal with -fno-fast-math" OFF)
@@ -320,6 +322,7 @@ set(GGML_PUBLIC_HEADERS
320322
include/ggml-opt.h
321323
include/ggml-metal.h
322324
include/ggml-rpc.h
325+
include/ggml-virtgpu.h
323326
include/ggml-sycl.h
324327
include/ggml-vulkan.h
325328
include/ggml-webgpu.h

ggml/include/ggml-virtgpu.h

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
#pragma once
2+
3+
#include "ggml.h"
4+
#include "ggml-backend.h"
5+
6+
#ifdef __cplusplus
7+
extern "C" {
8+
#endif
9+
10+
#define GGML_REMOTING_FRONTEND_NAME "RemotingFrontend"
11+
12+
GGML_BACKEND_API ggml_backend_reg_t ggml_backend_virtgpu_reg();
13+
14+
#ifdef __cplusplus
15+
}
16+
#endif

ggml/src/CMakeLists.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -451,6 +451,7 @@ ggml_add_backend(HIP)
451451
ggml_add_backend(METAL)
452452
ggml_add_backend(MUSA)
453453
ggml_add_backend(RPC)
454+
ggml_add_backend(VirtGPU)
454455
ggml_add_backend(SYCL)
455456
ggml_add_backend(Vulkan)
456457
ggml_add_backend(WebGPU)

ggml/src/ggml-backend-reg.cpp

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,10 @@
6969
#include "ggml-rpc.h"
7070
#endif
7171

72+
#ifdef GGML_USE_VIRTGPU_FRONTEND
73+
#include "ggml-virtgpu.h"
74+
#endif
75+
7276
#ifdef GGML_USE_CANN
7377
#include "ggml-cann.h"
7478
#endif
@@ -180,14 +184,23 @@ struct ggml_backend_registry {
180184
register_backend(ggml_backend_sycl_reg());
181185
#endif
182186
#ifdef GGML_USE_VULKAN
187+
// Add runtime disable check
188+
if (getenv("GGML_DISABLE_VULKAN") == nullptr) {
183189
register_backend(ggml_backend_vk_reg());
190+
} else {
191+
GGML_LOG_DEBUG("Vulkan backend disabled by GGML_DISABLE_VULKAN environment variable\n");
192+
}
184193
#endif
185194
#ifdef GGML_USE_WEBGPU
186195
register_backend(ggml_backend_webgpu_reg());
187196
#endif
188197
#ifdef GGML_USE_ZDNN
189198
register_backend(ggml_backend_zdnn_reg());
190199
#endif
200+
#ifdef GGML_USE_VIRTGPU_FRONTEND
201+
register_backend(ggml_backend_virtgpu_reg());
202+
#endif
203+
191204
#ifdef GGML_USE_OPENCL
192205
register_backend(ggml_backend_opencl_reg());
193206
#endif
@@ -604,6 +617,7 @@ void ggml_backend_load_all_from_path(const char * dir_path) {
604617
ggml_backend_load_best("rpc", silent, dir_path);
605618
ggml_backend_load_best("sycl", silent, dir_path);
606619
ggml_backend_load_best("vulkan", silent, dir_path);
620+
ggml_backend_load_best("virtgpu", silent, dir_path);
607621
ggml_backend_load_best("opencl", silent, dir_path);
608622
ggml_backend_load_best("hexagon", silent, dir_path);
609623
ggml_backend_load_best("musa", silent, dir_path);
Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
cmake_minimum_required(VERSION 3.19)
2+
cmake_policy(SET CMP0114 NEW)
3+
4+
include(ExternalProject)
5+
6+
message(STATUS "Including the VirtGPU/Virglrenderer API Remoting")
7+
8+
# Download venus_hw.h from virglrenderer repository
9+
ExternalProject_Add(
10+
venus_hw_header
11+
URL https://gitlab.freedesktop.org/virgl/virglrenderer/-/raw/virglrenderer-1.2.0/src/venus_hw.h
12+
DOWNLOAD_NO_EXTRACT YES
13+
DOWNLOAD_DIR ${CMAKE_CURRENT_SOURCE_DIR}/include
14+
DOWNLOAD_NAME venus_hw.h
15+
CONFIGURE_COMMAND ""
16+
BUILD_COMMAND ""
17+
INSTALL_COMMAND ""
18+
LOG_DOWNLOAD ON
19+
)
20+
21+
if (NOT GGML_VIRTGPU_BACKEND STREQUAL "ONLY")
22+
message(STATUS "Enable the VirtGPU/Virglrenderer API Remoting frontend library")
23+
24+
find_package(PkgConfig REQUIRED)
25+
pkg_check_modules(DRM REQUIRED libdrm)
26+
if (NOT GGML_BACKEND_DL)
27+
# cannot simply use USE_VIRTGPU, as in the 'else()' case the
28+
# frontend isn't compiled
29+
target_compile_definitions(ggml PUBLIC "GGML_USE_VIRTGPU_FRONTEND")
30+
endif()
31+
32+
ggml_add_backend_library(ggml-virtgpu
33+
ggml-backend-buffer.cpp
34+
ggml-backend.cpp
35+
ggml-backend-device.cpp
36+
ggml-backend-reg.cpp
37+
ggml-backend-buffer-type.cpp
38+
virtgpu-apir.h
39+
virtgpu-forward.gen.h
40+
virtgpu.cpp
41+
virtgpu-shm.cpp
42+
virtgpu-utils.cpp
43+
virtgpu-forward-device.cpp
44+
virtgpu-forward-buffer-type.cpp
45+
virtgpu-forward-buffer.cpp
46+
virtgpu-forward-backend.cpp
47+
virtgpu-forward-impl.h
48+
apir_cs_ggml-rpc-front.cpp
49+
../../include/ggml-virtgpu.h)
50+
51+
target_include_directories(ggml-virtgpu PUBLIC /usr/include/libdrm/)
52+
53+
target_link_libraries(ggml-virtgpu PUBLIC ${DRM_LIBRARIES})
54+
target_include_directories(ggml-virtgpu PUBLIC ${DRM_INCLUDE_DIRS})
55+
target_compile_options(ggml-virtgpu PUBLIC ${DRM_CFLAGS_OTHER})
56+
57+
target_include_directories(ggml-virtgpu PUBLIC ./include)
58+
target_include_directories(ggml-virtgpu PRIVATE ${CMAKE_CURRENT_BINARY_DIR})
59+
60+
# Ensure venus_hw.h is downloaded before building ggml-virtgpu
61+
add_dependencies(ggml-virtgpu venus_hw_header)
62+
63+
target_compile_options(ggml-virtgpu PRIVATE -std=c++20)
64+
else()
65+
message(STATUS "Not building the VirtGPU/Virglrenderer API Remoting frontend library")
66+
endif()
67+
68+
if (NOT GGML_VIRTGPU_BACKEND STREQUAL "OFF")
69+
add_subdirectory("backend")
70+
endif()
Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
#include "backend/shared/apir_cs_rpc.h"
2+
#include "ggml-backend-impl.h"
3+
#include "ggml-impl.h"
4+
#include "ggml-remoting.h"
5+
6+
#include <cinttypes>
7+
#include <unordered_map>
8+
#include <unordered_set>
9+
#include <vector>
10+
11+
apir_rpc_tensor apir_serialize_tensor(const ggml_tensor * tensor) {
12+
apir_rpc_tensor result;
13+
result.id = reinterpret_cast<uint64_t>(tensor);
14+
result.type = tensor->type;
15+
if (tensor->buffer) {
16+
ggml_backend_buffer_t buffer = tensor->buffer;
17+
18+
result.buffer = BUFFER_TO_HOST_HANDLE(buffer);
19+
} else {
20+
result.buffer = 0;
21+
}
22+
for (uint32_t i = 0; i < GGML_MAX_DIMS; i++) {
23+
result.ne[i] = tensor->ne[i];
24+
result.nb[i] = tensor->nb[i];
25+
}
26+
result.op = tensor->op;
27+
for (uint32_t i = 0; i < GGML_MAX_OP_PARAMS / sizeof(int32_t); i++) {
28+
result.op_params[i] = tensor->op_params[i];
29+
}
30+
result.flags = tensor->flags;
31+
for (uint32_t i = 0; i < GGML_MAX_SRC; i++) {
32+
result.src[i] = reinterpret_cast<uint64_t>(tensor->src[i]);
33+
}
34+
result.view_src = reinterpret_cast<uint64_t>(tensor->view_src);
35+
result.view_offs = tensor->view_offs;
36+
result.data = reinterpret_cast<uint64_t>(tensor->data);
37+
if (tensor->data) {
38+
if (!tensor->buffer) {
39+
GGML_ABORT("tensor has data but not buffer");
40+
}
41+
// tensor->data is serialized as an offset to the buffer base address
42+
result.data -= reinterpret_cast<uint64_t>(BUFFER_TO_GGML_CONTEXT(tensor->buffer)->base);
43+
}
44+
snprintf(result.name, GGML_MAX_NAME, "%s", tensor->name);
45+
return result;
46+
}
47+
48+
void apir_add_tensor(ggml_tensor * tensor,
49+
std::vector<apir_rpc_tensor> & tensors,
50+
std::unordered_set<ggml_tensor *> & visited) {
51+
if (tensor == nullptr) {
52+
return;
53+
}
54+
if (visited.find(tensor) != visited.end()) {
55+
return;
56+
}
57+
visited.insert(tensor);
58+
for (int i = 0; i < GGML_MAX_SRC; i++) {
59+
apir_add_tensor(tensor->src[i], tensors, visited);
60+
}
61+
apir_add_tensor(tensor->view_src, tensors, visited);
62+
tensors.push_back(apir_serialize_tensor(tensor));
63+
}
64+
65+
void apir_serialize_graph(const ggml_cgraph * cgraph, std::vector<uint8_t> & output) {
66+
uint32_t n_nodes = cgraph->n_nodes;
67+
std::vector<apir_rpc_tensor> tensors;
68+
std::unordered_set<ggml_tensor *> visited;
69+
for (uint32_t i = 0; i < n_nodes; i++) {
70+
apir_add_tensor(cgraph->nodes[i], tensors, visited);
71+
}
72+
// serialization format:
73+
// | n_nodes (4 bytes) | nodes (n_nodes * sizeof(uint64_t) | n_tensors (4 bytes) | tensors (n_tensors * sizeof(apir_rpc_tensor)) |
74+
uint32_t n_tensors = tensors.size();
75+
int output_size =
76+
sizeof(uint32_t) + n_nodes * sizeof(uint64_t) + sizeof(uint32_t) + n_tensors * sizeof(apir_rpc_tensor);
77+
output.resize(output_size, 0);
78+
memcpy(output.data(), &n_nodes, sizeof(n_nodes));
79+
for (uint32_t i = 0; i < n_nodes; i++) {
80+
memcpy(output.data() + sizeof(n_nodes) + i * sizeof(uint64_t), &cgraph->nodes[i], sizeof(uint64_t));
81+
}
82+
uint32_t * out_ntensors = (uint32_t *) (output.data() + sizeof(n_nodes) + n_nodes * sizeof(uint64_t));
83+
*out_ntensors = n_tensors;
84+
apir_rpc_tensor * out_tensors =
85+
(apir_rpc_tensor *) (output.data() + sizeof(n_nodes) + n_nodes * sizeof(uint64_t) + sizeof(uint32_t));
86+
memcpy(out_tensors, tensors.data(), n_tensors * sizeof(apir_rpc_tensor));
87+
}
Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
cmake_minimum_required(VERSION 3.19)
2+
cmake_policy(SET CMP0114 NEW)
3+
4+
message(STATUS "Enable the VirtGPU/Virglrenderer backend library")
5+
6+
ggml_add_backend_library(ggml-virtgpu-backend
7+
backend.cpp
8+
backend-dispatched.cpp
9+
backend-dispatched-backend.cpp
10+
backend-dispatched-device.cpp
11+
backend-dispatched-buffer.cpp
12+
backend-dispatched-buffer-type.cpp
13+
shared/api_remoting.h
14+
shared/apir_backend.h
15+
shared/apir_cs.h
16+
apir_cs_ggml-rpc-back.cpp)
17+
18+
target_compile_options(ggml-virtgpu-backend PRIVATE -std=c++20)
19+
20+
# Add include directory for ggml-backend-impl.h and other core headers
21+
target_include_directories(ggml-virtgpu-backend PRIVATE ../..)
Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
#include "ggml-backend-impl.h"
2+
#include "ggml-impl.h"
3+
#include "shared/apir_cs_rpc.h"
4+
5+
#include <cinttypes>
6+
#include <unordered_map>
7+
#include <unordered_set>
8+
#include <vector>
9+
10+
std::unordered_set<ggml_backend_buffer_t> backend_buffers;
11+
12+
void apir_track_backend_buffer(ggml_backend_buffer_t buffer) {
13+
backend_buffers.insert(buffer);
14+
}
15+
16+
bool apir_untrack_backend_buffer(ggml_backend_buffer_t buffer) {
17+
auto it = backend_buffers.find(buffer);
18+
if (it == backend_buffers.end()) {
19+
return false;
20+
}
21+
22+
backend_buffers.erase(it);
23+
return true;
24+
}
25+
26+
std::unordered_set<ggml_backend_buffer_t> apir_get_track_backend_buffers() {
27+
return backend_buffers;
28+
}
29+
30+
ggml_tensor * apir_deserialize_tensor(ggml_context * ctx, const apir_rpc_tensor * tensor) {
31+
ggml_tensor * result =
32+
ggml_new_tensor_4d(ctx, (ggml_type) tensor->type, tensor->ne[0], tensor->ne[1], tensor->ne[2], tensor->ne[3]);
33+
for (uint32_t i = 0; i < GGML_MAX_DIMS; i++) {
34+
result->nb[i] = tensor->nb[i];
35+
}
36+
result->buffer = reinterpret_cast<ggml_backend_buffer_t>(tensor->buffer);
37+
if (result->buffer && backend_buffers.find(result->buffer) == backend_buffers.end()) {
38+
printf("WARNING: HOST BUFFER NOT FOUND | %p\n", (void *) result->buffer);
39+
result->buffer = nullptr;
40+
}
41+
42+
uint64_t tensor_data = tensor->data;
43+
if (result->buffer) {
44+
// require that the tensor data does not go beyond the buffer end
45+
uint64_t tensor_size = (uint64_t) ggml_nbytes(result);
46+
uint64_t buffer_start = (uint64_t) ggml_backend_buffer_get_base(result->buffer);
47+
uint64_t buffer_size = (uint64_t) ggml_backend_buffer_get_size(result->buffer);
48+
49+
// tensor->data is serialized as an offset to the buffer base address
50+
tensor_data += buffer_start;
51+
52+
GGML_ASSERT(tensor_data + tensor_size >= tensor_data); // check for overflow
53+
GGML_ASSERT(tensor_data >= buffer_start && tensor_data + tensor_size <= buffer_start + buffer_size);
54+
}
55+
56+
result->op = (ggml_op) tensor->op;
57+
for (uint32_t i = 0; i < GGML_MAX_OP_PARAMS / sizeof(int32_t); i++) {
58+
result->op_params[i] = tensor->op_params[i];
59+
}
60+
result->flags = tensor->flags;
61+
result->data = reinterpret_cast<void *>(tensor_data);
62+
ggml_set_name(result, tensor->name);
63+
return result;
64+
}
65+
66+
ggml_tensor * apir_create_node(uint64_t id,
67+
ggml_context * ctx,
68+
const std::unordered_map<uint64_t, const apir_rpc_tensor *> & tensor_ptrs,
69+
std::unordered_map<uint64_t, ggml_tensor *> & tensor_map) {
70+
if (id == 0) {
71+
return nullptr;
72+
}
73+
if (tensor_map.find(id) != tensor_map.end()) {
74+
return tensor_map[id];
75+
}
76+
const apir_rpc_tensor * tensor = tensor_ptrs.at(id);
77+
ggml_tensor * result = apir_deserialize_tensor(ctx, tensor);
78+
if (result == nullptr) {
79+
return nullptr;
80+
}
81+
tensor_map[id] = result;
82+
for (int i = 0; i < GGML_MAX_SRC; i++) {
83+
result->src[i] = apir_create_node(tensor->src[i], ctx, tensor_ptrs, tensor_map);
84+
}
85+
result->view_src = apir_create_node(tensor->view_src, ctx, tensor_ptrs, tensor_map);
86+
result->view_offs = tensor->view_offs;
87+
return result;
88+
}
89+
90+
ggml_cgraph * apir_deserialize_graph(uint32_t n_nodes,
91+
uint32_t n_tensors,
92+
const apir_rpc_tensor * tensors,
93+
const uint64_t * nodes) {
94+
size_t buf_size = ggml_tensor_overhead() * (n_nodes + n_tensors) + ggml_graph_overhead_custom(n_nodes, false);
95+
ggml_init_params params = {
96+
/*.mem_size =*/buf_size,
97+
/*.mem_buffer =*/NULL,
98+
/*.no_alloc =*/true,
99+
};
100+
ggml_context * ctx = ggml_init(params);
101+
ggml_cgraph * graph = ggml_new_graph_custom(ctx, n_nodes, false);
102+
graph->n_nodes = n_nodes;
103+
std::unordered_map<uint64_t, const apir_rpc_tensor *> tensor_ptrs;
104+
for (uint32_t i = 0; i < n_tensors; i++) {
105+
tensor_ptrs[tensors[i].id] = &tensors[i];
106+
}
107+
std::unordered_map<uint64_t, ggml_tensor *> tensor_map;
108+
for (uint32_t i = 0; i < n_nodes; i++) {
109+
int64_t id;
110+
memcpy(&id, &nodes[i], sizeof(id));
111+
graph->nodes[i] = apir_create_node(id, ctx, tensor_ptrs, tensor_map);
112+
}
113+
114+
return graph;
115+
}

0 commit comments

Comments
 (0)