Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions ggml/UPSTREAM
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ patches:
patches/ggml/0001-fix-threadpool-oversubscription.patch
patches/ggml/0002-backend-reg-filter.patch
patches/ggml/0003-metal-mapped-buffer-null.patch
patches/ggml/0005-vulkan-keep-physical-device-handles.patch

This directory is generated from the upstream ggml tree at the SHA above, minus
.github/, with the listed downstream patches applied in order. Do not edit it by
Expand Down
3 changes: 3 additions & 0 deletions ggml/src/ggml-vulkan/ggml-vulkan-types.h
Original file line number Diff line number Diff line change
Expand Up @@ -1345,6 +1345,9 @@ struct vk_instance_t {
PFN_vkCmdInsertDebugUtilsLabelEXT pfn_vkCmdInsertDebugUtilsLabelEXT = {};

std::vector<size_t> device_indices;
// physical device handles from the enumeration that produced device_indices;
// the loader's order can change later (e.g. on Windows), the handles do not
std::vector<vk::PhysicalDevice> physical_devices;
std::vector<bool> device_supports_membudget;
vk_device devices[GGML_VK_MAX_DEVICES];
};
Expand Down
43 changes: 19 additions & 24 deletions ggml/src/ggml-vulkan/ggml-vulkan.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -3999,16 +3999,14 @@ vk_device ggml_vk_get_device(size_t idx) {

device->memory_logger = std::unique_ptr<vk_memory_logger>(new vk_memory_logger());

size_t dev_num = vk_instance.device_indices[idx];

std::vector<vk::PhysicalDevice> physical_devices = vk_instance.instance.enumeratePhysicalDevices();

if (dev_num >= physical_devices.size()) {
std::cerr << "ggml_vulkan: Device with index " << dev_num << " does not exist." << std::endl;
if (idx >= vk_instance.physical_devices.size()) {
std::cerr << "ggml_vulkan: Device with index " << idx << " does not exist." << std::endl;
throw std::runtime_error("Device not found");
}

device->physical_device = physical_devices[dev_num];
// use the handle from instance init rather than re-enumerating: the loader's
// order can change in between, and device_indices would then pick another GPU
device->physical_device = vk_instance.physical_devices[idx];
const std::vector<vk::ExtensionProperties> ext_props = device->physical_device.enumerateDeviceExtensionProperties();

device->architecture = get_device_architecture(device->physical_device);
Expand Down Expand Up @@ -4933,15 +4931,9 @@ static void ggml_vk_print_gpu_info(size_t idx) {
size_t dev_num = vk_instance.device_indices[idx];
VK_LOG_DEBUG("ggml_vk_print_gpu_info(" << dev_num << ")");
GGML_ASSERT(vk_instance_initialized);
GGML_ASSERT(idx < vk_instance.physical_devices.size());

std::vector<vk::PhysicalDevice> devices = vk_instance.instance.enumeratePhysicalDevices();

if (dev_num >= devices.size()) {
std::cerr << "ggml_vulkan: Device with index " << dev_num << " does not exist." << std::endl;
throw std::runtime_error("Device not found");
}

vk::PhysicalDevice physical_device = devices[dev_num];
vk::PhysicalDevice physical_device = vk_instance.physical_devices[idx];
std::vector<vk::ExtensionProperties> ext_props = physical_device.enumerateDeviceExtensionProperties();

bool fp16_storage = false;
Expand Down Expand Up @@ -5410,10 +5402,15 @@ void ggml_vk_instance_init() {
return;
}
}

for (size_t idx : vk_instance.device_indices) {
vk_instance.physical_devices.push_back(devices[idx]);
}

GGML_LOG_DEBUG("ggml_vulkan: Found %zu Vulkan devices:\n", vk_instance.device_indices.size());

for (size_t i = 0; i < vk_instance.device_indices.size(); i++) {
vk::PhysicalDevice vkdev = devices[vk_instance.device_indices[i]];
vk::PhysicalDevice vkdev = vk_instance.physical_devices[i];
std::vector<vk::ExtensionProperties> extensionprops = vkdev.enumerateDeviceExtensionProperties();

bool membudget_supported = false;
Expand Down Expand Up @@ -12903,11 +12900,10 @@ int ggml_vk_get_device_count() {

void ggml_vk_get_device_description(int device, char * description, size_t description_size) {
ggml_vk_instance_init();

std::vector<vk::PhysicalDevice> devices = vk_instance.instance.enumeratePhysicalDevices();
GGML_ASSERT(device >= 0 && device < (int) vk_instance.physical_devices.size());

vk::PhysicalDeviceProperties props;
devices[device].getProperties(&props);
vk_instance.physical_devices[device].getProperties(&props);

snprintf(description, description_size, "%s", props.deviceName.data());
}
Expand Down Expand Up @@ -15181,15 +15177,14 @@ int ggml_backend_vk_get_device_count() {

void ggml_backend_vk_get_device_description(int device, char * description, size_t description_size) {
GGML_ASSERT(device < (int) vk_instance.device_indices.size());
int dev_idx = vk_instance.device_indices[device];
ggml_vk_get_device_description(dev_idx, description, description_size);
ggml_vk_get_device_description(device, description, description_size);
}

void ggml_backend_vk_get_device_memory(int device, size_t * free, size_t * total) {
GGML_ASSERT(device < (int) vk_instance.device_indices.size());
GGML_ASSERT(device < (int) vk_instance.device_supports_membudget.size());

vk::PhysicalDevice vkdev = vk_instance.instance.enumeratePhysicalDevices()[vk_instance.device_indices[device]];
vk::PhysicalDevice vkdev = vk_instance.physical_devices[device];
vk::PhysicalDeviceMemoryBudgetPropertiesEXT budgetprops;
vk::PhysicalDeviceMemoryProperties2 memprops = {};
const bool membudget_supported = vk_instance.device_supports_membudget[device];
Expand Down Expand Up @@ -15221,7 +15216,7 @@ void ggml_backend_vk_get_device_memory(int device, size_t * free, size_t * total
static vk::PhysicalDeviceType ggml_backend_vk_get_device_type(int device_idx) {
GGML_ASSERT(device_idx >= 0 && device_idx < (int) vk_instance.device_indices.size());

vk::PhysicalDevice device = vk_instance.instance.enumeratePhysicalDevices()[vk_instance.device_indices[device_idx]];
vk::PhysicalDevice device = vk_instance.physical_devices[device_idx];

vk::PhysicalDeviceProperties2 props = {};
device.getProperties2(&props);
Expand All @@ -15232,7 +15227,7 @@ static vk::PhysicalDeviceType ggml_backend_vk_get_device_type(int device_idx) {
static std::string ggml_backend_vk_get_device_pci_id(int device_idx) {
GGML_ASSERT(device_idx >= 0 && device_idx < (int) vk_instance.device_indices.size());

vk::PhysicalDevice device = vk_instance.instance.enumeratePhysicalDevices()[vk_instance.device_indices[device_idx]];
vk::PhysicalDevice device = vk_instance.physical_devices[device_idx];

const std::vector<vk::ExtensionProperties> ext_props = device.enumerateDeviceExtensionProperties();

Expand Down
125 changes: 125 additions & 0 deletions patches/ggml/0005-vulkan-keep-physical-device-handles.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
diff --git a/src/ggml-vulkan/ggml-vulkan-types.h b/src/ggml-vulkan/ggml-vulkan-types.h
index 4158ff77..8251e0a9 100644
--- a/src/ggml-vulkan/ggml-vulkan-types.h
+++ b/src/ggml-vulkan/ggml-vulkan-types.h
@@ -1345,6 +1345,9 @@ struct vk_instance_t {
PFN_vkCmdInsertDebugUtilsLabelEXT pfn_vkCmdInsertDebugUtilsLabelEXT = {};

std::vector<size_t> device_indices;
+ // physical device handles from the enumeration that produced device_indices;
+ // the loader's order can change later (e.g. on Windows), the handles do not
+ std::vector<vk::PhysicalDevice> physical_devices;
std::vector<bool> device_supports_membudget;
vk_device devices[GGML_VK_MAX_DEVICES];
};
diff --git a/src/ggml-vulkan/ggml-vulkan.cpp b/src/ggml-vulkan/ggml-vulkan.cpp
index 5587dfc2..ebc1c24a 100644
--- a/src/ggml-vulkan/ggml-vulkan.cpp
+++ b/src/ggml-vulkan/ggml-vulkan.cpp
@@ -3999,16 +3999,14 @@ vk_device ggml_vk_get_device(size_t idx) {

device->memory_logger = std::unique_ptr<vk_memory_logger>(new vk_memory_logger());

- size_t dev_num = vk_instance.device_indices[idx];
-
- std::vector<vk::PhysicalDevice> physical_devices = vk_instance.instance.enumeratePhysicalDevices();
-
- if (dev_num >= physical_devices.size()) {
- std::cerr << "ggml_vulkan: Device with index " << dev_num << " does not exist." << std::endl;
+ if (idx >= vk_instance.physical_devices.size()) {
+ std::cerr << "ggml_vulkan: Device with index " << idx << " does not exist." << std::endl;
throw std::runtime_error("Device not found");
}

- device->physical_device = physical_devices[dev_num];
+ // use the handle from instance init rather than re-enumerating: the loader's
+ // order can change in between, and device_indices would then pick another GPU
+ device->physical_device = vk_instance.physical_devices[idx];
const std::vector<vk::ExtensionProperties> ext_props = device->physical_device.enumerateDeviceExtensionProperties();

device->architecture = get_device_architecture(device->physical_device);
@@ -4933,15 +4931,9 @@ static void ggml_vk_print_gpu_info(size_t idx) {
size_t dev_num = vk_instance.device_indices[idx];
VK_LOG_DEBUG("ggml_vk_print_gpu_info(" << dev_num << ")");
GGML_ASSERT(vk_instance_initialized);
+ GGML_ASSERT(idx < vk_instance.physical_devices.size());

- std::vector<vk::PhysicalDevice> devices = vk_instance.instance.enumeratePhysicalDevices();
-
- if (dev_num >= devices.size()) {
- std::cerr << "ggml_vulkan: Device with index " << dev_num << " does not exist." << std::endl;
- throw std::runtime_error("Device not found");
- }
-
- vk::PhysicalDevice physical_device = devices[dev_num];
+ vk::PhysicalDevice physical_device = vk_instance.physical_devices[idx];
std::vector<vk::ExtensionProperties> ext_props = physical_device.enumerateDeviceExtensionProperties();

bool fp16_storage = false;
@@ -5410,10 +5402,15 @@ void ggml_vk_instance_init() {
return;
}
}
+
+ for (size_t idx : vk_instance.device_indices) {
+ vk_instance.physical_devices.push_back(devices[idx]);
+ }
+
GGML_LOG_DEBUG("ggml_vulkan: Found %zu Vulkan devices:\n", vk_instance.device_indices.size());

for (size_t i = 0; i < vk_instance.device_indices.size(); i++) {
- vk::PhysicalDevice vkdev = devices[vk_instance.device_indices[i]];
+ vk::PhysicalDevice vkdev = vk_instance.physical_devices[i];
std::vector<vk::ExtensionProperties> extensionprops = vkdev.enumerateDeviceExtensionProperties();

bool membudget_supported = false;
@@ -12903,11 +12900,10 @@ int ggml_vk_get_device_count() {

void ggml_vk_get_device_description(int device, char * description, size_t description_size) {
ggml_vk_instance_init();
-
- std::vector<vk::PhysicalDevice> devices = vk_instance.instance.enumeratePhysicalDevices();
+ GGML_ASSERT(device >= 0 && device < (int) vk_instance.physical_devices.size());

vk::PhysicalDeviceProperties props;
- devices[device].getProperties(&props);
+ vk_instance.physical_devices[device].getProperties(&props);

snprintf(description, description_size, "%s", props.deviceName.data());
}
@@ -15181,15 +15177,14 @@ int ggml_backend_vk_get_device_count() {

void ggml_backend_vk_get_device_description(int device, char * description, size_t description_size) {
GGML_ASSERT(device < (int) vk_instance.device_indices.size());
- int dev_idx = vk_instance.device_indices[device];
- ggml_vk_get_device_description(dev_idx, description, description_size);
+ ggml_vk_get_device_description(device, description, description_size);
}

void ggml_backend_vk_get_device_memory(int device, size_t * free, size_t * total) {
GGML_ASSERT(device < (int) vk_instance.device_indices.size());
GGML_ASSERT(device < (int) vk_instance.device_supports_membudget.size());

- vk::PhysicalDevice vkdev = vk_instance.instance.enumeratePhysicalDevices()[vk_instance.device_indices[device]];
+ vk::PhysicalDevice vkdev = vk_instance.physical_devices[device];
vk::PhysicalDeviceMemoryBudgetPropertiesEXT budgetprops;
vk::PhysicalDeviceMemoryProperties2 memprops = {};
const bool membudget_supported = vk_instance.device_supports_membudget[device];
@@ -15221,7 +15216,7 @@ void ggml_backend_vk_get_device_memory(int device, size_t * free, size_t * total
static vk::PhysicalDeviceType ggml_backend_vk_get_device_type(int device_idx) {
GGML_ASSERT(device_idx >= 0 && device_idx < (int) vk_instance.device_indices.size());

- vk::PhysicalDevice device = vk_instance.instance.enumeratePhysicalDevices()[vk_instance.device_indices[device_idx]];
+ vk::PhysicalDevice device = vk_instance.physical_devices[device_idx];

vk::PhysicalDeviceProperties2 props = {};
device.getProperties2(&props);
@@ -15232,7 +15227,7 @@ static vk::PhysicalDeviceType ggml_backend_vk_get_device_type(int device_idx) {
static std::string ggml_backend_vk_get_device_pci_id(int device_idx) {
GGML_ASSERT(device_idx >= 0 && device_idx < (int) vk_instance.device_indices.size());

- vk::PhysicalDevice device = vk_instance.instance.enumeratePhysicalDevices()[vk_instance.device_indices[device_idx]];
+ vk::PhysicalDevice device = vk_instance.physical_devices[device_idx];

const std::vector<vk::ExtensionProperties> ext_props = device.enumerateDeviceExtensionProperties();