Merge pull request #8564 from lat9nq/dinner-fork

yuzu: Streamline broken Vulkan handling

1 files changed, 136 insertions, 0 deletions

diff --git a/src/yuzu/startup_checks.cpp b/src/yuzu/startup_checks.cpp
new file mode 100644
index 000000000..8421280bf
--- /dev/null
+++ b/src/yuzu/startup_checks.cpp
@@ -0,0 +1,136 @@
+// SPDX-FileCopyrightText: Copyright 2022 yuzu Emulator Project
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+#include "video_core/vulkan_common/vulkan_wrapper.h"
+
+#ifdef _WIN32
+#include <cstring> // for memset, strncpy
+#include <processthreadsapi.h>
+#include <windows.h>
+#elif defined(YUZU_UNIX)
+#include <errno.h>
+#include <sys/wait.h>
+#include <unistd.h>
+#endif
+
+#include <cstdio>
+#include "video_core/vulkan_common/vulkan_instance.h"
+#include "video_core/vulkan_common/vulkan_library.h"
+#include "yuzu/startup_checks.h"
+
+void CheckVulkan() {
+    // Just start the Vulkan loader, this will crash if something is wrong
+    try {
+        Vulkan::vk::InstanceDispatch dld;
+        const Common::DynamicLibrary library = Vulkan::OpenLibrary();
+        const Vulkan::vk::Instance instance =
+            Vulkan::CreateInstance(library, dld, VK_API_VERSION_1_0);
+
+    } catch (const Vulkan::vk::Exception& exception) {
+        std::fprintf(stderr, "Failed to initialize Vulkan: %s\n", exception.what());
+    }
+}
+
+bool StartupChecks(const char* arg0, bool* has_broken_vulkan) {
+#ifdef _WIN32
+    // Check environment variable to see if we are the child
+    char variable_contents[8];
+    const DWORD startup_check_var =
+        GetEnvironmentVariableA(STARTUP_CHECK_ENV_VAR, variable_contents, 8);
+    if (startup_check_var > 0 && std::strncmp(variable_contents, "ON", 8) == 0) {
+        CheckVulkan();
+        return true;
+    }
+
+    // Set the startup variable for child processes
+    const bool env_var_set = SetEnvironmentVariableA(STARTUP_CHECK_ENV_VAR, "ON");
+    if (!env_var_set) {
+        std::fprintf(stderr, "SetEnvironmentVariableA failed to set %s with error %d\n",
+                     STARTUP_CHECK_ENV_VAR, GetLastError());
+        return false;
+    }
+
+    PROCESS_INFORMATION process_info;
+    std::memset(&process_info, '\0', sizeof(process_info));
+
+    if (!SpawnChild(arg0, &process_info)) {
+        return false;
+    }
+
+    // Wait until the processs exits and get exit code from it
+    WaitForSingleObject(process_info.hProcess, INFINITE);
+    DWORD exit_code = STILL_ACTIVE;
+    const int err = GetExitCodeProcess(process_info.hProcess, &exit_code);
+    if (err == 0) {
+        std::fprintf(stderr, "GetExitCodeProcess failed with error %d\n", GetLastError());
+    }
+
+    // Vulkan is broken if the child crashed (return value is not zero)
+    *has_broken_vulkan = (exit_code != 0);
+
+    if (CloseHandle(process_info.hProcess) == 0) {
+        std::fprintf(stderr, "CloseHandle failed with error %d\n", GetLastError());
+    }
+    if (CloseHandle(process_info.hThread) == 0) {
+        std::fprintf(stderr, "CloseHandle failed with error %d\n", GetLastError());
+    }
+
+    if (!SetEnvironmentVariableA(STARTUP_CHECK_ENV_VAR, nullptr)) {
+        std::fprintf(stderr, "SetEnvironmentVariableA failed to clear %s with error %d\n",
+                     STARTUP_CHECK_ENV_VAR, GetLastError());
+    }
+
+#elif defined(YUZU_UNIX)
+    const pid_t pid = fork();
+    if (pid == 0) {
+        CheckVulkan();
+        return true;
+    } else if (pid == -1) {
+        const int err = errno;
+        std::fprintf(stderr, "fork failed with error %d\n", err);
+        return false;
+    }
+
+    // Get exit code from child process
+    int status;
+    const int r_val = wait(&status);
+    if (r_val == -1) {
+        const int err = errno;
+        std::fprintf(stderr, "wait failed with error %d\n", err);
+        return false;
+    }
+    // Vulkan is broken if the child crashed (return value is not zero)
+    *has_broken_vulkan = (status != 0);
+#endif
+    return false;
+}
+
+#ifdef _WIN32
+bool SpawnChild(const char* arg0, PROCESS_INFORMATION* pi) {
+    STARTUPINFOA startup_info;
+
+    std::memset(&startup_info, '\0', sizeof(startup_info));
+    startup_info.cb = sizeof(startup_info);
+
+    char p_name[255];
+    std::strncpy(p_name, arg0, 255);
+
+    const bool process_created = CreateProcessA(nullptr,       // lpApplicationName
+                                                p_name,        // lpCommandLine
+                                                nullptr,       // lpProcessAttributes
+                                                nullptr,       // lpThreadAttributes
+                                                false,         // bInheritHandles
+                                                0,             // dwCreationFlags
+                                                nullptr,       // lpEnvironment
+                                                nullptr,       // lpCurrentDirectory
+                                                &startup_info, // lpStartupInfo
+                                                pi             // lpProcessInformation
+    );
+    if (!process_created) {
+        std::fprintf(stderr, "CreateProcessA failed with error %d\n", GetLastError());
+        return false;
+    }
+
+    return true;
+}
+#endif