startblit() and endblit(): watch for mutex contention. the mutex is unfair on linux, so sleep if there's contention

src/qt/qt_mainwindow.cpp

@@ -905,7 +905,9 @@ video_toggle_option(QAction* action, int *val)
 
 void MainWindow::on_actionInverted_VGA_monitor_triggered() {
     video_toggle_option(ui->actionInverted_VGA_monitor, &invert_display);
+    startblit();
     video_copy = (video_grayscale || invert_display) ? video_transform_copy : memcpy;
+    endblit();
 }
 
 static void update_scaled_checkboxes(Ui::MainWindow* ui, QAction* selected) {

src/qt/qt_platform.cpp

@@ -1,6 +1,7 @@
 #include <cstdio>
 
 #include <mutex>
+#include <thread>
 #include <memory>
 #include <algorithm>
 
@@ -21,6 +22,8 @@
 // static QByteArray buf;
 extern QElapsedTimer elapsed_timer;
 QElapsedTimer elapsed_timer;
+
+static std::atomic_int blitmx_contention = 0;
 static std::mutex blitmx;
 
 class CharPointer {
@@ -382,12 +385,24 @@ void joystick_close(void) {}
 void joystick_process(void) {}
 void startblit()
 {
+    blitmx_contention++;
+    if (blitmx.try_lock()) {
+        return;
+    }
+
     blitmx.lock();
 }
 
 void endblit()
 {
+    blitmx_contention--;
     blitmx.unlock();
+    if (blitmx_contention > 0) {
+        // a deadlock has been observed on linux when toggling via video_toggle_option
+        // because the mutex is typically unfair on linux
+        // => sleep if there's contention
+        std::this_thread::sleep_for(std::chrono::milliseconds(1));
+    }
 }
 
 }