shm: try with MFD_NOEXEC_SEAL first, then without

MFD_NOEXEC_SEAL is only supported on kernels 6.3 and later.

If we were compiled on linux >= 6.3, but run on linux < 6.3, we'd exit
with an error, due to memfd_create() failing with EINVAL.

This patch fixes the problem by first trying to call
memfd_create() *with* MFD_NOEXEC_SEAL, and if that fails with EINVAL,
we try again without it.

Also seal the memory FD once mmap() has been called.

1 files changed, 31 insertions, 2 deletions

diff --git a/shm.c b/shm.c
index adc91df..bd4ce26 100644
--- a/shm.c
+++ b/shm.c
@@ -1,7 +1,9 @@
 #include "shm.h"
 
-#include <unistd.h>
 #include <assert.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <unistd.h>
 
 #include <sys/types.h>
 #include <sys/mman.h>
@@ -13,6 +15,10 @@
 #include "log.h"
 #include "stride.h"
 
+#if !defined(MAP_UNINITIALIZED)
+ #define MAP_UNINITIALIZED 0
+#endif
+
 static void
 buffer_destroy(struct buffer *buf)
 {
@@ -53,7 +59,21 @@ shm_get_buffer(struct wl_shm *shm, int width, int height, unsigned long cookie)
     pixman_image_t *pix = NULL;
 
     /* Backing memory for SHM */
-    pool_fd = memfd_create("wbg-wayland-shm-buffer-pool", MFD_CLOEXEC);
+
+    /*
+     * Older kernels reject MFD_NOEXEC_SEAL with EINVAL. Try first
+     * *with* it, and if that fails, try again *without* it.
+     */
+    errno = 0;
+    pool_fd = memfd_create(
+        "wbg-wayland-shm-buffer-pool",
+        MFD_CLOEXEC | MFD_ALLOW_SEALING | MFD_NOEXEC_SEAL);
+
+    if (pool_fd < 0 && errno == EINVAL) {
+        pool_fd = memfd_create(
+            "wbg-wayland-shm-buffer-pool", MFD_CLOEXEC | MFD_ALLOW_SEALING);
+    }
+
     if (pool_fd == -1) {
         LOG_ERRNO("failed to create SHM backing memory file");
         goto err;
@@ -73,6 +93,15 @@ shm_get_buffer(struct wl_shm *shm, int width, int height, unsigned long cookie)
         goto err;
     }
 
+    /* Seal file - we no longer allow any kind of resizing */
+    /* TODO: wayland mmaps(PROT_WRITE), for some unknown reason, hence we cannot use F_SEAL_FUTURE_WRITE */
+    if (fcntl(pool_fd, F_ADD_SEALS,
+              F_SEAL_GROW | F_SEAL_SHRINK | /*F_SEAL_FUTURE_WRITE |*/ F_SEAL_SEAL) < 0)
+    {
+        LOG_ERRNO("failed to seal SHM backing memory file");
+        /* This is not a fatal error */
+    }
+
     pool = wl_shm_create_pool(shm, pool_fd, size);
     if (pool == NULL) {
         LOG_ERR("failed to create SHM pool");