work around broken kernel struct ipc_perm on some big endian archs

the mode member of struct ipc_perm is specified by POSIX to have type
mode_t, which is uniformly defined as unsigned int. however, Linux
defines it with type __kernel_mode_t, and defines __kernel_mode_t as
unsigned short on some archs. since there is a subsequent padding
field, treating it as a 32-bit unsigned int works on little endian
archs, but the order is backwards on big endian archs with the
erroneous definition.

since multiple archs are affected, remedy the situation with fixup
code in the affected functions (shmctl, semctl, and msgctl) rather
than repeating the same shims in syscall_arch.h for every affected
arch.

1 files changed, 24 insertions, 2 deletions

diff --git a/src/ipc/shmctl.c b/src/ipc/shmctl.c
index e2879f20..c951a581 100644
--- a/src/ipc/shmctl.c
+++ b/src/ipc/shmctl.c
@@ -1,12 +1,34 @@
 #include <sys/shm.h>
+#include <endian.h>
 #include "syscall.h"
 #include "ipc.h"
 
+#if __BYTE_ORDER != __BIG_ENDIAN
+#undef SYSCALL_IPC_BROKEN_MODE
+#endif
+
 int shmctl(int id, int cmd, struct shmid_ds *buf)
 {
+#ifdef SYSCALL_IPC_BROKEN_MODE
+	struct shmid_ds tmp;
+	if (cmd == IPC_SET) {
+		tmp = *buf;
+		tmp.shm_perm.mode *= 0x10000U;
+		buf = &tmp;
+	}
+#endif
 #ifdef SYS_shmctl
-	return syscall(SYS_shmctl, id, cmd | IPC_64, buf);
+	int r = __syscall(SYS_shmctl, id, cmd | IPC_64, buf);
 #else
-	return syscall(SYS_ipc, IPCOP_shmctl, id, cmd | IPC_64, 0, buf, 0);
+	int r = __syscall(SYS_ipc, IPCOP_shmctl, id, cmd | IPC_64, 0, buf, 0);
+#endif
+#ifdef SYSCALL_IPC_BROKEN_MODE
+	if (r >= 0) switch (cmd) {
+	case IPC_STAT:
+	case SHM_STAT:
+	case SHM_STAT_ANY:
+		buf->shm_perm.mode >>= 16;
+	}
 #endif
+	return __syscall_ret(r);
 }