Fix dup3 syscall to correctly handle flags and O_CLOEXEC

src/lib/libsyscall.js

@@ -1015,16 +1015,18 @@ var SyscallsLibrary = {
     return 0;
   },
   __syscall_dup3: (fd, newfd, flags) => {
+    if (fd === newfd) return -{{{ cDefs.EINVAL }}};
+    if (flags & ~{{{ cDefs.O_CLOEXEC }}}) return -{{{ cDefs.EINVAL }}};
     var old = SYSCALLS.getStreamFromFD(fd);
-#if ASSERTIONS
-    assert(!flags);
-#endif
-    if (old.fd === newfd) return -{{{ cDefs.EINVAL }}};
     // Check newfd is within range of valid open file descriptors.
     if (newfd < 0 || newfd >= FS.MAX_OPEN_FDS) return -{{{ cDefs.EBADF }}};
     var existing = FS.getStream(newfd);
     if (existing) FS.close(existing);
-    return FS.dupStream(old, newfd).fd;
+    var stream = FS.dupStream(old, newfd);
+    if (flags & {{{ cDefs.O_CLOEXEC }}}) {
+      stream.flags |= {{{ cDefs.O_CLOEXEC }}};
+    }
+    return stream.fd;
   },
 };
 

system/lib/wasmfs/syscalls.cpp

@@ -50,8 +50,10 @@ extern "C" {
 using namespace wasmfs;
 
 int __syscall_dup3(int oldfd, int newfd, int flags) {
-  if (flags & !O_CLOEXEC) {
-    // TODO: Test this case.
+  if (flags & ~O_CLOEXEC) {
+    return -EINVAL;
+  }
+  if (oldfd == newfd) {
     return -EINVAL;
   }
 
@@ -63,9 +65,6 @@ int __syscall_dup3(int oldfd, int newfd, int flags) {
   if (newfd < 0 || newfd >= WASMFS_FD_MAX) {
     return -EBADF;
   }
-  if (oldfd == newfd) {
-    return -EINVAL;
-  }
 
   // If the file descriptor newfd was previously open, it will just be
   // overwritten silently.

test/codesize/test_codesize_hello_dylink_all.json

@@ -1,7 +1,7 @@
 {
-  "a.out.js": 244191,
+  "a.out.js": 244238,
   "a.out.nodebug.wasm": 577664,
-  "total": 821855,
+  "total": 821902,
   "sent": [
     "IMG_Init",
     "IMG_Load",

test/unistd/dup.c

@@ -86,6 +86,55 @@ int main() {
   assert(strcmp(buf, "abc") == 0);
   printf("\n");
 
+  printf("DUP3\n");
+  f = open("/", O_RDONLY);
+  f2 = open("/", O_RDONLY);
+  f3 = dup3(f, f2, O_CLOEXEC);
+  assert(f != -1);
+  assert(f2 != -1);
+  assert(f3 != -1);
+  printf("errno: %s\n", strerror(errno));
+  printf("f: %d\n", f != f2 && f != f3);
+  printf("f2,f3: %d\n", f2 == f3);
+  printf("close(f1): %d\n", close(f));
+  printf("close(f2): %d\n", close(f2));
+  printf("close(f3): %d\n", close(f3));
+  printf("\n");
+  errno = 0;
+
+  printf("DUP3 bad fds\n");
+  f = dup3(-2, -2, 0);
+  printf("f: %d\n", f);
+  assert(f == -1);
+  printf("errno: %s\n", strerror(errno));
+  printf("close(f): %d\n", close(f));
+  printf("\n");
+  errno = 0;
+
+  printf("DUP3 same fds\n");
+  f = open("/", O_RDONLY);
+  assert(f != -1);
+  f2 = dup3(f, f, 0);
+  printf("f2: %d\n", f2);
+  assert(f2 == -1);
+  printf("errno: %s\n", strerror(errno));
+  assert(errno == EINVAL);
+  close(f);
+  printf("\n");
+  errno = 0;
+
+  printf("DUP3 invalid flags\n");
+  f = open("/", O_RDONLY);
+  assert(f != -1);
+  f2 = dup3(f, 100, 0x12345678);
+  printf("f2: %d\n", f2);
+  assert(f2 == -1);
+  printf("errno: %s\n", strerror(errno));
+  assert(errno == EINVAL);
+  close(f);
+  printf("\n");
+  errno = 0;
+
   printf("DUP shared seek position\n");
   f = open("./blah.txt", O_RDWR | O_CREAT | O_EXCL, 0600);
   f2 = dup(f);

test/unistd/dup.out

@@ -29,6 +29,27 @@ close(f1): 0
 DUP2 pipe
 buf: abc
 
+DUP3
+errno: Success
+f: 1
+f2,f3: 1
+close(f1): 0
+close(f2): 0
+close(f3): -1
+
+DUP3 bad fds
+f: -1
+errno: Invalid argument
+close(f): -1
+
+DUP3 same fds
+f2: -1
+errno: Invalid argument
+
+DUP3 invalid flags
+f2: -1
+errno: Invalid argument
+
 DUP shared seek position
 close(f): 0
 close(f2): 0