diff --git a/libcontainer/nsenter/cloned_binary.c b/libcontainer/nsenter/cloned_binary.c index 73c0a27b5..06c7b00b9 100644 --- a/libcontainer/nsenter/cloned_binary.c +++ b/libcontainer/nsenter/cloned_binary.c @@ -49,6 +49,7 @@ #include #include +#include #include #include #include @@ -56,7 +57,12 @@ #include #include #include +#include #include +#include + +#include "ipc.h" +#include "log.h" /* Use our own wrapper for memfd_create. */ #ifndef SYS_memfd_create @@ -394,6 +400,123 @@ static int seal_execfd(int *fd, int fdtype) return -1; } +struct bindfd_child_args { + int sockfd; + const char *mount_target; +}; + +static int bindfd_in_subprocess(void *arg) +{ + /* + * In the interests of efficiency (read: minimizing the syscall count) + * and conciseness, no attempt is made to release resources which would + * be cleaned up automatically on process exit, i.e. when this function + * returns. This includes filesystem mounts, as this function is + * executed in a dedicated mount namespace. + */ + + /* + * For obvious reasons this won't work in rootless mode because we + * haven't created a userns -- but getting that to work will be a bit + * complicated and it's only worth doing if someone actually needs it. + */ + if (mount("none", "/", NULL, MS_SLAVE | MS_REC, NULL) < 0) + return errno; + /* + * The kernel resolves the magic symlink /proc/self/exe to the real file + * _in the original mount namespace_. Cross-namespace bind mounts are + * not allowed, so we must locate the file inside the current mount + * namespace to be able to bind-mount it. (The mount(8) command resolves + * symlinks, which is why it appears to work at first glance.) + */ + char linkbuf[PATH_MAX + 1] = { 0 }; + ssize_t linkpathlen = readlink("/proc/self/exe", linkbuf, sizeof(linkbuf)); + if (linkpathlen < 0) + return errno; + if (linkpathlen == sizeof(linkbuf)) { + /* + * The link path is longer than PATH_MAX, and the contents of + * linkbuf might have been truncated. A truncated path could + * happen to be a valid path to a different file, which could + * allow for local privilege escalation if we were to exec it. + * The mount syscall doesn't accept paths longer than PATH_MAX, + * anyway. + */ + return ENAMETOOLONG; + } + + int srcfd = open(linkbuf, O_PATH | O_CLOEXEC); + if (srcfd < 0) + return errno; + /* + * linkbuf holds the path to the binary which the parent process was + * launched from. Someone could have moved a different file to that path + * in the interim, in which case srcfd is not the file we want to + * bind-mount. Guard against this situation by verifying srcfd is the + * same file as /proc/self/exe. + */ + struct stat realexe = { 0 }; + if (stat("/proc/self/exe", &realexe) < 0) + return errno; + struct stat resolved = { 0 }; + if (fstat(srcfd, &resolved) < 0) + return errno; + if (resolved.st_dev != realexe.st_dev || resolved.st_ino != realexe.st_ino) + return ENOENT; + if (snprintf(linkbuf, sizeof(linkbuf), "/proc/self/fd/%d", srcfd) == sizeof(linkbuf)) + return ENAMETOOLONG; + + const struct bindfd_child_args *args = arg; + if (mount(linkbuf, args->mount_target, "", MS_BIND, "") < 0) + return errno; + if (mount("", args->mount_target, "", MS_REMOUNT | MS_BIND | MS_RDONLY, "") < 0) + return errno; + + int fd = open(args->mount_target, O_PATH | O_CLOEXEC); + if (fd < 0) + return errno; + + /* + * Make sure the MNT_DETACH works, otherwise we could get remounted + * read-write and that would be quite bad. + */ + if (umount2(args->mount_target, MNT_DETACH) < 0) + return errno; + + if (send_fd(args->sockfd, fd) < 0) + return errno; + return 0; +} + +static int spawn_bindfd_child(const struct bindfd_child_args *args) __attribute__((noinline)); +static int spawn_bindfd_child(const struct bindfd_child_args *args) +{ + /* + * Carve out a chunk of our call stack for the child process to use as + * we can be sure it is correctly mapped for use as stack. (Technically + * only the libc clone() wrapper writes to this buffer. The child + * process operates on a copy of the parent's virtual memory space and + * so can safely overflow into the rest of the stack memory region + * without consequence.) + */ + char stack[4 * 1024] __attribute__((aligned(16))); + int tid = clone(bindfd_in_subprocess, + /* + * Assume stack grows down, as HP-PA, the only Linux + * platform where stack grows up, is obsolete. + */ + stack + sizeof(stack), + /* + * Suspend the parent process until the child has exited to + * save an unnecessary context switch as we'd just be + * waiting for the child process to exit anyway. + */ + CLONE_NEWNS | CLONE_VFORK, (void *)args); + if (tid < 0) + return -errno; + return tid; +} + static int try_bindfd(void) { int fd, ret = -1; @@ -415,32 +538,53 @@ static int try_bindfd(void) close(fd); /* - * For obvious reasons this won't work in rootless mode because we haven't - * created a userns+mntns -- but getting that to work will be a bit - * complicated and it's only worth doing if someone actually needs it. + * Daemons such as systemd and udisks2 watch /proc/self/mountinfo and + * re-parse it on every change, which gets expensive when the mount table + * is large and/or changes frequently. Perform the mount operations in a + * new, private mount namespace so as not to wake up those processes + * every time we nsexec into a container. We clone a child process into + * a new mount namespace to do the dirty work so the side effects of + * unsharing the mount namespace do not leak into the current process. */ - ret = -EPERM; - if (mount("/proc/self/exe", template, "", MS_BIND, "") < 0) - goto out; - if (mount("", template, "", MS_REMOUNT | MS_BIND | MS_RDONLY, "") < 0) - goto out_umount; - - /* Get read-only handle that we're sure can't be made read-write. */ - ret = open(template, O_PATH | O_CLOEXEC); - -out_umount: - /* - * Make sure the MNT_DETACH works, otherwise we could get remounted - * read-write and that would be quite bad (the fd would be made read-write - * too, invalidating the protection). - */ - if (umount2(template, MNT_DETACH) < 0) { - if (ret >= 0) - close(ret); - ret = -ENOTRECOVERABLE; + int sock[2]; + if (socketpair(AF_LOCAL, SOCK_STREAM, 0, sock) < 0) { + ret = -errno; + goto cleanup_unlink; } -out: + struct bindfd_child_args args = { + .sockfd = sock[0], + .mount_target = template, + }; + int cpid = spawn_bindfd_child(&args); + close(sock[0]); + if (cpid < 0) { + ret = cpid; + goto cleanup_socketpair; + } + + int wstatus = 0; + if (waitpid(cpid, &wstatus, __WCLONE) < 0) + bail("error waiting for bindfd child process to exit"); + if (WIFEXITED(wstatus)) { + if (WEXITSTATUS(wstatus)) { + ret = -WEXITSTATUS(wstatus); + goto cleanup_socketpair; + } + } else if (WIFSIGNALED(wstatus)) { + int sig = WTERMSIG(wstatus); + bail("bindfd child process terminated by signal %d (%s)", sig, strsignal(sig)); + } else { + /* Should never happen... */ + bail("unexpected waitpid() status for bindfd child process: 0x%x", wstatus); + } + + ret = receive_fd(sock[1]); + +cleanup_socketpair: + close(sock[1]); + +cleanup_unlink: /* * We don't care about unlink errors, the worst that happens is that * there's an empty file left around in STATEDIR. diff --git a/libcontainer/nsenter/getenv.c b/libcontainer/nsenter/getenv.c new file mode 100644 index 000000000..b0ee6ac78 --- /dev/null +++ b/libcontainer/nsenter/getenv.c @@ -0,0 +1,27 @@ +#define _GNU_SOURCE +#include +#include +#include "getenv.h" +#include "log.h" + +int getenv_int(const char *name) +{ + char *val, *endptr; + int ret; + + val = getenv(name); + /* Treat empty value as unset variable. */ + if (val == NULL || *val == '\0') + return -ENOENT; + + ret = strtol(val, &endptr, 10); + if (val == endptr || *endptr != '\0') + bail("unable to parse %s=%s", name, val); + /* + * Sanity check: this must be a non-negative number. + */ + if (ret < 0) + bail("bad value for %s=%s (%d)", name, val, ret); + + return ret; +} diff --git a/libcontainer/nsenter/getenv.h b/libcontainer/nsenter/getenv.h new file mode 100644 index 000000000..6f66f34c5 --- /dev/null +++ b/libcontainer/nsenter/getenv.h @@ -0,0 +1,13 @@ +#ifndef NSENTER_GETENV_H +#define NSENTER_GETENV_H + +/* + * Returns an environment variable value as a non-negative integer, or -ENOENT + * if the variable was not found or has an empty value. + * + * If the value can not be converted to an integer, or the result is out of + * range, the function bails out. + */ +int getenv_int(const char *name); + +#endif /* NSENTER_GETENV_H */ diff --git a/libcontainer/nsenter/ipc.c b/libcontainer/nsenter/ipc.c new file mode 100644 index 000000000..01321a718 --- /dev/null +++ b/libcontainer/nsenter/ipc.c @@ -0,0 +1,84 @@ +#define _GNU_SOURCE +#include +#include +#include +#include +#include "ipc.h" +#include "log.h" + +int receive_fd(int sockfd) +{ + int bytes_read; + struct msghdr msg = { }; + struct cmsghdr *cmsg; + struct iovec iov = { }; + char null_byte = '\0'; + int ret; + int fd_count; + + iov.iov_base = &null_byte; + iov.iov_len = 1; + + msg.msg_iov = &iov; + msg.msg_iovlen = 1; + + msg.msg_controllen = CMSG_SPACE(sizeof(int)); + msg.msg_control = malloc(msg.msg_controllen); + if (msg.msg_control == NULL) { + bail("Can't allocate memory to receive fd."); + } + + memset(msg.msg_control, 0, msg.msg_controllen); + + bytes_read = recvmsg(sockfd, &msg, MSG_CMSG_CLOEXEC); + if (bytes_read != 1) + bail("failed to receive fd from unix socket %d", sockfd); + if (msg.msg_flags & MSG_CTRUNC) + bail("received truncated control message from unix socket %d", sockfd); + + cmsg = CMSG_FIRSTHDR(&msg); + if (!cmsg) + bail("received message from unix socket %d without control message", sockfd); + + if (cmsg->cmsg_level != SOL_SOCKET) + bail("received unknown control message from unix socket %d: cmsg_level=%d", sockfd, cmsg->cmsg_level); + + if (cmsg->cmsg_type != SCM_RIGHTS) + bail("received unknown control message from unix socket %d: cmsg_type=%d", sockfd, cmsg->cmsg_type); + + fd_count = (cmsg->cmsg_len - CMSG_LEN(0)) / sizeof(int); + if (fd_count != 1) + bail("received control message from unix socket %d with too many fds: %d", sockfd, fd_count); + + ret = *(int *)CMSG_DATA(cmsg); + free(msg.msg_control); + return ret; +} + +int send_fd(int sockfd, int fd) +{ + struct msghdr msg = { }; + struct cmsghdr *cmsg; + struct iovec iov[1] = { }; + char null_byte = '\0'; + + iov[0].iov_base = &null_byte; + iov[0].iov_len = 1; + + msg.msg_iov = iov; + msg.msg_iovlen = 1; + + /* We send only one fd as specified by cmsg->cmsg_len below, even + * though msg.msg_controllen might have more space due to alignment. */ + msg.msg_controllen = CMSG_SPACE(sizeof(int)); + msg.msg_control = alloca(msg.msg_controllen); + memset(msg.msg_control, 0, msg.msg_controllen); + + cmsg = CMSG_FIRSTHDR(&msg); + cmsg->cmsg_level = SOL_SOCKET; + cmsg->cmsg_type = SCM_RIGHTS; + cmsg->cmsg_len = CMSG_LEN(sizeof(int)); + memcpy(CMSG_DATA(cmsg), &fd, sizeof(int)); + + return sendmsg(sockfd, &msg, 0); +} diff --git a/libcontainer/nsenter/ipc.h b/libcontainer/nsenter/ipc.h new file mode 100644 index 000000000..6e5972697 --- /dev/null +++ b/libcontainer/nsenter/ipc.h @@ -0,0 +1,12 @@ +#ifndef NSENTER_IPC_H +#define NSENTER_IPC_H + +int receive_fd(int sockfd); + +/* + * send_fd passes the open file descriptor fd to another process via the UNIX + * domain socket sockfd. The return value of the sendmsg(2) call is returned. + */ +int send_fd(int sockfd, int fd); + +#endif /* NSENTER_IPC_H */ diff --git a/libcontainer/nsenter/log.c b/libcontainer/nsenter/log.c new file mode 100644 index 000000000..086b53983 --- /dev/null +++ b/libcontainer/nsenter/log.c @@ -0,0 +1,83 @@ +#define _GNU_SOURCE +#include +#include +#include +#include +#include + +#include "log.h" +#include "getenv.h" + +static const char *level_str[] = { "panic", "fatal", "error", "warning", "info", "debug", "trace" }; + +int logfd = -1; +static int loglevel = DEBUG; + +extern char *escape_json_string(char *str); +void setup_logpipe(void) +{ + int i; + + i = getenv_int("_LIBCONTAINER_LOGPIPE"); + if (i < 0) { + /* We are not runc init, or log pipe was not provided. */ + return; + } + logfd = i; + + i = getenv_int("_LIBCONTAINER_LOGLEVEL"); + if (i < 0) + return; + loglevel = i; +} + +/* Defined in nsexec.c */ +extern int current_stage; + +void write_log(int level, const char *format, ...) +{ + char *message = NULL, *stage = NULL, *json = NULL; + va_list args; + int ret; + + if (logfd < 0 || level > loglevel) + goto out; + + va_start(args, format); + ret = vasprintf(&message, format, args); + va_end(args); + if (ret < 0) { + message = NULL; + goto out; + } + + message = escape_json_string(message); + + if (current_stage < 0) { + stage = strdup("nsexec"); + if (stage == NULL) + goto out; + } else { + ret = asprintf(&stage, "nsexec-%d", current_stage); + if (ret < 0) { + stage = NULL; + goto out; + } + } + ret = asprintf(&json, "{\"level\":\"%s\", \"msg\": \"%s[%d]: %s\"}\n", + level_str[level], stage, getpid(), message); + if (ret < 0) { + json = NULL; + goto out; + } + + /* This logging is on a best-effort basis. In case of a short or failed + * write there is nothing we can do, so just ignore write() errors. + */ + ssize_t __attribute__((unused)) __res = write(logfd, json, ret); + +out: + free(message); + free(stage); + free(json); +} diff --git a/libcontainer/nsenter/log.h b/libcontainer/nsenter/log.h new file mode 100644 index 000000000..1fe95a111 --- /dev/null +++ b/libcontainer/nsenter/log.h @@ -0,0 +1,37 @@ +#ifndef NSENTER_LOG_H +#define NSENTER_LOG_H + +#include + +/* + * Log levels are the same as in logrus. + */ +#define PANIC 0 +#define FATAL 1 +#define ERROR 2 +#define WARNING 3 +#define INFO 4 +#define DEBUG 5 +#define TRACE 6 + +/* + * Sets up logging by getting log fd and log level from the environment, + * if available. + */ +void setup_logpipe(void); + +void write_log(int level, const char *format, ...) __attribute__((format(printf, 2, 3))); + +extern int logfd; +#define bail(fmt, ...) \ + do { \ + if (logfd < 0) \ + fprintf(stderr, "FATAL: " fmt ": %m\n", \ + ##__VA_ARGS__); \ + else \ + write_log(FATAL, fmt ": %m", ##__VA_ARGS__); \ + exit(1); \ + } while(0) + + +#endif /* NSENTER_LOG_H */ diff --git a/libcontainer/nsenter/nsexec.c b/libcontainer/nsenter/nsexec.c index 0e60bef47..748791d6b 100644 --- a/libcontainer/nsenter/nsexec.c +++ b/libcontainer/nsenter/nsexec.c @@ -27,11 +27,12 @@ #include #include +#include "getenv.h" +#include "ipc.h" +#include "log.h" /* Get all of the CLONE_NEW* flags. */ #include "namespace.h" -extern char *escape_json_string(char *str); - /* Synchronisation values. */ enum sync_t { SYNC_USERMAP_PLS = 0x40, /* Request parent to map our users. */ @@ -96,22 +97,6 @@ struct nlconfig_t { size_t mountsources_len; }; -/* - * Log levels are the same as in logrus. - */ -#define PANIC 0 -#define FATAL 1 -#define ERROR 2 -#define WARNING 3 -#define INFO 4 -#define DEBUG 5 -#define TRACE 6 - -static const char *level_str[] = { "panic", "fatal", "error", "warning", "info", "debug", "trace" }; - -static int logfd = -1; -static int loglevel = DEBUG; - /* * List of netlink message types sent to us as part of bootstrapping the init. * These constants are defined in libcontainer/message_linux.go. @@ -149,67 +134,9 @@ int setns(int fd, int nstype) } #endif -static void write_log(int level, const char *format, ...) -{ - char *message = NULL, *stage = NULL, *json = NULL; - va_list args; - int ret; - - if (logfd < 0 || level > loglevel) - goto out; - - va_start(args, format); - ret = vasprintf(&message, format, args); - va_end(args); - if (ret < 0) { - message = NULL; - goto out; - } - - message = escape_json_string(message); - - if (current_stage == STAGE_SETUP) { - stage = strdup("nsexec"); - if (stage == NULL) - goto out; - } else { - ret = asprintf(&stage, "nsexec-%d", current_stage); - if (ret < 0) { - stage = NULL; - goto out; - } - } - ret = asprintf(&json, "{\"level\":\"%s\", \"msg\": \"%s[%d]: %s\"}\n", - level_str[level], stage, getpid(), message); - if (ret < 0) { - json = NULL; - goto out; - } - - /* This logging is on a best-effort basis. In case of a short or failed - * write there is nothing we can do, so just ignore write() errors. - */ - ssize_t __attribute__((unused)) __res = write(logfd, json, ret); - -out: - free(message); - free(stage); - free(json); -} - /* XXX: This is ugly. */ static int syncfd = -1; -#define bail(fmt, ...) \ - do { \ - if (logfd < 0) \ - fprintf(stderr, "FATAL: " fmt ": %m\n", \ - ##__VA_ARGS__); \ - else \ - write_log(FATAL, fmt ": %m", ##__VA_ARGS__); \ - exit(1); \ - } while(0) - static int write_file(char *data, size_t data_len, char *pathfmt, ...) { int fd, len, ret = 0; @@ -397,56 +324,6 @@ static int clone_parent(jmp_buf *env, int jmpval) return clone(child_func, ca.stack_ptr, CLONE_PARENT | SIGCHLD, &ca); } -/* - * Returns an environment variable value as a non-negative integer, or -ENOENT - * if the variable was not found or has an empty value. - * - * If the value can not be converted to an integer, or the result is out of - * range, the function bails out. - */ -static int getenv_int(const char *name) -{ - char *val, *endptr; - int ret; - - val = getenv(name); - /* Treat empty value as unset variable. */ - if (val == NULL || *val == '\0') - return -ENOENT; - - ret = strtol(val, &endptr, 10); - if (val == endptr || *endptr != '\0') - bail("unable to parse %s=%s", name, val); - /* - * Sanity check: this must be a non-negative number. - */ - if (ret < 0) - bail("bad value for %s=%s (%d)", name, val, ret); - - return ret; -} - -/* - * Sets up logging by getting log fd and log level from the environment, - * if available. - */ -static void setup_logpipe(void) -{ - int i; - - i = getenv_int("_LIBCONTAINER_LOGPIPE"); - if (i < 0) { - /* We are not runc init, or log pipe was not provided. */ - return; - } - logfd = i; - - i = getenv_int("_LIBCONTAINER_LOGLEVEL"); - if (i < 0) - return; - loglevel = i; -} - /* Returns the clone(2) flag for a namespace, given the name of a namespace. */ static int nsflag(char *name) { @@ -645,101 +522,6 @@ static inline int sane_kill(pid_t pid, int signum) return 0; } -void receive_fd(int sockfd, int new_fd) -{ - int bytes_read; - struct msghdr msg = { }; - struct cmsghdr *cmsg; - struct iovec iov = { }; - char null_byte = '\0'; - int ret; - int fd_count; - int *fd_payload; - - iov.iov_base = &null_byte; - iov.iov_len = 1; - - msg.msg_iov = &iov; - msg.msg_iovlen = 1; - - msg.msg_controllen = CMSG_SPACE(sizeof(int)); - msg.msg_control = malloc(msg.msg_controllen); - if (msg.msg_control == NULL) { - bail("Can't allocate memory to receive fd."); - } - - memset(msg.msg_control, 0, msg.msg_controllen); - - bytes_read = recvmsg(sockfd, &msg, 0); - if (bytes_read != 1) - bail("failed to receive fd from unix socket %d", sockfd); - if (msg.msg_flags & MSG_CTRUNC) - bail("received truncated control message from unix socket %d", sockfd); - - cmsg = CMSG_FIRSTHDR(&msg); - if (!cmsg) - bail("received message from unix socket %d without control message", sockfd); - - if (cmsg->cmsg_level != SOL_SOCKET) - bail("received unknown control message from unix socket %d: cmsg_level=%d", sockfd, cmsg->cmsg_level); - - if (cmsg->cmsg_type != SCM_RIGHTS) - bail("received unknown control message from unix socket %d: cmsg_type=%d", sockfd, cmsg->cmsg_type); - - fd_count = (cmsg->cmsg_len - CMSG_LEN(0)) / sizeof(int); - if (fd_count != 1) - bail("received control message from unix socket %d with too many fds: %d", sockfd, fd_count); - - fd_payload = (int *)CMSG_DATA(cmsg); - ret = dup3(*fd_payload, new_fd, O_CLOEXEC); - if (ret < 0) - bail("cannot dup3 fd %d to %d", *fd_payload, new_fd); - - free(msg.msg_control); - - ret = close(*fd_payload); - if (ret < 0) - bail("cannot close fd %d", *fd_payload); -} - -void send_fd(int sockfd, int fd) -{ - int bytes_written; - struct msghdr msg = { }; - struct cmsghdr *cmsg; - struct iovec iov[1] = { }; - char null_byte = '\0'; - - iov[0].iov_base = &null_byte; - iov[0].iov_len = 1; - - msg.msg_iov = iov; - msg.msg_iovlen = 1; - - /* We send only one fd as specified by cmsg->cmsg_len below, even - * though msg.msg_controllen might have more space due to alignment. */ - msg.msg_controllen = CMSG_SPACE(sizeof(int)); - msg.msg_control = malloc(msg.msg_controllen); - if (msg.msg_control == NULL) { - bail("Can't allocate memory to send fd."); - } - - memset(msg.msg_control, 0, msg.msg_controllen); - - cmsg = CMSG_FIRSTHDR(&msg); - cmsg->cmsg_level = SOL_SOCKET; - cmsg->cmsg_type = SCM_RIGHTS; - cmsg->cmsg_len = CMSG_LEN(sizeof(int)); - memcpy(CMSG_DATA(cmsg), &fd, sizeof(int)); - - bytes_written = sendmsg(sockfd, &msg, 0); - - free(msg.msg_control); - - if (bytes_written != 1) - bail("failed to send fd %d via unix socket %d", fd, sockfd); -} - void receive_mountsources(int sockfd) { char *mount_fds, *endp; @@ -770,7 +552,13 @@ void receive_mountsources(int sockfd) bail("malformed _LIBCONTAINER_MOUNT_FDS env var: fds out of range"); } - receive_fd(sockfd, new_fd); + int recv_fd = receive_fd(sockfd); + if (dup3(recv_fd, new_fd, O_CLOEXEC) < 0) { + bail("cannot dup3 fd %d to %ld", recv_fd, new_fd); + } + if (close(recv_fd) < 0) { + bail("cannot close fd %d", recv_fd); + } } } @@ -813,7 +601,8 @@ void send_mountsources(int sockfd, pid_t child, char *mountsources, size_t mount bail("failed to open mount source %s", mountsources); write_log(DEBUG, "~> sending fd for: %s", mountsources); - send_fd(sockfd, fd); + if (send_fd(sockfd, fd) < 0) + bail("failed to send fd %d via unix socket %d", fd, sockfd); ret = close(fd); if (ret != 0)