diff --git a/libcontainer/container_linux.go b/libcontainer/container_linux.go
index b790c0714..26f523a90 100644
--- a/libcontainer/container_linux.go
+++ b/libcontainer/container_linux.go
@@ -302,6 +302,13 @@ func (c *Container) start(process *Process) (retErr error) {
 	if c.config.Cgroups.Resources.SkipDevices {
 		return errors.New("can't start container with SkipDevices set")
 	}
+
+	if c.config.RootlessEUID && len(process.AdditionalGroups) > 0 {
+		// We cannot set any additional groups in a rootless container
+		// and thus we bail if the user asked us to do so.
+		return errors.New("cannot set any additional groups in a rootless container")
+	}
+
 	if process.Init {
 		if c.initProcessStartTime != 0 {
 			return errors.New("container already has init process")
diff --git a/libcontainer/init_linux.go b/libcontainer/init_linux.go
index 75bef0315..cff21e1bc 100644
--- a/libcontainer/init_linux.go
+++ b/libcontainer/init_linux.go
@@ -438,14 +438,6 @@ func syncParentSeccomp(pipe *syncSocket, seccompFd int) error {
 
 // setupUser changes the groups, gid, and uid for the user inside the container.
 func setupUser(config *initConfig) error {
-	if config.RootlessEUID && len(config.AdditionalGroups) > 0 {
-		// We cannot set any additional groups in a rootless container and thus
-		// we bail if the user asked us to do so. TODO: We currently can't do
-		// this check earlier, but if libcontainer.Process.User was typesafe
-		// this might work.
-		return errors.New("cannot set any additional groups in a rootless container")
-	}
-
 	// Before we change to the container's user make sure that the processes
 	// STDIO is correctly owned by the user that we are switching to.
 	if err := fixStdioPermissions(config.UID); err != nil {