diff --git a/libcontainer/setns_init_linux.go b/libcontainer/setns_init_linux.go
index d1885b3fd..e1cce1211 100644
--- a/libcontainer/setns_init_linux.go
+++ b/libcontainer/setns_init_linux.go
@@ -33,10 +33,12 @@ func (l *linuxSetnsInit) getSessionRingName() string {
 
 func (l *linuxSetnsInit) Init() error {
 	if !l.config.Config.NoNewKeyring {
-		if err := selinux.SetKeyLabel(l.config.ProcessLabel); err != nil {
-			return err
+		if l.config.ProcessLabel != "" {
+			if err := selinux.SetKeyLabel(l.config.ProcessLabel); err != nil {
+				return err
+			}
+			defer selinux.SetKeyLabel("") //nolint: errcheck
 		}
-		defer selinux.SetKeyLabel("") //nolint: errcheck
 		// Do not inherit the parent's session keyring.
 		if _, err := keys.JoinSessionKeyring(l.getSessionRingName()); err != nil {
 			// Same justification as in standart_init_linux.go as to why we
@@ -84,11 +86,12 @@ func (l *linuxSetnsInit) Init() error {
 	if err := syncParentReady(l.pipe); err != nil {
 		return fmt.Errorf("sync ready: %w", err)
 	}
-
-	if err := selinux.SetExecLabel(l.config.ProcessLabel); err != nil {
-		return err
+	if l.config.ProcessLabel != "" {
+		if err := selinux.SetExecLabel(l.config.ProcessLabel); err != nil {
+			return err
+		}
+		defer selinux.SetExecLabel("") //nolint: errcheck
 	}
-	defer selinux.SetExecLabel("") //nolint: errcheck
 	// Without NoNewPrivileges seccomp is a privileged operation, so we need to
 	// do this before dropping capabilities; otherwise do it as late as possible
 	// just before execve so as few syscalls take place after it as possible.
diff --git a/libcontainer/standard_init_linux.go b/libcontainer/standard_init_linux.go
index 9517820bc..8ea9e9457 100644
--- a/libcontainer/standard_init_linux.go
+++ b/libcontainer/standard_init_linux.go
@@ -47,10 +47,12 @@ func (l *linuxStandardInit) getSessionRingParams() (string, uint32, uint32) {
 
 func (l *linuxStandardInit) Init() error {
 	if !l.config.Config.NoNewKeyring {
-		if err := selinux.SetKeyLabel(l.config.ProcessLabel); err != nil {
-			return err
+		if l.config.ProcessLabel != "" {
+			if err := selinux.SetKeyLabel(l.config.ProcessLabel); err != nil {
+				return err
+			}
+			defer selinux.SetKeyLabel("") //nolint: errcheck
 		}
-		defer selinux.SetKeyLabel("") //nolint: errcheck
 		ringname, keepperms, newperms := l.getSessionRingParams()
 
 		// Do not inherit the parent's session keyring.
@@ -169,10 +171,12 @@ func (l *linuxStandardInit) Init() error {
 	if err := syncParentReady(l.pipe); err != nil {
 		return fmt.Errorf("sync ready: %w", err)
 	}
-	if err := selinux.SetExecLabel(l.config.ProcessLabel); err != nil {
-		return fmt.Errorf("can't set process label: %w", err)
+	if l.config.ProcessLabel != "" {
+		if err := selinux.SetExecLabel(l.config.ProcessLabel); err != nil {
+			return fmt.Errorf("can't set process label: %w", err)
+		}
+		defer selinux.SetExecLabel("") //nolint: errcheck
 	}
-	defer selinux.SetExecLabel("") //nolint: errcheck
 	// Without NoNewPrivileges seccomp is a privileged operation, so we need to
 	// do this before dropping capabilities; otherwise do it as late as possible
 	// just before execve so as few syscalls take place after it as possible.