tls: enable support for encryption of memory pages

This patch enables encryption of memory pages using the AES-XTS.
This block cipher is more efficient when encrypting consecutive
blocks of data (memory pages) and allows the use of hardware
acceleration available in modern CPUs.

XTS uses two 256-bits AES keys. One key is used to perform block
encryption, and the other is used to encrypt a "tweak value".
The encrypted tweak is further modified with a Galois polynomial
function and XOR-ed with both the plain text and the cipher text
of each block. This ensures that encrypting multiple blocks with
identical data will produce different ciphertext.

Suggested-by: Daiki Ueno <[email protected]>
Signed-off-by: Radostin Stoyanov <[email protected]>

criu/include/restorer.h

@@ -238,6 +238,10 @@ struct task_restore_args {
 	 */
 	struct rst_rseq_param libc_rseq;
 
+	bool encrypted_pages;
+	int decryption_pipe_fd_r;
+	int decryption_pipe_fd_w;
+
 	uid_t uid;
 	u32 cap_eff[CR_CAP_SIZE];
 } __aligned(64);

criu/include/tls.h

@@ -24,6 +24,9 @@ int tls_encrypt_file_data(int fd_in, int fd_out, size_t data_size);
 int tls_decrypt_file_data(int fd_in, int fd_out, size_t data_size);
 int tls_encryption_pipe(int output_file_fd, int pipe_read_fd);
 int tls_decryption_pipe(int intput_file_fd, int pipe_write_fd);
+int tls_block_cipher_encrypt_data(void *ptext, size_t ptext_len);
+int tls_block_cipher_decrypt_data(void *ctext, size_t ctext_len);
+int tls_vma_io_pipe(int pages_img_fd, int pipe_read_fd, int pipe_write_fd);
 
 #else /* CONFIG_GNUTLS */
 
@@ -42,6 +45,9 @@ int tls_decryption_pipe(int intput_file_fd, int pipe_write_fd);
 #define tls_decrypt_file_data(fd_in, fd_out, data_size) (-1)
 #define tls_encryption_pipe(output_file_fd, pipe_read_fd) (-1)
 #define tls_decryption_pipe(intput_file_fd, pipe_write_fd) (-1)
+#define tls_block_cipher_encrypt_data(ptext, ptext_len)		(-1)
+#define tls_block_cipher_decrypt_data(ctext, ctext_len)		(-1)
+#define tls_vma_io_pipe(pages_img_fd, pipe_read_fd, pipe_write_fd) (-1)
 #define write_img_cipher() (0)
 
 #endif /* CONFIG_HAS_GNUTLS */

criu/mem.c

@@ -31,6 +31,7 @@
 #include "prctl.h"
 #include "compel/infect-util.h"
 #include "pidfd-store.h"
+#include "tls.h"
 
 #include "protobuf.h"
 #include "images/pagemap.pb-c.h"
@@ -1386,6 +1387,7 @@ int open_vmas(struct pstree_item *t)
 static int prepare_vma_ios(struct pstree_item *t, struct task_restore_args *ta)
 {
 	struct cr_img *pages;
+	int pipe_fds[2][2];
 
 	/*
 	 * We optimize the case when rsti(t)->vma_io is empty.
@@ -1410,6 +1412,33 @@ static int prepare_vma_ios(struct pstree_item *t, struct task_restore_args *ta)
 		return -1;
 
 	ta->vma_ios_fd = img_raw_fd(pages);
+
+	if (!opts.encrypt) {
+		ta->encrypted_pages = false;
+		ta->decryption_pipe_fd_w = -1;
+		ta->decryption_pipe_fd_r = -1;
+	} else {
+		ta->encrypted_pages = true;
+
+		if (pipe(pipe_fds[0])) {
+			pr_perror("Failed to create pipe");
+			return -1;
+		}
+		if (pipe(pipe_fds[1])) {
+			pr_perror("Failed to create pipe");
+			return -1;
+		}
+
+		if (tls_vma_io_pipe(ta->vma_ios_fd, pipe_fds[0][0], pipe_fds[1][1])) {
+			pr_err("Failed to setup VMA IO pipe\n");
+			return -1;
+		}
+		close(pipe_fds[0][0]);
+		close(pipe_fds[1][1]);
+		ta->decryption_pipe_fd_w = pipe_fds[0][1];
+		ta->decryption_pipe_fd_r = pipe_fds[1][0];
+	}
+
 	return pagemap_render_iovec(&rsti(t)->vma_io, ta);
 }
 

criu/page-xfer.c

@@ -257,19 +257,52 @@ static int write_pages_loc(struct page_xfer *xfer, int p, unsigned long len)
 	ssize_t ret;
 	ssize_t curr = 0;
 
-	while (1) {
-		ret = splice(p, NULL, img_raw_fd(xfer->pi), NULL, len - curr, SPLICE_F_MOVE);
-		if (ret == -1) {
-			pr_perror("Unable to spice data");
-			return -1;
+	if (opts.encrypt) {
+		uint8_t buf[PAGE_SIZE];
+
+		/* We encrypt each page separately to enable decryption
+		 * of arbitrary pages during restore. This is required
+		 * for auto-deduplication and incremental checkpointing.
+		 */
+		BUG_ON((len % PAGE_SIZE) != 0);
+
+		for (curr = 0; curr < len; curr += PAGE_SIZE) {
+			ret = read(p, buf, PAGE_SIZE);
+			if (ret < 0) {
+				pr_perror("Unable to read data");
+				return -1;
+			}
+			if (ret == 0) {
+				pr_err("A pipe was closed unexpectedly\n");
+				return -1;
+			}
+			BUG_ON(ret != PAGE_SIZE);
+
+			if (tls_block_cipher_encrypt_data(buf, PAGE_SIZE)) {
+				pr_err("Failed to encrypt data\n");
+				return -1;
+			}
+			ret = write(img_raw_fd(xfer->pi), buf, PAGE_SIZE);
+			if (ret != PAGE_SIZE) {
+				pr_perror("Unable to write data %zd", ret);
+				return -1;
+			}
 		}
-		if (ret == 0) {
-			pr_err("A pipe was closed unexpectedly\n");
-			return -1;
+	} else {
+		while (1) {
+			ret = splice(p, NULL, img_raw_fd(xfer->pi), NULL, len - curr, SPLICE_F_MOVE);
+			if (ret == -1) {
+				pr_perror("Unable to spice data");
+				return -1;
+			}
+			if (ret == 0) {
+				pr_err("A pipe was closed unexpectedly\n");
+				return -1;
+			}
+			curr += ret;
+			if (curr == len)
+				break;
 		}
-		curr += ret;
-		if (curr == len)
-			break;
 	}
 
 	return 0;

criu/pagemap.c

@@ -13,6 +13,7 @@
 #include "restorer.h"
 #include "rst-malloc.h"
 #include "page-xfer.h"
+#include "tls.h"
 
 #include "fault-injection.h"
 #include "xmalloc.h"
@@ -261,6 +262,15 @@ static int read_local_page(struct page_read *pr, unsigned long vaddr, unsigned l
 			break;
 	}
 
+	if (opts.encrypt) {
+		for (int i = 0; i < len; i += PAGE_SIZE) {
+			if (tls_block_cipher_decrypt_data(buf + i, PAGE_SIZE)) {
+				pr_err("Failed to decrypt data\n");
+				return -1;
+			}
+		}
+	}
+
 	if (opts.auto_dedup) {
 		ret = punch_hole(pr, pr->pi_off, len, false);
 		if (ret == -1)
@@ -559,6 +569,17 @@ static int process_async_reads(struct page_read *pr)
 			return -1;
 		}
 
+		if (opts.encrypt) {
+			for (int i = 0; i < piov->nr; i++) {
+				for (int j = 0; j < piov->to[i].iov_len; j += PAGE_SIZE) {
+					if (tls_block_cipher_decrypt_data(piov->to[i].iov_base + j, PAGE_SIZE)) {
+						pr_err("Failed to decrypt data\n");
+						exit(1);
+					}
+				}
+			}
+		}
+
 		if (opts.auto_dedup && punch_hole(pr, piov->from, ret, false))
 			return -1;
 

criu/pie/restorer.c

@@ -1517,6 +1517,75 @@ static ssize_t preadv_limited(int fd, struct iovec *iovs, int nr, off_t offs, si
 	return ret;
 }
 
+/**
+ * Similar to preadv_limited(), but uses pipes to communicate with
+ * a process running in parallel that decrypts the data as the gnutls
+ * library cannot be used in restorer.
+ */
+static ssize_t decrypt_preadv_limited(int rfd, int wfd, struct iovec *iovs, int nr, off_t offs, size_t max_to_read)
+{
+	size_t saved_last_iov_len = 0;
+	ssize_t ret;
+	ssize_t preadv_ret;
+	pid_t local_pid = sys_getpid();
+
+	if (max_to_read) {
+		for (int i = 0; i < nr; ++i) {
+			if (iovs[i].iov_len <= max_to_read) {
+				max_to_read -= iovs[i].iov_len;
+				continue;
+			}
+
+			if (!max_to_read) {
+				nr = i;
+				break;
+			}
+
+			saved_last_iov_len = iovs[i].iov_len;
+			iovs[i].iov_len = max_to_read;
+			nr = i + 1;
+			break;
+		}
+	}
+
+	ret = sys_write(wfd, &local_pid, sizeof(pid_t));
+	if (ret < 0) {
+		return -1;
+	}
+
+	ret = sys_write(wfd, &offs, sizeof(off_t));
+	if (ret < 0) {
+		return -1;
+	}
+
+	ret = sys_write(wfd, &nr, sizeof(int));
+	if (ret < 0) {
+		return -1;
+	}
+
+	for (int i = 0; i < nr; i++) {
+		ret = sys_write(wfd, &iovs[i].iov_len, sizeof(size_t));
+		if (ret < 0) {
+			return -1;
+		}
+
+		ret = sys_write(wfd, &iovs[i].iov_base, sizeof(void *));
+		if (ret < 0) {
+			return -1;
+		}
+	}
+
+	ret = sys_read(rfd, &preadv_ret, sizeof(ssize_t));
+	if (ret < 0) {
+		return -1;
+	}
+
+	if (saved_last_iov_len)
+		iovs[nr - 1].iov_len = saved_last_iov_len;
+
+	return preadv_ret;
+}
+
 /*
  * In the worst case buf size should be:
  *   sizeof(struct inotify_event) * 2 + PATH_MAX
@@ -1792,8 +1861,15 @@ __visible long __export_restore_task(struct task_restore_args *args)
 			 * If we're requested to punch holes in the file after reading we do
 			 * it to save memory. Limit the reads then to an arbitrary block size.
 			 */
-			r = preadv_limited(args->vma_ios_fd, iovs, nr, rio->off,
-					   args->auto_dedup ? AUTO_DEDUP_OVERHEAD_BYTES : 0);
+			if (args->encrypted_pages) {
+				r = decrypt_preadv_limited(args->decryption_pipe_fd_r, args->decryption_pipe_fd_w, iovs,
+							   nr, rio->off,
+							   args->auto_dedup ? AUTO_DEDUP_OVERHEAD_BYTES : 0);
+			} else {
+				r = preadv_limited(args->vma_ios_fd, iovs, nr, rio->off,
+						   args->auto_dedup ? AUTO_DEDUP_OVERHEAD_BYTES : 0);
+			}
+
 			if (r < 0) {
 				pr_err("Can't read pages data (%d)\n", (int)r);
 				goto core_restore_end;
@@ -1829,6 +1905,12 @@ __visible long __export_restore_task(struct task_restore_args *args)
 		rio = ((void *)rio) + RIO_SIZE(rio->nr_iovs);
 	}
 
+	if (args->encrypted_pages) {
+		pr_debug("Closing decryption pipe\n");
+		sys_close(args->decryption_pipe_fd_r);
+		sys_close(args->decryption_pipe_fd_w);
+	}
+
 	if (args->vma_ios_fd != -1)
 		sys_close(args->vma_ios_fd);