]>
Commit | Line | Data |
---|---|---|
5d752600 MK |
1 | /* |
2 | * memfd_create system call and file sealing support | |
3 | * | |
4 | * Code was originally included in shmem.c, and broken out to facilitate | |
5 | * use by hugetlbfs as well as tmpfs. | |
6 | * | |
7 | * This file is released under the GPL. | |
8 | */ | |
9 | ||
10 | #include <linux/fs.h> | |
11 | #include <linux/vfs.h> | |
12 | #include <linux/pagemap.h> | |
13 | #include <linux/file.h> | |
14 | #include <linux/mm.h> | |
15 | #include <linux/sched/signal.h> | |
16 | #include <linux/khugepaged.h> | |
17 | #include <linux/syscalls.h> | |
18 | #include <linux/hugetlb.h> | |
19 | #include <linux/shmem_fs.h> | |
20 | #include <linux/memfd.h> | |
105ff533 | 21 | #include <linux/pid_namespace.h> |
5d752600 MK |
22 | #include <uapi/linux/memfd.h> |
23 | ||
24 | /* | |
2313216f | 25 | * We need a tag: a new tag would expand every xa_node by 8 bytes, |
5d752600 MK |
26 | * so reuse a tag which we firmly believe is never set or cleared on tmpfs |
27 | * or hugetlbfs because they are memory only filesystems. | |
28 | */ | |
29 | #define MEMFD_TAG_PINNED PAGECACHE_TAG_TOWRITE | |
30 | #define LAST_SCAN 4 /* about 150ms max */ | |
31 | ||
b4d02baa DH |
32 | static bool memfd_folio_has_extra_refs(struct folio *folio) |
33 | { | |
34 | return folio_ref_count(folio) - folio_mapcount(folio) != | |
35 | folio_nr_pages(folio); | |
36 | } | |
37 | ||
ef3038a5 | 38 | static void memfd_tag_pins(struct xa_state *xas) |
5d752600 | 39 | { |
b4d02baa | 40 | struct folio *folio; |
f2b277c4 | 41 | int latency = 0; |
5d752600 MK |
42 | |
43 | lru_add_drain(); | |
5d752600 | 44 | |
ef3038a5 | 45 | xas_lock_irq(xas); |
b4d02baa DH |
46 | xas_for_each(xas, folio, ULONG_MAX) { |
47 | if (!xa_is_value(folio) && memfd_folio_has_extra_refs(folio)) | |
ef3038a5 | 48 | xas_set_mark(xas, MEMFD_TAG_PINNED); |
5d752600 | 49 | |
b4d02baa | 50 | if (++latency < XA_CHECK_SCHED) |
ef3038a5 | 51 | continue; |
f2b277c4 | 52 | latency = 0; |
ef3038a5 MW |
53 | |
54 | xas_pause(xas); | |
55 | xas_unlock_irq(xas); | |
56 | cond_resched(); | |
57 | xas_lock_irq(xas); | |
5d752600 | 58 | } |
ef3038a5 | 59 | xas_unlock_irq(xas); |
5d752600 MK |
60 | } |
61 | ||
62 | /* | |
63 | * Setting SEAL_WRITE requires us to verify there's no pending writer. However, | |
64 | * via get_user_pages(), drivers might have some pending I/O without any active | |
b4d02baa | 65 | * user-space mappings (eg., direct-IO, AIO). Therefore, we look at all folios |
5d752600 MK |
66 | * and see whether it has an elevated ref-count. If so, we tag them and wait for |
67 | * them to be dropped. | |
68 | * The caller must guarantee that no new user will acquire writable references | |
b4d02baa | 69 | * to those folios to avoid races. |
5d752600 MK |
70 | */ |
71 | static int memfd_wait_for_pins(struct address_space *mapping) | |
72 | { | |
2313216f | 73 | XA_STATE(xas, &mapping->i_pages, 0); |
b4d02baa | 74 | struct folio *folio; |
5d752600 MK |
75 | int error, scan; |
76 | ||
ef3038a5 | 77 | memfd_tag_pins(&xas); |
5d752600 MK |
78 | |
79 | error = 0; | |
80 | for (scan = 0; scan <= LAST_SCAN; scan++) { | |
f2b277c4 | 81 | int latency = 0; |
2313216f MW |
82 | |
83 | if (!xas_marked(&xas, MEMFD_TAG_PINNED)) | |
5d752600 MK |
84 | break; |
85 | ||
86 | if (!scan) | |
87 | lru_add_drain_all(); | |
88 | else if (schedule_timeout_killable((HZ << scan) / 200)) | |
89 | scan = LAST_SCAN; | |
90 | ||
2313216f MW |
91 | xas_set(&xas, 0); |
92 | xas_lock_irq(&xas); | |
b4d02baa | 93 | xas_for_each_marked(&xas, folio, ULONG_MAX, MEMFD_TAG_PINNED) { |
2313216f | 94 | bool clear = true; |
f2b277c4 | 95 | |
b4d02baa DH |
96 | if (!xa_is_value(folio) && |
97 | memfd_folio_has_extra_refs(folio)) { | |
5d752600 MK |
98 | /* |
99 | * On the last scan, we clean up all those tags | |
100 | * we inserted; but make a note that we still | |
b4d02baa | 101 | * found folios pinned. |
5d752600 | 102 | */ |
2313216f MW |
103 | if (scan == LAST_SCAN) |
104 | error = -EBUSY; | |
105 | else | |
106 | clear = false; | |
5d752600 | 107 | } |
2313216f MW |
108 | if (clear) |
109 | xas_clear_mark(&xas, MEMFD_TAG_PINNED); | |
f2b277c4 | 110 | |
b4d02baa | 111 | if (++latency < XA_CHECK_SCHED) |
2313216f | 112 | continue; |
f2b277c4 | 113 | latency = 0; |
5d752600 | 114 | |
2313216f MW |
115 | xas_pause(&xas); |
116 | xas_unlock_irq(&xas); | |
117 | cond_resched(); | |
118 | xas_lock_irq(&xas); | |
5d752600 | 119 | } |
2313216f | 120 | xas_unlock_irq(&xas); |
5d752600 MK |
121 | } |
122 | ||
123 | return error; | |
124 | } | |
125 | ||
126 | static unsigned int *memfd_file_seals_ptr(struct file *file) | |
127 | { | |
128 | if (shmem_file(file)) | |
129 | return &SHMEM_I(file_inode(file))->seals; | |
130 | ||
131 | #ifdef CONFIG_HUGETLBFS | |
132 | if (is_file_hugepages(file)) | |
133 | return &HUGETLBFS_I(file_inode(file))->seals; | |
134 | #endif | |
135 | ||
136 | return NULL; | |
137 | } | |
138 | ||
139 | #define F_ALL_SEALS (F_SEAL_SEAL | \ | |
6fd73538 | 140 | F_SEAL_EXEC | \ |
5d752600 MK |
141 | F_SEAL_SHRINK | \ |
142 | F_SEAL_GROW | \ | |
ab3948f5 JFG |
143 | F_SEAL_WRITE | \ |
144 | F_SEAL_FUTURE_WRITE) | |
5d752600 MK |
145 | |
146 | static int memfd_add_seals(struct file *file, unsigned int seals) | |
147 | { | |
148 | struct inode *inode = file_inode(file); | |
149 | unsigned int *file_seals; | |
150 | int error; | |
151 | ||
152 | /* | |
153 | * SEALING | |
154 | * Sealing allows multiple parties to share a tmpfs or hugetlbfs file | |
155 | * but restrict access to a specific subset of file operations. Seals | |
156 | * can only be added, but never removed. This way, mutually untrusted | |
157 | * parties can share common memory regions with a well-defined policy. | |
158 | * A malicious peer can thus never perform unwanted operations on a | |
159 | * shared object. | |
160 | * | |
161 | * Seals are only supported on special tmpfs or hugetlbfs files and | |
162 | * always affect the whole underlying inode. Once a seal is set, it | |
163 | * may prevent some kinds of access to the file. Currently, the | |
164 | * following seals are defined: | |
165 | * SEAL_SEAL: Prevent further seals from being set on this file | |
166 | * SEAL_SHRINK: Prevent the file from shrinking | |
167 | * SEAL_GROW: Prevent the file from growing | |
168 | * SEAL_WRITE: Prevent write access to the file | |
6fd73538 | 169 | * SEAL_EXEC: Prevent modification of the exec bits in the file mode |
5d752600 MK |
170 | * |
171 | * As we don't require any trust relationship between two parties, we | |
172 | * must prevent seals from being removed. Therefore, sealing a file | |
173 | * only adds a given set of seals to the file, it never touches | |
174 | * existing seals. Furthermore, the "setting seals"-operation can be | |
175 | * sealed itself, which basically prevents any further seal from being | |
176 | * added. | |
177 | * | |
178 | * Semantics of sealing are only defined on volatile files. Only | |
179 | * anonymous tmpfs and hugetlbfs files support sealing. More | |
180 | * importantly, seals are never written to disk. Therefore, there's | |
181 | * no plan to support it on other file types. | |
182 | */ | |
183 | ||
184 | if (!(file->f_mode & FMODE_WRITE)) | |
185 | return -EPERM; | |
186 | if (seals & ~(unsigned int)F_ALL_SEALS) | |
187 | return -EINVAL; | |
188 | ||
189 | inode_lock(inode); | |
190 | ||
191 | file_seals = memfd_file_seals_ptr(file); | |
192 | if (!file_seals) { | |
193 | error = -EINVAL; | |
194 | goto unlock; | |
195 | } | |
196 | ||
197 | if (*file_seals & F_SEAL_SEAL) { | |
198 | error = -EPERM; | |
199 | goto unlock; | |
200 | } | |
201 | ||
202 | if ((seals & F_SEAL_WRITE) && !(*file_seals & F_SEAL_WRITE)) { | |
203 | error = mapping_deny_writable(file->f_mapping); | |
204 | if (error) | |
205 | goto unlock; | |
206 | ||
207 | error = memfd_wait_for_pins(file->f_mapping); | |
208 | if (error) { | |
209 | mapping_allow_writable(file->f_mapping); | |
210 | goto unlock; | |
211 | } | |
212 | } | |
213 | ||
c4f75bc8 JX |
214 | /* |
215 | * SEAL_EXEC implys SEAL_WRITE, making W^X from the start. | |
216 | */ | |
217 | if (seals & F_SEAL_EXEC && inode->i_mode & 0111) | |
218 | seals |= F_SEAL_SHRINK|F_SEAL_GROW|F_SEAL_WRITE|F_SEAL_FUTURE_WRITE; | |
219 | ||
5d752600 MK |
220 | *file_seals |= seals; |
221 | error = 0; | |
222 | ||
223 | unlock: | |
224 | inode_unlock(inode); | |
225 | return error; | |
226 | } | |
227 | ||
228 | static int memfd_get_seals(struct file *file) | |
229 | { | |
230 | unsigned int *seals = memfd_file_seals_ptr(file); | |
231 | ||
232 | return seals ? *seals : -EINVAL; | |
233 | } | |
234 | ||
f7b8f70b | 235 | long memfd_fcntl(struct file *file, unsigned int cmd, unsigned int arg) |
5d752600 MK |
236 | { |
237 | long error; | |
238 | ||
239 | switch (cmd) { | |
240 | case F_ADD_SEALS: | |
5d752600 MK |
241 | error = memfd_add_seals(file, arg); |
242 | break; | |
243 | case F_GET_SEALS: | |
244 | error = memfd_get_seals(file); | |
245 | break; | |
246 | default: | |
247 | error = -EINVAL; | |
248 | break; | |
249 | } | |
250 | ||
251 | return error; | |
252 | } | |
253 | ||
254 | #define MFD_NAME_PREFIX "memfd:" | |
255 | #define MFD_NAME_PREFIX_LEN (sizeof(MFD_NAME_PREFIX) - 1) | |
256 | #define MFD_NAME_MAX_LEN (NAME_MAX - MFD_NAME_PREFIX_LEN) | |
257 | ||
105ff533 | 258 | #define MFD_ALL_FLAGS (MFD_CLOEXEC | MFD_ALLOW_SEALING | MFD_HUGETLB | MFD_NOEXEC_SEAL | MFD_EXEC) |
5d752600 | 259 | |
72de2591 JX |
260 | static int check_sysctl_memfd_noexec(unsigned int *flags) |
261 | { | |
262 | #ifdef CONFIG_SYSCTL | |
9876cfe8 AS |
263 | struct pid_namespace *ns = task_active_pid_ns(current); |
264 | int sysctl = pidns_memfd_noexec_scope(ns); | |
72de2591 JX |
265 | |
266 | if (!(*flags & (MFD_EXEC | MFD_NOEXEC_SEAL))) { | |
202e1422 | 267 | if (sysctl >= MEMFD_NOEXEC_SCOPE_NOEXEC_SEAL) |
72de2591 JX |
268 | *flags |= MFD_NOEXEC_SEAL; |
269 | else | |
270 | *flags |= MFD_EXEC; | |
271 | } | |
272 | ||
202e1422 AS |
273 | if (!(*flags & MFD_NOEXEC_SEAL) && sysctl >= MEMFD_NOEXEC_SCOPE_NOEXEC_ENFORCED) { |
274 | pr_err_ratelimited( | |
275 | "%s[%d]: memfd_create() requires MFD_NOEXEC_SEAL with vm.memfd_noexec=%d\n", | |
276 | current->comm, task_pid_nr(current), sysctl); | |
72de2591 JX |
277 | return -EACCES; |
278 | } | |
279 | #endif | |
72de2591 JX |
280 | return 0; |
281 | } | |
282 | ||
5d752600 MK |
283 | SYSCALL_DEFINE2(memfd_create, |
284 | const char __user *, uname, | |
285 | unsigned int, flags) | |
286 | { | |
287 | unsigned int *file_seals; | |
288 | struct file *file; | |
289 | int fd, error; | |
290 | char *name; | |
291 | long len; | |
292 | ||
293 | if (!(flags & MFD_HUGETLB)) { | |
294 | if (flags & ~(unsigned int)MFD_ALL_FLAGS) | |
295 | return -EINVAL; | |
296 | } else { | |
297 | /* Allow huge page size encoding in flags. */ | |
298 | if (flags & ~(unsigned int)(MFD_ALL_FLAGS | | |
299 | (MFD_HUGE_MASK << MFD_HUGE_SHIFT))) | |
300 | return -EINVAL; | |
301 | } | |
302 | ||
105ff533 JX |
303 | /* Invalid if both EXEC and NOEXEC_SEAL are set.*/ |
304 | if ((flags & MFD_EXEC) && (flags & MFD_NOEXEC_SEAL)) | |
305 | return -EINVAL; | |
306 | ||
202e1422 AS |
307 | error = check_sysctl_memfd_noexec(&flags); |
308 | if (error < 0) | |
309 | return error; | |
72de2591 | 310 | |
5d752600 MK |
311 | /* length includes terminating zero */ |
312 | len = strnlen_user(uname, MFD_NAME_MAX_LEN + 1); | |
313 | if (len <= 0) | |
314 | return -EFAULT; | |
315 | if (len > MFD_NAME_MAX_LEN + 1) | |
316 | return -EINVAL; | |
317 | ||
318 | name = kmalloc(len + MFD_NAME_PREFIX_LEN, GFP_KERNEL); | |
319 | if (!name) | |
320 | return -ENOMEM; | |
321 | ||
322 | strcpy(name, MFD_NAME_PREFIX); | |
323 | if (copy_from_user(&name[MFD_NAME_PREFIX_LEN], uname, len)) { | |
324 | error = -EFAULT; | |
325 | goto err_name; | |
326 | } | |
327 | ||
328 | /* terminating-zero may have changed after strnlen_user() returned */ | |
329 | if (name[len + MFD_NAME_PREFIX_LEN - 1]) { | |
330 | error = -EFAULT; | |
331 | goto err_name; | |
332 | } | |
333 | ||
334 | fd = get_unused_fd_flags((flags & MFD_CLOEXEC) ? O_CLOEXEC : 0); | |
335 | if (fd < 0) { | |
336 | error = fd; | |
337 | goto err_name; | |
338 | } | |
339 | ||
340 | if (flags & MFD_HUGETLB) { | |
83c1fd76 | 341 | file = hugetlb_file_setup(name, 0, VM_NORESERVE, |
5d752600 MK |
342 | HUGETLB_ANONHUGE_INODE, |
343 | (flags >> MFD_HUGE_SHIFT) & | |
344 | MFD_HUGE_MASK); | |
345 | } else | |
346 | file = shmem_file_setup(name, 0, VM_NORESERVE); | |
347 | if (IS_ERR(file)) { | |
348 | error = PTR_ERR(file); | |
349 | goto err_fd; | |
350 | } | |
351 | file->f_mode |= FMODE_LSEEK | FMODE_PREAD | FMODE_PWRITE; | |
c9c554f2 | 352 | file->f_flags |= O_LARGEFILE; |
5d752600 | 353 | |
105ff533 JX |
354 | if (flags & MFD_NOEXEC_SEAL) { |
355 | struct inode *inode = file_inode(file); | |
356 | ||
357 | inode->i_mode &= ~0111; | |
358 | file_seals = memfd_file_seals_ptr(file); | |
935d44ac RS |
359 | if (file_seals) { |
360 | *file_seals &= ~F_SEAL_SEAL; | |
361 | *file_seals |= F_SEAL_EXEC; | |
362 | } | |
105ff533 JX |
363 | } else if (flags & MFD_ALLOW_SEALING) { |
364 | /* MFD_EXEC and MFD_ALLOW_SEALING are set */ | |
5d752600 | 365 | file_seals = memfd_file_seals_ptr(file); |
935d44ac RS |
366 | if (file_seals) |
367 | *file_seals &= ~F_SEAL_SEAL; | |
5d752600 MK |
368 | } |
369 | ||
370 | fd_install(fd, file); | |
371 | kfree(name); | |
372 | return fd; | |
373 | ||
374 | err_fd: | |
375 | put_unused_fd(fd); | |
376 | err_name: | |
377 | kfree(name); | |
378 | return error; | |
379 | } |