]> Git Repo - linux.git/blob - drivers/infiniband/ulp/iser/iser_memory.c
selinux: Remove security_ops extern
[linux.git] / drivers / infiniband / ulp / iser / iser_memory.c
1 /*
2  * Copyright (c) 2004, 2005, 2006 Voltaire, Inc. All rights reserved.
3  * Copyright (c) 2013-2014 Mellanox Technologies. All rights reserved.
4  *
5  * This software is available to you under a choice of one of two
6  * licenses.  You may choose to be licensed under the terms of the GNU
7  * General Public License (GPL) Version 2, available from the file
8  * COPYING in the main directory of this source tree, or the
9  * OpenIB.org BSD license below:
10  *
11  *     Redistribution and use in source and binary forms, with or
12  *     without modification, are permitted provided that the following
13  *     conditions are met:
14  *
15  *      - Redistributions of source code must retain the above
16  *        copyright notice, this list of conditions and the following
17  *        disclaimer.
18  *
19  *      - Redistributions in binary form must reproduce the above
20  *        copyright notice, this list of conditions and the following
21  *        disclaimer in the documentation and/or other materials
22  *        provided with the distribution.
23  *
24  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31  * SOFTWARE.
32  */
33 #include <linux/module.h>
34 #include <linux/kernel.h>
35 #include <linux/slab.h>
36 #include <linux/mm.h>
37 #include <linux/highmem.h>
38 #include <linux/scatterlist.h>
39
40 #include "iscsi_iser.h"
41
42 #define ISER_KMALLOC_THRESHOLD 0x20000 /* 128K - kmalloc limit */
43
44 /**
45  * iser_start_rdma_unaligned_sg
46  */
47 static int iser_start_rdma_unaligned_sg(struct iscsi_iser_task *iser_task,
48                                         struct iser_data_buf *data,
49                                         struct iser_data_buf *data_copy,
50                                         enum iser_data_dir cmd_dir)
51 {
52         struct ib_device *dev = iser_task->ib_conn->device->ib_device;
53         struct scatterlist *sgl = (struct scatterlist *)data->buf;
54         struct scatterlist *sg;
55         char *mem = NULL;
56         unsigned long  cmd_data_len = 0;
57         int dma_nents, i;
58
59         for_each_sg(sgl, sg, data->size, i)
60                 cmd_data_len += ib_sg_dma_len(dev, sg);
61
62         if (cmd_data_len > ISER_KMALLOC_THRESHOLD)
63                 mem = (void *)__get_free_pages(GFP_ATOMIC,
64                       ilog2(roundup_pow_of_two(cmd_data_len)) - PAGE_SHIFT);
65         else
66                 mem = kmalloc(cmd_data_len, GFP_ATOMIC);
67
68         if (mem == NULL) {
69                 iser_err("Failed to allocate mem size %d %d for copying sglist\n",
70                          data->size, (int)cmd_data_len);
71                 return -ENOMEM;
72         }
73
74         if (cmd_dir == ISER_DIR_OUT) {
75                 /* copy the unaligned sg the buffer which is used for RDMA */
76                 int i;
77                 char *p, *from;
78
79                 sgl = (struct scatterlist *)data->buf;
80                 p = mem;
81                 for_each_sg(sgl, sg, data->size, i) {
82                         from = kmap_atomic(sg_page(sg));
83                         memcpy(p,
84                                from + sg->offset,
85                                sg->length);
86                         kunmap_atomic(from);
87                         p += sg->length;
88                 }
89         }
90
91         sg_init_one(&data_copy->sg_single, mem, cmd_data_len);
92         data_copy->buf = &data_copy->sg_single;
93         data_copy->size = 1;
94         data_copy->copy_buf = mem;
95
96         dma_nents = ib_dma_map_sg(dev, &data_copy->sg_single, 1,
97                                   (cmd_dir == ISER_DIR_OUT) ?
98                                   DMA_TO_DEVICE : DMA_FROM_DEVICE);
99         BUG_ON(dma_nents == 0);
100
101         data_copy->dma_nents = dma_nents;
102         data_copy->data_len = cmd_data_len;
103
104         return 0;
105 }
106
107 /**
108  * iser_finalize_rdma_unaligned_sg
109  */
110
111 void iser_finalize_rdma_unaligned_sg(struct iscsi_iser_task *iser_task,
112                                      struct iser_data_buf *data,
113                                      struct iser_data_buf *data_copy,
114                                      enum iser_data_dir cmd_dir)
115 {
116         struct ib_device *dev;
117         unsigned long  cmd_data_len;
118
119         dev = iser_task->ib_conn->device->ib_device;
120
121         ib_dma_unmap_sg(dev, &data_copy->sg_single, 1,
122                         (cmd_dir == ISER_DIR_OUT) ?
123                         DMA_TO_DEVICE : DMA_FROM_DEVICE);
124
125         if (cmd_dir == ISER_DIR_IN) {
126                 char *mem;
127                 struct scatterlist *sgl, *sg;
128                 unsigned char *p, *to;
129                 unsigned int sg_size;
130                 int i;
131
132                 /* copy back read RDMA to unaligned sg */
133                 mem = data_copy->copy_buf;
134
135                 sgl = (struct scatterlist *)data->buf;
136                 sg_size = data->size;
137
138                 p = mem;
139                 for_each_sg(sgl, sg, sg_size, i) {
140                         to = kmap_atomic(sg_page(sg));
141                         memcpy(to + sg->offset,
142                                p,
143                                sg->length);
144                         kunmap_atomic(to);
145                         p += sg->length;
146                 }
147         }
148
149         cmd_data_len = data->data_len;
150
151         if (cmd_data_len > ISER_KMALLOC_THRESHOLD)
152                 free_pages((unsigned long)data_copy->copy_buf,
153                            ilog2(roundup_pow_of_two(cmd_data_len)) - PAGE_SHIFT);
154         else
155                 kfree(data_copy->copy_buf);
156
157         data_copy->copy_buf = NULL;
158 }
159
160 #define IS_4K_ALIGNED(addr)     ((((unsigned long)addr) & ~MASK_4K) == 0)
161
162 /**
163  * iser_sg_to_page_vec - Translates scatterlist entries to physical addresses
164  * and returns the length of resulting physical address array (may be less than
165  * the original due to possible compaction).
166  *
167  * we build a "page vec" under the assumption that the SG meets the RDMA
168  * alignment requirements. Other then the first and last SG elements, all
169  * the "internal" elements can be compacted into a list whose elements are
170  * dma addresses of physical pages. The code supports also the weird case
171  * where --few fragments of the same page-- are present in the SG as
172  * consecutive elements. Also, it handles one entry SG.
173  */
174
175 static int iser_sg_to_page_vec(struct iser_data_buf *data,
176                                struct ib_device *ibdev, u64 *pages,
177                                int *offset, int *data_size)
178 {
179         struct scatterlist *sg, *sgl = (struct scatterlist *)data->buf;
180         u64 start_addr, end_addr, page, chunk_start = 0;
181         unsigned long total_sz = 0;
182         unsigned int dma_len;
183         int i, new_chunk, cur_page, last_ent = data->dma_nents - 1;
184
185         /* compute the offset of first element */
186         *offset = (u64) sgl[0].offset & ~MASK_4K;
187
188         new_chunk = 1;
189         cur_page  = 0;
190         for_each_sg(sgl, sg, data->dma_nents, i) {
191                 start_addr = ib_sg_dma_address(ibdev, sg);
192                 if (new_chunk)
193                         chunk_start = start_addr;
194                 dma_len = ib_sg_dma_len(ibdev, sg);
195                 end_addr = start_addr + dma_len;
196                 total_sz += dma_len;
197
198                 /* collect page fragments until aligned or end of SG list */
199                 if (!IS_4K_ALIGNED(end_addr) && i < last_ent) {
200                         new_chunk = 0;
201                         continue;
202                 }
203                 new_chunk = 1;
204
205                 /* address of the first page in the contiguous chunk;
206                    masking relevant for the very first SG entry,
207                    which might be unaligned */
208                 page = chunk_start & MASK_4K;
209                 do {
210                         pages[cur_page++] = page;
211                         page += SIZE_4K;
212                 } while (page < end_addr);
213         }
214
215         *data_size = total_sz;
216         iser_dbg("page_vec->data_size:%d cur_page %d\n",
217                  *data_size, cur_page);
218         return cur_page;
219 }
220
221
222 /**
223  * iser_data_buf_aligned_len - Tries to determine the maximal correctly aligned
224  * for RDMA sub-list of a scatter-gather list of memory buffers, and  returns
225  * the number of entries which are aligned correctly. Supports the case where
226  * consecutive SG elements are actually fragments of the same physcial page.
227  */
228 static int iser_data_buf_aligned_len(struct iser_data_buf *data,
229                                       struct ib_device *ibdev)
230 {
231         struct scatterlist *sgl, *sg, *next_sg = NULL;
232         u64 start_addr, end_addr;
233         int i, ret_len, start_check = 0;
234
235         if (data->dma_nents == 1)
236                 return 1;
237
238         sgl = (struct scatterlist *)data->buf;
239         start_addr  = ib_sg_dma_address(ibdev, sgl);
240
241         for_each_sg(sgl, sg, data->dma_nents, i) {
242                 if (start_check && !IS_4K_ALIGNED(start_addr))
243                         break;
244
245                 next_sg = sg_next(sg);
246                 if (!next_sg)
247                         break;
248
249                 end_addr    = start_addr + ib_sg_dma_len(ibdev, sg);
250                 start_addr  = ib_sg_dma_address(ibdev, next_sg);
251
252                 if (end_addr == start_addr) {
253                         start_check = 0;
254                         continue;
255                 } else
256                         start_check = 1;
257
258                 if (!IS_4K_ALIGNED(end_addr))
259                         break;
260         }
261         ret_len = (next_sg) ? i : i+1;
262         iser_dbg("Found %d aligned entries out of %d in sg:0x%p\n",
263                  ret_len, data->dma_nents, data);
264         return ret_len;
265 }
266
267 static void iser_data_buf_dump(struct iser_data_buf *data,
268                                struct ib_device *ibdev)
269 {
270         struct scatterlist *sgl = (struct scatterlist *)data->buf;
271         struct scatterlist *sg;
272         int i;
273
274         for_each_sg(sgl, sg, data->dma_nents, i)
275                 iser_dbg("sg[%d] dma_addr:0x%lX page:0x%p "
276                          "off:0x%x sz:0x%x dma_len:0x%x\n",
277                          i, (unsigned long)ib_sg_dma_address(ibdev, sg),
278                          sg_page(sg), sg->offset,
279                          sg->length, ib_sg_dma_len(ibdev, sg));
280 }
281
282 static void iser_dump_page_vec(struct iser_page_vec *page_vec)
283 {
284         int i;
285
286         iser_err("page vec length %d data size %d\n",
287                  page_vec->length, page_vec->data_size);
288         for (i = 0; i < page_vec->length; i++)
289                 iser_err("%d %lx\n",i,(unsigned long)page_vec->pages[i]);
290 }
291
292 static void iser_page_vec_build(struct iser_data_buf *data,
293                                 struct iser_page_vec *page_vec,
294                                 struct ib_device *ibdev)
295 {
296         int page_vec_len = 0;
297
298         page_vec->length = 0;
299         page_vec->offset = 0;
300
301         iser_dbg("Translating sg sz: %d\n", data->dma_nents);
302         page_vec_len = iser_sg_to_page_vec(data, ibdev, page_vec->pages,
303                                            &page_vec->offset,
304                                            &page_vec->data_size);
305         iser_dbg("sg len %d page_vec_len %d\n", data->dma_nents, page_vec_len);
306
307         page_vec->length = page_vec_len;
308
309         if (page_vec_len * SIZE_4K < page_vec->data_size) {
310                 iser_err("page_vec too short to hold this SG\n");
311                 iser_data_buf_dump(data, ibdev);
312                 iser_dump_page_vec(page_vec);
313                 BUG();
314         }
315 }
316
317 int iser_dma_map_task_data(struct iscsi_iser_task *iser_task,
318                             struct iser_data_buf *data,
319                             enum iser_data_dir iser_dir,
320                             enum dma_data_direction dma_dir)
321 {
322         struct ib_device *dev;
323
324         iser_task->dir[iser_dir] = 1;
325         dev = iser_task->ib_conn->device->ib_device;
326
327         data->dma_nents = ib_dma_map_sg(dev, data->buf, data->size, dma_dir);
328         if (data->dma_nents == 0) {
329                 iser_err("dma_map_sg failed!!!\n");
330                 return -EINVAL;
331         }
332         return 0;
333 }
334
335 void iser_dma_unmap_task_data(struct iscsi_iser_task *iser_task,
336                               struct iser_data_buf *data)
337 {
338         struct ib_device *dev;
339
340         dev = iser_task->ib_conn->device->ib_device;
341         ib_dma_unmap_sg(dev, data->buf, data->size, DMA_FROM_DEVICE);
342 }
343
344 static int fall_to_bounce_buf(struct iscsi_iser_task *iser_task,
345                               struct ib_device *ibdev,
346                               struct iser_data_buf *mem,
347                               struct iser_data_buf *mem_copy,
348                               enum iser_data_dir cmd_dir,
349                               int aligned_len)
350 {
351         struct iscsi_conn    *iscsi_conn = iser_task->ib_conn->iscsi_conn;
352
353         iscsi_conn->fmr_unalign_cnt++;
354         iser_warn("rdma alignment violation (%d/%d aligned) or FMR not supported\n",
355                   aligned_len, mem->size);
356
357         if (iser_debug_level > 0)
358                 iser_data_buf_dump(mem, ibdev);
359
360         /* unmap the command data before accessing it */
361         iser_dma_unmap_task_data(iser_task, mem);
362
363         /* allocate copy buf, if we are writing, copy the */
364         /* unaligned scatterlist, dma map the copy        */
365         if (iser_start_rdma_unaligned_sg(iser_task, mem, mem_copy, cmd_dir) != 0)
366                 return -ENOMEM;
367
368         return 0;
369 }
370
371 /**
372  * iser_reg_rdma_mem_fmr - Registers memory intended for RDMA,
373  * using FMR (if possible) obtaining rkey and va
374  *
375  * returns 0 on success, errno code on failure
376  */
377 int iser_reg_rdma_mem_fmr(struct iscsi_iser_task *iser_task,
378                           enum iser_data_dir cmd_dir)
379 {
380         struct iser_conn     *ib_conn = iser_task->ib_conn;
381         struct iser_device   *device = ib_conn->device;
382         struct ib_device     *ibdev = device->ib_device;
383         struct iser_data_buf *mem = &iser_task->data[cmd_dir];
384         struct iser_regd_buf *regd_buf;
385         int aligned_len;
386         int err;
387         int i;
388         struct scatterlist *sg;
389
390         regd_buf = &iser_task->rdma_regd[cmd_dir];
391
392         aligned_len = iser_data_buf_aligned_len(mem, ibdev);
393         if (aligned_len != mem->dma_nents) {
394                 err = fall_to_bounce_buf(iser_task, ibdev, mem,
395                                          &iser_task->data_copy[cmd_dir],
396                                          cmd_dir, aligned_len);
397                 if (err) {
398                         iser_err("failed to allocate bounce buffer\n");
399                         return err;
400                 }
401                 mem = &iser_task->data_copy[cmd_dir];
402         }
403
404         /* if there a single dma entry, FMR is not needed */
405         if (mem->dma_nents == 1) {
406                 sg = (struct scatterlist *)mem->buf;
407
408                 regd_buf->reg.lkey = device->mr->lkey;
409                 regd_buf->reg.rkey = device->mr->rkey;
410                 regd_buf->reg.len  = ib_sg_dma_len(ibdev, &sg[0]);
411                 regd_buf->reg.va   = ib_sg_dma_address(ibdev, &sg[0]);
412                 regd_buf->reg.is_mr = 0;
413
414                 iser_dbg("PHYSICAL Mem.register: lkey: 0x%08X rkey: 0x%08X  "
415                          "va: 0x%08lX sz: %ld]\n",
416                          (unsigned int)regd_buf->reg.lkey,
417                          (unsigned int)regd_buf->reg.rkey,
418                          (unsigned long)regd_buf->reg.va,
419                          (unsigned long)regd_buf->reg.len);
420         } else { /* use FMR for multiple dma entries */
421                 iser_page_vec_build(mem, ib_conn->fmr.page_vec, ibdev);
422                 err = iser_reg_page_vec(ib_conn, ib_conn->fmr.page_vec,
423                                         &regd_buf->reg);
424                 if (err && err != -EAGAIN) {
425                         iser_data_buf_dump(mem, ibdev);
426                         iser_err("mem->dma_nents = %d (dlength = 0x%x)\n",
427                                  mem->dma_nents,
428                                  ntoh24(iser_task->desc.iscsi_header.dlength));
429                         iser_err("page_vec: data_size = 0x%x, length = %d, offset = 0x%x\n",
430                                  ib_conn->fmr.page_vec->data_size,
431                                  ib_conn->fmr.page_vec->length,
432                                  ib_conn->fmr.page_vec->offset);
433                         for (i = 0; i < ib_conn->fmr.page_vec->length; i++)
434                                 iser_err("page_vec[%d] = 0x%llx\n", i,
435                                          (unsigned long long) ib_conn->fmr.page_vec->pages[i]);
436                 }
437                 if (err)
438                         return err;
439         }
440         return 0;
441 }
442
443 static inline enum ib_t10_dif_type
444 scsi2ib_prot_type(unsigned char prot_type)
445 {
446         switch (prot_type) {
447         case SCSI_PROT_DIF_TYPE0:
448                 return IB_T10DIF_NONE;
449         case SCSI_PROT_DIF_TYPE1:
450                 return IB_T10DIF_TYPE1;
451         case SCSI_PROT_DIF_TYPE2:
452                 return IB_T10DIF_TYPE2;
453         case SCSI_PROT_DIF_TYPE3:
454                 return IB_T10DIF_TYPE3;
455         default:
456                 return IB_T10DIF_NONE;
457         }
458 }
459
460
461 static int
462 iser_set_sig_attrs(struct scsi_cmnd *sc, struct ib_sig_attrs *sig_attrs)
463 {
464         unsigned char scsi_ptype = scsi_get_prot_type(sc);
465
466         sig_attrs->mem.sig_type = IB_SIG_TYPE_T10_DIF;
467         sig_attrs->wire.sig_type = IB_SIG_TYPE_T10_DIF;
468         sig_attrs->mem.sig.dif.pi_interval = sc->device->sector_size;
469         sig_attrs->wire.sig.dif.pi_interval = sc->device->sector_size;
470
471         switch (scsi_get_prot_op(sc)) {
472         case SCSI_PROT_WRITE_INSERT:
473         case SCSI_PROT_READ_STRIP:
474                 sig_attrs->mem.sig.dif.type = IB_T10DIF_NONE;
475                 sig_attrs->wire.sig.dif.type = scsi2ib_prot_type(scsi_ptype);
476                 sig_attrs->wire.sig.dif.bg_type = IB_T10DIF_CRC;
477                 sig_attrs->wire.sig.dif.ref_tag = scsi_get_lba(sc) &
478                                                   0xffffffff;
479                 break;
480         case SCSI_PROT_READ_INSERT:
481         case SCSI_PROT_WRITE_STRIP:
482                 sig_attrs->mem.sig.dif.type = scsi2ib_prot_type(scsi_ptype);
483                 sig_attrs->mem.sig.dif.bg_type = IB_T10DIF_CRC;
484                 sig_attrs->mem.sig.dif.ref_tag = scsi_get_lba(sc) &
485                                                  0xffffffff;
486                 sig_attrs->wire.sig.dif.type = IB_T10DIF_NONE;
487                 break;
488         case SCSI_PROT_READ_PASS:
489         case SCSI_PROT_WRITE_PASS:
490                 sig_attrs->mem.sig.dif.type = scsi2ib_prot_type(scsi_ptype);
491                 sig_attrs->mem.sig.dif.bg_type = IB_T10DIF_CRC;
492                 sig_attrs->mem.sig.dif.ref_tag = scsi_get_lba(sc) &
493                                                  0xffffffff;
494                 sig_attrs->wire.sig.dif.type = scsi2ib_prot_type(scsi_ptype);
495                 sig_attrs->wire.sig.dif.bg_type = IB_T10DIF_CRC;
496                 sig_attrs->wire.sig.dif.ref_tag = scsi_get_lba(sc) &
497                                                   0xffffffff;
498                 break;
499         default:
500                 iser_err("Unsupported PI operation %d\n",
501                          scsi_get_prot_op(sc));
502                 return -EINVAL;
503         }
504         return 0;
505 }
506
507
508 static int
509 iser_set_prot_checks(struct scsi_cmnd *sc, u8 *mask)
510 {
511         switch (scsi_get_prot_type(sc)) {
512         case SCSI_PROT_DIF_TYPE0:
513                 *mask = 0x0;
514                 break;
515         case SCSI_PROT_DIF_TYPE1:
516         case SCSI_PROT_DIF_TYPE2:
517                 *mask = ISER_CHECK_GUARD | ISER_CHECK_REFTAG;
518                 break;
519         case SCSI_PROT_DIF_TYPE3:
520                 *mask = ISER_CHECK_GUARD;
521                 break;
522         default:
523                 iser_err("Unsupported protection type %d\n",
524                          scsi_get_prot_type(sc));
525                 return -EINVAL;
526         }
527
528         return 0;
529 }
530
531 static int
532 iser_reg_sig_mr(struct iscsi_iser_task *iser_task,
533                 struct fast_reg_descriptor *desc, struct ib_sge *data_sge,
534                 struct ib_sge *prot_sge, struct ib_sge *sig_sge)
535 {
536         struct iser_conn *ib_conn = iser_task->ib_conn;
537         struct iser_pi_context *pi_ctx = desc->pi_ctx;
538         struct ib_send_wr sig_wr, inv_wr;
539         struct ib_send_wr *bad_wr, *wr = NULL;
540         struct ib_sig_attrs sig_attrs;
541         int ret;
542         u32 key;
543
544         memset(&sig_attrs, 0, sizeof(sig_attrs));
545         ret = iser_set_sig_attrs(iser_task->sc, &sig_attrs);
546         if (ret)
547                 goto err;
548
549         ret = iser_set_prot_checks(iser_task->sc, &sig_attrs.check_mask);
550         if (ret)
551                 goto err;
552
553         if (!(desc->reg_indicators & ISER_SIG_KEY_VALID)) {
554                 memset(&inv_wr, 0, sizeof(inv_wr));
555                 inv_wr.opcode = IB_WR_LOCAL_INV;
556                 inv_wr.wr_id = ISER_FASTREG_LI_WRID;
557                 inv_wr.ex.invalidate_rkey = pi_ctx->sig_mr->rkey;
558                 wr = &inv_wr;
559                 /* Bump the key */
560                 key = (u8)(pi_ctx->sig_mr->rkey & 0x000000FF);
561                 ib_update_fast_reg_key(pi_ctx->sig_mr, ++key);
562         }
563
564         memset(&sig_wr, 0, sizeof(sig_wr));
565         sig_wr.opcode = IB_WR_REG_SIG_MR;
566         sig_wr.wr_id = ISER_FASTREG_LI_WRID;
567         sig_wr.sg_list = data_sge;
568         sig_wr.num_sge = 1;
569         sig_wr.wr.sig_handover.sig_attrs = &sig_attrs;
570         sig_wr.wr.sig_handover.sig_mr = pi_ctx->sig_mr;
571         if (scsi_prot_sg_count(iser_task->sc))
572                 sig_wr.wr.sig_handover.prot = prot_sge;
573         sig_wr.wr.sig_handover.access_flags = IB_ACCESS_LOCAL_WRITE |
574                                               IB_ACCESS_REMOTE_READ |
575                                               IB_ACCESS_REMOTE_WRITE;
576
577         if (!wr)
578                 wr = &sig_wr;
579         else
580                 wr->next = &sig_wr;
581
582         ret = ib_post_send(ib_conn->qp, wr, &bad_wr);
583         if (ret) {
584                 iser_err("reg_sig_mr failed, ret:%d\n", ret);
585                 goto err;
586         }
587         desc->reg_indicators &= ~ISER_SIG_KEY_VALID;
588
589         sig_sge->lkey = pi_ctx->sig_mr->lkey;
590         sig_sge->addr = 0;
591         sig_sge->length = data_sge->length + prot_sge->length;
592         if (scsi_get_prot_op(iser_task->sc) == SCSI_PROT_WRITE_INSERT ||
593             scsi_get_prot_op(iser_task->sc) == SCSI_PROT_READ_STRIP) {
594                 sig_sge->length += (data_sge->length /
595                                    iser_task->sc->device->sector_size) * 8;
596         }
597
598         iser_dbg("sig_sge: addr: 0x%llx  length: %u lkey: 0x%x\n",
599                  sig_sge->addr, sig_sge->length,
600                  sig_sge->lkey);
601 err:
602         return ret;
603 }
604
605 static int iser_fast_reg_mr(struct iscsi_iser_task *iser_task,
606                             struct iser_regd_buf *regd_buf,
607                             struct iser_data_buf *mem,
608                             enum iser_reg_indicator ind,
609                             struct ib_sge *sge)
610 {
611         struct fast_reg_descriptor *desc = regd_buf->reg.mem_h;
612         struct iser_conn *ib_conn = iser_task->ib_conn;
613         struct iser_device *device = ib_conn->device;
614         struct ib_device *ibdev = device->ib_device;
615         struct ib_mr *mr;
616         struct ib_fast_reg_page_list *frpl;
617         struct ib_send_wr fastreg_wr, inv_wr;
618         struct ib_send_wr *bad_wr, *wr = NULL;
619         u8 key;
620         int ret, offset, size, plen;
621
622         /* if there a single dma entry, dma mr suffices */
623         if (mem->dma_nents == 1) {
624                 struct scatterlist *sg = (struct scatterlist *)mem->buf;
625
626                 sge->lkey = device->mr->lkey;
627                 sge->addr   = ib_sg_dma_address(ibdev, &sg[0]);
628                 sge->length  = ib_sg_dma_len(ibdev, &sg[0]);
629
630                 iser_dbg("Single DMA entry: lkey=0x%x, addr=0x%llx, length=0x%x\n",
631                          sge->lkey, sge->addr, sge->length);
632                 return 0;
633         }
634
635         if (ind == ISER_DATA_KEY_VALID) {
636                 mr = desc->data_mr;
637                 frpl = desc->data_frpl;
638         } else {
639                 mr = desc->pi_ctx->prot_mr;
640                 frpl = desc->pi_ctx->prot_frpl;
641         }
642
643         plen = iser_sg_to_page_vec(mem, device->ib_device, frpl->page_list,
644                                    &offset, &size);
645         if (plen * SIZE_4K < size) {
646                 iser_err("fast reg page_list too short to hold this SG\n");
647                 return -EINVAL;
648         }
649
650         if (!(desc->reg_indicators & ind)) {
651                 memset(&inv_wr, 0, sizeof(inv_wr));
652                 inv_wr.wr_id = ISER_FASTREG_LI_WRID;
653                 inv_wr.opcode = IB_WR_LOCAL_INV;
654                 inv_wr.ex.invalidate_rkey = mr->rkey;
655                 wr = &inv_wr;
656                 /* Bump the key */
657                 key = (u8)(mr->rkey & 0x000000FF);
658                 ib_update_fast_reg_key(mr, ++key);
659         }
660
661         /* Prepare FASTREG WR */
662         memset(&fastreg_wr, 0, sizeof(fastreg_wr));
663         fastreg_wr.wr_id = ISER_FASTREG_LI_WRID;
664         fastreg_wr.opcode = IB_WR_FAST_REG_MR;
665         fastreg_wr.wr.fast_reg.iova_start = frpl->page_list[0] + offset;
666         fastreg_wr.wr.fast_reg.page_list = frpl;
667         fastreg_wr.wr.fast_reg.page_list_len = plen;
668         fastreg_wr.wr.fast_reg.page_shift = SHIFT_4K;
669         fastreg_wr.wr.fast_reg.length = size;
670         fastreg_wr.wr.fast_reg.rkey = mr->rkey;
671         fastreg_wr.wr.fast_reg.access_flags = (IB_ACCESS_LOCAL_WRITE  |
672                                                IB_ACCESS_REMOTE_WRITE |
673                                                IB_ACCESS_REMOTE_READ);
674
675         if (!wr)
676                 wr = &fastreg_wr;
677         else
678                 wr->next = &fastreg_wr;
679
680         ret = ib_post_send(ib_conn->qp, wr, &bad_wr);
681         if (ret) {
682                 iser_err("fast registration failed, ret:%d\n", ret);
683                 return ret;
684         }
685         desc->reg_indicators &= ~ind;
686
687         sge->lkey = mr->lkey;
688         sge->addr = frpl->page_list[0] + offset;
689         sge->length = size;
690
691         return ret;
692 }
693
694 /**
695  * iser_reg_rdma_mem_fastreg - Registers memory intended for RDMA,
696  * using Fast Registration WR (if possible) obtaining rkey and va
697  *
698  * returns 0 on success, errno code on failure
699  */
700 int iser_reg_rdma_mem_fastreg(struct iscsi_iser_task *iser_task,
701                               enum iser_data_dir cmd_dir)
702 {
703         struct iser_conn *ib_conn = iser_task->ib_conn;
704         struct iser_device *device = ib_conn->device;
705         struct ib_device *ibdev = device->ib_device;
706         struct iser_data_buf *mem = &iser_task->data[cmd_dir];
707         struct iser_regd_buf *regd_buf = &iser_task->rdma_regd[cmd_dir];
708         struct fast_reg_descriptor *desc = NULL;
709         struct ib_sge data_sge;
710         int err, aligned_len;
711         unsigned long flags;
712
713         aligned_len = iser_data_buf_aligned_len(mem, ibdev);
714         if (aligned_len != mem->dma_nents) {
715                 err = fall_to_bounce_buf(iser_task, ibdev, mem,
716                                          &iser_task->data_copy[cmd_dir],
717                                          cmd_dir, aligned_len);
718                 if (err) {
719                         iser_err("failed to allocate bounce buffer\n");
720                         return err;
721                 }
722                 mem = &iser_task->data_copy[cmd_dir];
723         }
724
725         if (mem->dma_nents != 1 ||
726             scsi_get_prot_op(iser_task->sc) != SCSI_PROT_NORMAL) {
727                 spin_lock_irqsave(&ib_conn->lock, flags);
728                 desc = list_first_entry(&ib_conn->fastreg.pool,
729                                         struct fast_reg_descriptor, list);
730                 list_del(&desc->list);
731                 spin_unlock_irqrestore(&ib_conn->lock, flags);
732                 regd_buf->reg.mem_h = desc;
733         }
734
735         err = iser_fast_reg_mr(iser_task, regd_buf, mem,
736                                ISER_DATA_KEY_VALID, &data_sge);
737         if (err)
738                 goto err_reg;
739
740         if (scsi_get_prot_op(iser_task->sc) != SCSI_PROT_NORMAL) {
741                 struct ib_sge prot_sge, sig_sge;
742
743                 memset(&prot_sge, 0, sizeof(prot_sge));
744                 if (scsi_prot_sg_count(iser_task->sc)) {
745                         mem = &iser_task->prot[cmd_dir];
746                         aligned_len = iser_data_buf_aligned_len(mem, ibdev);
747                         if (aligned_len != mem->dma_nents) {
748                                 err = fall_to_bounce_buf(iser_task, ibdev, mem,
749                                                          &iser_task->prot_copy[cmd_dir],
750                                                          cmd_dir, aligned_len);
751                                 if (err) {
752                                         iser_err("failed to allocate bounce buffer\n");
753                                         return err;
754                                 }
755                                 mem = &iser_task->prot_copy[cmd_dir];
756                         }
757
758                         err = iser_fast_reg_mr(iser_task, regd_buf, mem,
759                                                ISER_PROT_KEY_VALID, &prot_sge);
760                         if (err)
761                                 goto err_reg;
762                 }
763
764                 err = iser_reg_sig_mr(iser_task, desc, &data_sge,
765                                       &prot_sge, &sig_sge);
766                 if (err) {
767                         iser_err("Failed to register signature mr\n");
768                         return err;
769                 }
770                 desc->reg_indicators |= ISER_FASTREG_PROTECTED;
771
772                 regd_buf->reg.lkey = sig_sge.lkey;
773                 regd_buf->reg.rkey = desc->pi_ctx->sig_mr->rkey;
774                 regd_buf->reg.va = sig_sge.addr;
775                 regd_buf->reg.len = sig_sge.length;
776                 regd_buf->reg.is_mr = 1;
777         } else {
778                 if (desc) {
779                         regd_buf->reg.rkey = desc->data_mr->rkey;
780                         regd_buf->reg.is_mr = 1;
781                 } else {
782                         regd_buf->reg.rkey = device->mr->rkey;
783                         regd_buf->reg.is_mr = 0;
784                 }
785
786                 regd_buf->reg.lkey = data_sge.lkey;
787                 regd_buf->reg.va = data_sge.addr;
788                 regd_buf->reg.len = data_sge.length;
789         }
790
791         return 0;
792 err_reg:
793         if (desc) {
794                 spin_lock_irqsave(&ib_conn->lock, flags);
795                 list_add_tail(&desc->list, &ib_conn->fastreg.pool);
796                 spin_unlock_irqrestore(&ib_conn->lock, flags);
797         }
798
799         return err;
800 }
This page took 0.082525 seconds and 4 git commands to generate.