diff --git a/drivers/crypto/ccp/hygon/psp-dev.c b/drivers/crypto/ccp/hygon/psp-dev.c index 1c554e4d08b8ee651f93786e28cd21f66d1aae87..d68c6abdaa2d910c5d9f7655f1e027f1964a6e02 100644 --- a/drivers/crypto/ccp/hygon/psp-dev.c +++ b/drivers/crypto/ccp/hygon/psp-dev.c @@ -794,7 +794,7 @@ void psp_ringbuffer_queue_free(struct csv_ringbuffer_queue *ring_buffer) } static int __psp_do_generic_ringbuf_cmds_locked(struct csv_ringbuffer_queue *ring_buffer, - int *pspret) + int *pspret, bool overcommit) { int cmd = PSP_CMD_RING_BUFFER; struct csv_ringbuffer_queue *que = NULL; @@ -815,7 +815,7 @@ static int __psp_do_generic_ringbuf_cmds_locked(struct csv_ringbuffer_queue *rin psp_grb_cmdbuf->low.cmdptr_address = __psp_pa(que->cmd_ptr.data_align); psp_grb_cmdbuf->low.statval_address = __psp_pa(que->stat_val.data_align); psp_grb_cmdbuf->low.head = cmd_queue_head(&que->cmd_ptr); - if (psp_rb_oc_supported) + if (psp_rb_oc_supported && overcommit) psp_grb_cmdbuf->low.tail = cmd_queue_overcommit_tail(&que->cmd_ptr); else psp_grb_cmdbuf->low.tail = cmd_queue_tail(&que->cmd_ptr); @@ -862,7 +862,8 @@ static int __psp_ringbuffer_enter_locked(struct csv_ringbuffer_queue *ring_buffe return ret; } -static int __psp_do_ringbuffer_cmds_locked(struct csv_ringbuffer_queue *ring_buffer, int *psp_ret) +static int __psp_do_ringbuffer_cmds_locked(struct csv_ringbuffer_queue *ring_buffer, int *psp_ret, + bool overcommit) { struct psp_device *psp = psp_master; unsigned int rb_tail, rb_head; @@ -883,7 +884,7 @@ static int __psp_do_ringbuffer_cmds_locked(struct csv_ringbuffer_queue *ring_buf rb_tail &= (~PSP_RBTAIL_QHI_TAIL_MASK); rb_tail |= (cmd_queue_tail(&hi_rb->cmd_ptr) << PSP_RBTAIL_QHI_TAIL_SHIFT); rb_tail &= (~PSP_RBTAIL_QLO_TAIL_MASK); - if (psp_rb_oc_supported) + if (psp_rb_oc_supported && overcommit) rb_tail |= cmd_queue_overcommit_tail(&low_rb->cmd_ptr); else rb_tail |= cmd_queue_tail(&low_rb->cmd_ptr); @@ -927,18 +928,19 @@ static int __psp_do_ringbuffer_cmds_locked(struct csv_ringbuffer_queue *ring_buf return ret; } -int psp_do_ringbuffer_cmds_locked(struct csv_ringbuffer_queue *ring_buffer, int *psp_ret) +int psp_do_ringbuffer_cmds_locked(struct csv_ringbuffer_queue *ring_buffer, int *psp_ret, + bool overcommit) { int rc = 0; if (psp_generic_rb_supported) { - rc = __psp_do_generic_ringbuf_cmds_locked(ring_buffer, psp_ret); + rc = __psp_do_generic_ringbuf_cmds_locked(ring_buffer, psp_ret, overcommit); } else { rc = __psp_ringbuffer_enter_locked(ring_buffer, psp_ret); if (rc) goto end; - rc = __psp_do_ringbuffer_cmds_locked(ring_buffer, psp_ret); + rc = __psp_do_ringbuffer_cmds_locked(ring_buffer, psp_ret, overcommit); } end: return rc; diff --git a/drivers/crypto/ccp/hygon/psp-dev.h b/drivers/crypto/ccp/hygon/psp-dev.h index d730d1803bb981e3a97c824d14d573e77a90205f..bb9673359e3a86f16dd9411a1a212a81ce063b7f 100644 --- a/drivers/crypto/ccp/hygon/psp-dev.h +++ b/drivers/crypto/ccp/hygon/psp-dev.h @@ -77,6 +77,34 @@ struct tkm_cmdresp_device_info_get { struct tkm_device_info dev_info; } __packed; +struct vtkm_cmdresp_key_backup { + struct tkm_cmdresp_head head; + uint16_t vid; + struct { + uint32_t scheme : 31; + uint32_t has_shared_data : 1; + }; + uint64_t image_location; /*In*/ + uint32_t image_length; /*In Out*/ + uint64_t cert_chain_location; + uint32_t cert_chain_len; + uint64_t shared_data_location; + uint32_t shared_data_len; +} __packed; + +struct vtkm_cmdresp_key_restore { + struct tkm_cmdresp_head head; + uint16_t vid; + struct { + uint32_t scheme : 31; + uint32_t has_shared_data : 1; + }; + uint64_t image_location; /*In*/ + uint32_t image_length; /*In*/ + uint64_t shared_data_location; + uint32_t shared_data_len; +} __packed; + struct queue_info { uint32_t head; /* In|Out */ uint32_t tail; /* In */ @@ -97,6 +125,19 @@ struct psp_ringbuffer_cmd_buf { #define PSP_CMD_STATUS_RUNNING 0xffff #define PSP_RB_OVERCOMMIT_SIZE 1024 #define TKM_DEVICE_INFO_GET 0x1001 +#define VTKM_KEY_BACKUP 0x905 +#define VTKM_KEY_RESTORE 0x906 +#define KEY_PROT_SAME_GENERATION 1 +#define TKM_CMDRESP_MIN_SIZE 4096 + +// multiple level pointer or offset mode for tkm command +#define OFFSET_MODE_BIT_SHIFT 63 +#define LOC_IS_OFFSET_MODE(location) (((uint64_t)(location) >> OFFSET_MODE_BIT_SHIFT) == 1) +#define LOC_IS_POINTER_MODE(location) (!LOC_IS_OFFSET_MODE(location)) +#define LOC_CLEAR_MODE_BIT(location) ((uint64_t)(location) & ~((uint64_t)1 \ + << OFFSET_MODE_BIT_SHIFT)) +#define LOC_FILL_MODE_BIT(location) ((uint64_t)(location) | ((uint64_t)1 \ + << OFFSET_MODE_BIT_SHIFT)) #define PSP_CMD_RING_BUFFER 0x304 #define PSP_DO_CMD_OP_PHYADDR BIT(0) // Input data as physical address @@ -148,6 +189,7 @@ void psp_ringbuffer_check_support(void); * psp_do_ringbuffer_cmds_locked is a no-wait PSP operation. * Must register notify via psp_worker_register_notify before use. */ -int psp_do_ringbuffer_cmds_locked(struct csv_ringbuffer_queue *ring_buffer, int *psp_ret); +int psp_do_ringbuffer_cmds_locked(struct csv_ringbuffer_queue *ring_buffer, int *psp_ret, + bool overcommit); #endif /* __CCP_HYGON_PSP_DEV_H__ */ diff --git a/drivers/crypto/ccp/hygon/vpsp.c b/drivers/crypto/ccp/hygon/vpsp.c index 8c73010f9518386801b273e5525408aa1882122c..b211a310c9d60d28f5945188395c3b28a3230934 100644 --- a/drivers/crypto/ccp/hygon/vpsp.c +++ b/drivers/crypto/ccp/hygon/vpsp.c @@ -30,6 +30,11 @@ #define pr_fmt(fmt) "vpsp: " fmt #define VTKM_VM_BIND 0x904 +static int vpsp_backup_key(struct key_img_ctl *ctl); +static int vpsp_restore_key(struct key_img_ctl *ctl); +static int vpsp_backup_cmd_ctx(struct cmd_ctx_ctl *ctl); +static int vpsp_restore_cmd_ctx(struct cmd_ctx_ctl *ctl); + /* * The file mainly implements the base execution logic of virtual PSP in kernel mode, * which mainly includes: @@ -84,6 +89,30 @@ static struct vpsp_cmd_ctx *vpsp_hashtable_find_cmd_ctx(gpa_t key1, pid_t key2) return entry; } +static int vpsp_hashtable_save_cmd_ctx_by_pid(pid_t pid, struct vpsp_cmd_ctx *ctx[], uint32_t *nums) +{ + struct vpsp_cmd_ctx *entry = NULL; + struct hlist_node *tmp; + int i; + + if (!nums) + return -EINVAL; + *nums = 0; + + read_lock(&table_rwlock); + // Allowed entries to be deleted during traversal + hash_for_each_safe(vpsp_cmd_ctx_table, i, tmp, entry, node) { + if (entry->key2 == pid) { + if (ctx) + ctx[*nums] = entry; + ++(*nums); + } + } + read_unlock(&table_rwlock); + + return 0; +} + static void vpsp_hashtable_add_cmd_ctx(struct vpsp_cmd_ctx *ctx) { struct vpsp_cmd_ctx *entry = NULL; @@ -100,6 +129,8 @@ static void vpsp_hashtable_add_cmd_ctx(struct vpsp_cmd_ctx *ctx) write_unlock(&table_rwlock); vpsp_cmd_ctx_obj_get(ctx); + + pr_debug("add cmd ctx gpa 0x%llx pid %d\n", ctx->key1, ctx->key2); } static void vpsp_hashtable_remove_cmd_ctx(struct vpsp_cmd_ctx *ctx) @@ -108,6 +139,7 @@ static void vpsp_hashtable_remove_cmd_ctx(struct vpsp_cmd_ctx *ctx) hash_del(&ctx->node); write_unlock(&table_rwlock); + pr_debug("remove cmd ctx gpa 0x%llx pid %d\n", ctx->key1, ctx->key2); vpsp_cmd_ctx_obj_put(ctx, false); } @@ -155,6 +187,11 @@ static void vpsp_cmd_ctx_destroy(struct vpsp_cmd_ctx *cmd_ctx) { if (!cmd_ctx) return; + + if (cmd_ctx->statval == VPSP_RUNNING) + pr_warn("destroy cmd_ctx is running, gpa 0x%llx, pid %d", + cmd_ctx->key1, cmd_ctx->key2); + /** * The initial refcount is 1, * need to additional decrement a refcount. @@ -457,15 +494,28 @@ static int vpsp_try_bind_vtkm(struct kvm_vpsp *vpsp, struct vpsp_dev_ctx *vpsp_c int ret; struct vpsp_cmd *vcmd = (struct vpsp_cmd *)&cmd; - if (vpsp_ctx && !vpsp_ctx->vm_is_bound && vpsp->is_csv_guest) { + /** + * The vpsp_ctx->locked ensures that kvm_bind_vtkm is + * only executed once. + * + * otherwise error code -62 will be thrown. + */ + while (vpsp_ctx && !vpsp_ctx->vm_is_bound && vpsp->is_csv_guest) { + if (atomic64_xchg(&vpsp_ctx->locked, 1)) { + cond_resched(); + continue; + } + ret = kvm_bind_vtkm(vpsp->vm_handle, vcmd->cmd_id, vpsp_ctx->vid, psp_ret); if (ret || *psp_ret) { pr_err("[%s] kvm bind vtkm failed with ret: %d, pspret: %d\n", __func__, ret, *psp_ret); + atomic64_xchg(&vpsp_ctx->locked, 0); return ret; } vpsp_ctx->vm_is_bound = 1; + atomic64_xchg(&vpsp_ctx->locked, 0); } return 0; } @@ -789,15 +839,34 @@ static int vpsp_del_vid(void) { pid_t cur_pid = task_pid_nr(current); int i, ret = -ENOENT; + struct vpsp_cmd_ctx **cmd_ctx = NULL; + uint32_t cmd_ctx_nums = 0; + int j; write_lock(&vpsp_dev_rwlock); for (i = 0; i < g_vpsp_vid_num; ++i) { if (g_vpsp_context_array[i].pid == cur_pid) { + /** + * In some cases, such as live migration, + * there may be residual cmd ctx that needs to be forcibly destroyed. + */ + vpsp_hashtable_save_cmd_ctx_by_pid(cur_pid, NULL, &cmd_ctx_nums); + cmd_ctx = kcalloc(cmd_ctx_nums, sizeof(struct vpsp_cmd_ctx *), GFP_KERNEL); + if (!cmd_ctx) { + ret = -ENOMEM; + goto end; + } + + vpsp_hashtable_save_cmd_ctx_by_pid(cur_pid, cmd_ctx, &cmd_ctx_nums); + for (j = 0; j < cmd_ctx_nums; ++j) + vpsp_cmd_ctx_destroy(cmd_ctx[j]); + --g_vpsp_vid_num; pr_info("PSP: delete vid %d, by pid %d, total vid num is %d\n", g_vpsp_context_array[i].vid, cur_pid, g_vpsp_vid_num); memmove(&g_vpsp_context_array[i], &g_vpsp_context_array[i + 1], sizeof(struct vpsp_dev_ctx) * (g_vpsp_vid_num - i)); + ret = 0; goto end; } @@ -805,6 +874,7 @@ static int vpsp_del_vid(void) end: write_unlock(&vpsp_dev_rwlock); + kfree(cmd_ctx); return ret; } @@ -852,10 +922,26 @@ int do_vpsp_op_ioctl(struct vpsp_dev_ctrl *ctrl) ret = vpsp_set_gpa_range(ctrl->data.gpa.gpa_start, ctrl->data.gpa.gpa_end); break; + case VPSP_OP_BACKUP_KEY: + ret = vpsp_backup_key(&ctrl->data.key_img_ctl); + break; + + case VPSP_OP_RESTORE_KEY: + ret = vpsp_restore_key(&ctrl->data.key_img_ctl); + break; + + case VPSP_OP_BACKUP_CTX: + ret = vpsp_backup_cmd_ctx(&ctrl->data.cmd_ctx_ctl); + break; + + case VPSP_OP_RESTORE_CTX: + ret = vpsp_restore_cmd_ctx(&ctrl->data.cmd_ctx_ctl); + break; default: ret = -EINVAL; break; } + return ret; } @@ -1012,7 +1098,7 @@ int vpsp_try_get_result(struct vpsp_cmd_ctx *cmd_ctx, struct vpsp_ret *psp_ret) } } else { psp_worker_register_notify(vpsp_ringbuffer_wakeup_locked); - ret = psp_do_ringbuffer_cmds_locked(vpsp_ring_buffer, (int *)psp_ret); + ret = psp_do_ringbuffer_cmds_locked(vpsp_ring_buffer, (int *)psp_ret, true); if (unlikely(ret)) { pr_err("[%s]: psp ringbuf execute failed %d\n", __func__, ret); @@ -1126,3 +1212,502 @@ int vpsp_parse_ringbuffer_cmd_prio(uint8_t *prio, return rb_supported; } + +static int vpsp_backup_key(struct key_img_ctl *ctl) +{ + int ret = 0; + int pspret = 0; + uint32_t cmdbuf_size; + uint8_t *img; + struct vtkm_cmdresp_key_backup *cmdresp = NULL; + struct vpsp_dev_ctx *vpsp_dev_ctx = NULL; + uint32_t image_offset; + pid_t cur_pid = task_tgid_nr(current); + + if (!ctl) + return -EINVAL; + + vpsp_get_dev_ctx(&vpsp_dev_ctx, cur_pid); + if (!vpsp_dev_ctx) { + pr_err("PSP: %s get vpsp_context failed from pid %d\n", __func__, cur_pid); + return -ENOENT; + } + + cmdbuf_size = ctl->img_len + sizeof(struct vtkm_cmdresp_key_backup); + cmdbuf_size = cmdbuf_size > TKM_CMDRESP_MIN_SIZE ? cmdbuf_size : TKM_CMDRESP_MIN_SIZE; + cmdresp = kzalloc(cmdbuf_size, GFP_KERNEL); + if (!cmdresp) + return -ENOMEM; + + cmdresp->head.buf_size = cmdbuf_size; + cmdresp->head.cmdresp_size = cmdbuf_size; + cmdresp->head.cmdresp_code = VTKM_KEY_BACKUP; + cmdresp->vid = vpsp_dev_ctx->vid; + cmdresp->image_length = 0; + cmdresp->scheme = KEY_PROT_SAME_GENERATION; + + // get actual image length + if (!ctl->img_len) { + ret = psp_do_cmd(TKM_PSP_CMDID_OFFSET, cmdresp, &pspret); + if (ret == -EIO && pspret == TKM_ERR_SIZE_SMALL) { + ctl->img_len = cmdresp->image_length; + ret = 0; + } else { + pr_err("psp_do_cmd ret %d, pspret %d\n", ret, pspret); + } + goto end; + } + + image_offset = sizeof(*cmdresp); + cmdresp->image_location = LOC_FILL_MODE_BIT(image_offset); + cmdresp->image_length = ctl->img_len; + + ret = psp_do_cmd(TKM_PSP_CMDID_OFFSET, cmdresp, &pspret); + if (!ret && !pspret) { + img = (uint8_t *)(((uintptr_t)cmdresp) + image_offset); + ctl->img_len = cmdresp->image_length; + + if (copy_to_user(ctl->key_img_ptr, img, ctl->img_len)) + return -EFAULT; + } else { + pr_err("psp_do_cmd ret %d, pspret %d\n", ret, pspret); + } + +end: + kfree(cmdresp); + return ret; +} + +static int vpsp_restore_key(struct key_img_ctl *ctl) +{ + int ret = 0; + int pspret = 0; + uint32_t cmdbuf_size, image_offset; + uint8_t *img; + struct vtkm_cmdresp_key_restore *cmdresp = NULL; + struct vpsp_dev_ctx *vpsp_dev_ctx = NULL; + pid_t cur_pid = task_tgid_nr(current); + + if (!ctl || !ctl->img_len) + return -EINVAL; + + vpsp_get_dev_ctx(&vpsp_dev_ctx, cur_pid); + if (!vpsp_dev_ctx) { + pr_err("PSP: %s get vpsp_context failed from pid %d\n", __func__, cur_pid); + return -ENOENT; + } + + cmdbuf_size = ctl->img_len + sizeof(struct vtkm_cmdresp_key_restore); + cmdbuf_size = cmdbuf_size > TKM_CMDRESP_MIN_SIZE ? cmdbuf_size : TKM_CMDRESP_MIN_SIZE; + cmdresp = kzalloc(cmdbuf_size, GFP_KERNEL); + if (!cmdresp) + return -ENOMEM; + + image_offset = sizeof(*cmdresp); + + cmdresp->head.buf_size = cmdbuf_size; + cmdresp->head.cmdresp_size = cmdbuf_size; + cmdresp->head.cmdresp_code = VTKM_KEY_RESTORE; + cmdresp->vid = vpsp_dev_ctx->vid; + cmdresp->scheme = KEY_PROT_SAME_GENERATION; + cmdresp->image_location = LOC_FILL_MODE_BIT(image_offset); + cmdresp->image_length = ctl->img_len; + + img = (uint8_t *)(((uintptr_t)cmdresp) + image_offset); + if (copy_from_user(img, ctl->key_img_ptr, ctl->img_len)) { + ret = -EFAULT; + goto end; + } + + ret = psp_do_cmd(TKM_PSP_CMDID_OFFSET, cmdresp, &pspret); +end: + kfree(cmdresp); + return ret; +} + +/** + * vpsp_cmd_ctx_serialize - Serialize an array of vpsp command contexts + * @ctx: Array of command context pointers to serialize + * @ctx_nums: [IN] Number of contexts in array + * [OUT] Actual number of contexts serialized (on success) + * @buf: Output buffer for serialized data (NULL for size query) + * @buflen: [IN] Size of output buffer + * [OUT] Required buffer size or actual used size + * + * VERSION 1 Serialization format: + * +------------------------------------------------+ + * | [HEADER AREA]: | + * | | + * | struct vpsp_serialized_header | + * | magic: 0x56505350 ("VPSP") | + * | buffer_len: Total serialized data length | + * | version: Format version (1) | + * | ctx_count: Number of contexts | + * +------------------------------------------------+ + * | [METADATA AREA]: | + * | | + * | struct vpsp_ctx_serialized[0] | + * | gpa: Guest Physical Address | + * | statval: Context status value | + * | data_size: Size of context data | + * | data_offset: Offset to data in buffer | + * |------------------------------------------------| + * | struct vpsp_ctx_serialized[1] | + * | ... | + * |------------------------------------------------| + * | ... (additional context metadata) | + * +------------------------------------------------+ + * | [DATA AREA]: | + * | | + * | Context 0 data | + * |------------------------------------------------| + * | Context 1 data | + * | ... | + * +------------------------------------------------+ + * + * Return: 0 on success, negative error code on failure: + * -EINVAL: Invalid parameters + * -ENOBUFS: Buffer too small (query mode) + */ +static int vpsp_cmd_ctx_serialize(struct vpsp_cmd_ctx **ctx, + uint32_t ctx_nums, uint8_t *buf, uint32_t *buflen) +{ + struct vpsp_serialized_header *header; + struct vpsp_ctx_serialized *ctx_meta; + uint32_t total_size = 0; + uint8_t *data_area = NULL; + int i; + + if (ctx_nums == 0) { + *buflen = 0; + return 0; + } + + /* calculate total size */ + total_size = sizeof(struct vpsp_serialized_header); + for (i = 0; i < ctx_nums; i++) { + total_size += sizeof(struct vpsp_ctx_serialized); + total_size += ctx[i]->data_size; + } + + /* ensure buffer is enough */ + if (*buflen < total_size) { + /* return actual buffer size */ + *buflen = total_size; + return -ENOBUFS; + } + + /* fill header */ + header = (struct vpsp_serialized_header *)buf; + header->magic = VPSP_MAGIC_NUM; + header->version = VPSP_SERIALIZED_VERSION; + header->ctx_count = ctx_nums; + header->buffer_len = total_size; + + /* calculate data storage area address */ + data_area = buf + sizeof(*header) + (ctx_nums * sizeof(struct vpsp_ctx_serialized)); + + /* fill serialize information for each vpsp cmd context */ + for (i = 0; i < ctx_nums; i++) { + ctx_meta = &header->ctx_meta[i]; + ctx_meta->gpa = ctx[i]->key1; + ctx_meta->statval = ctx[i]->statval; + ctx_meta->data_size = ctx[i]->data_size; + + /* store pointer data */ + if (ctx[i]->data_size > 0) { + if (!ctx[i]->data) + return -EINVAL; + + ctx_meta->data_offset = (uint32_t)(data_area - buf); + memcpy(data_area, ctx[i]->data, ctx[i]->data_size); + data_area += ctx[i]->data_size; + } else { + /* no data need to storage */ + ctx_meta->data_offset = 0; + } + } + + *buflen = total_size; + return 0; +} + +/** + * vpsp_cmd_ctx_deserialize - Deserialize command contexts from buffer + * @ctx: Input vpsp_cmd_ctx pointer array for store context pointers + * @ctx_nums: [IN] Capacity of context array + * [OUT] Actual number of contexts deserialized + * @buf: Input buffer containing serialized data + * @buflen: Length of input buffer + * + * Deserialization process: + * 1. Verify magic and version + * 2. Validate buffer integrity + * 3. Parse header information + * 4. Load context data from data area + * + * Return: 0 on success, negative error code on failure: + * -EINVAL: Invalid header or corrupted data + * -ENOBUFS: Insufficient context array capacity + * -ENOMEM: Memory allocation failure + */ +static int vpsp_cmd_ctx_deserialize(struct vpsp_cmd_ctx **ctx, + uint32_t *ctx_nums, uint8_t *buf, uint32_t buflen) +{ + const struct vpsp_serialized_header *header; + const struct vpsp_ctx_serialized *ctx_meta; + pid_t cur_pid = task_tgid_nr(current); + uint32_t parsed_count = 0; + int i, ret = 0; + + if (!ctx_nums || !buf) + return -EINVAL; + + if (*ctx_nums && !ctx) + return -EINVAL; + + /* check header length */ + if (buflen < sizeof(*header)) + return -EINVAL; + + header = (const struct vpsp_serialized_header *)buf; + + /* ensure data is valid */ + if (header->magic != VPSP_MAGIC_NUM || header->version != VPSP_SERIALIZED_VERSION) + return -EINVAL; + + /* check total size */ + if (buflen < header->buffer_len) + return -EINVAL; + + /* ensure the vpsp context number is enough */ + if (header->ctx_count > *ctx_nums) { + /* return actual number */ + *ctx_nums = header->ctx_count; + return -ENOBUFS; + } + + /* ensure context count is valid */ + if (header->buffer_len < header->ctx_count * sizeof(struct vpsp_ctx_serialized)) + return -EINVAL; + + /* parse data for vpsp context */ + for (i = 0; i < header->ctx_count; i++) { + if (!ctx[parsed_count]) + return -ENOBUFS; + + ctx_meta = &header->ctx_meta[i]; + ctx[parsed_count]->key1 = ctx_meta->gpa; + ctx[parsed_count]->key2 = cur_pid; + ctx[parsed_count]->statval = ctx_meta->statval; + ctx[parsed_count]->data_size = ctx_meta->data_size; + + /* alloc new buffer restore data pointer */ + if (ctx_meta->data_size > 0) { + if (ctx_meta->data_offset == 0 || + ctx_meta->data_offset + ctx_meta->data_size > buflen) { + /* offset invalid */ + ret = -EINVAL; + goto failed; + } + + ctx[parsed_count]->data = kmalloc(ctx_meta->data_size, GFP_KERNEL); + if (!ctx[parsed_count]->data) { + ret = -ENOMEM; + goto failed; + } + + memcpy(ctx[parsed_count]->data, buf + ctx_meta->data_offset, + ctx_meta->data_size); + } else { + ctx[parsed_count]->data = NULL; + } + + parsed_count++; + } + + *ctx_nums = parsed_count; +failed: + while (ret && parsed_count-- > 0) { + if (ctx[parsed_count]->data_size > 0) + kfree(ctx[parsed_count]->data); + } + return ret; +} + +static int vpsp_force_cmd_ctx_completion(struct vpsp_cmd_ctx *ctx[], uint32_t nums) +{ + int i, ret = 0, psp_ret = 0; + + for (i = 0; i < nums; ++i) { + bool mutex_locked = false; + + while (true) { + if (ctx[i]->statval != PSP_CMD_STATUS_RUNNING) + break; + + if (vpsp_psp_mutex_trylock()) { + mutex_locked = true; + break; + } + + cond_resched(); + } + + if (!mutex_locked) + continue; + + /** + * There are commands in a running state update all queued and + * pending commands via ringbuffer operations. + * + * Disable overcommit feature to avoid the additional latency + * caused by failing to fully utilize the allocated overcommit size. + */ + ret = psp_do_ringbuffer_cmds_locked(vpsp_ring_buffer, (int *)&psp_ret, false); + if (unlikely(ret)) { + pr_err("[%s]: psp ringbuffer execute failed %d\n", + __func__, ret); + vpsp_psp_mutex_unlock(); + return ret; + } + + vpsp_ringbuffer_wakeup_locked(NULL); + vpsp_psp_mutex_unlock(); + break; + } + + return ret; +} + +static int vpsp_backup_cmd_ctx(struct cmd_ctx_ctl *ctl) +{ + int ret = 0; + uint32_t buflen = 0; + pid_t cur_pid = task_tgid_nr(current); + void *cmd_ctx_buffer = NULL; + struct vpsp_cmd_ctx **cmd_ctx = NULL; + uint32_t cmd_ctx_nums = 0; + + vpsp_hashtable_save_cmd_ctx_by_pid(cur_pid, NULL, &cmd_ctx_nums); + cmd_ctx = kcalloc(cmd_ctx_nums, sizeof(struct vpsp_cmd_ctx *), GFP_KERNEL); + if (!cmd_ctx) { + ret = -ENOMEM; + goto end; + } + + vpsp_hashtable_save_cmd_ctx_by_pid(cur_pid, cmd_ctx, &cmd_ctx_nums); + + /* Check for unprocessed commands pending PSP handling */ + ret = vpsp_force_cmd_ctx_completion(cmd_ctx, cmd_ctx_nums); + if (ret) { + pr_err("%s vpsp_force_cmd_ctx_completion failed %d\n", __func__, ret); + goto end; + } + + ret = vpsp_cmd_ctx_serialize(cmd_ctx, cmd_ctx_nums, NULL, &buflen); + if (ret && ret != -ENOBUFS) { + pr_err("%s vpsp_cmd_ctx_serialize failed %d\n", __func__, ret); + goto end; + } + + if (ctl->buffer_len < buflen) { + ctl->buffer_len = buflen; + ret = 0; + goto end; + } + + cmd_ctx_buffer = kzalloc(ctl->buffer_len, GFP_KERNEL); + if (!cmd_ctx_buffer) { + ret = -ENOMEM; + goto end; + } + + ret = vpsp_cmd_ctx_serialize(cmd_ctx, cmd_ctx_nums, + cmd_ctx_buffer, &ctl->buffer_len); + if (ret) { + pr_err("%s vpsp_cmd_ctx_serialize failed %d\n", __func__, ret); + goto end; + } + + if (copy_to_user(ctl->cmd_ctx_ptr, cmd_ctx_buffer, ctl->buffer_len)) { + ret = -EFAULT; + goto end; + } + + pr_info("migrate(%d) save: serialization length %d, a total of %d cmd ctx were saved\n", + cur_pid, ctl->buffer_len, cmd_ctx_nums); +end: + kfree(cmd_ctx); + kfree(cmd_ctx_buffer); + return ret; +} + +static int vpsp_restore_cmd_ctx(struct cmd_ctx_ctl *ctl) +{ + int ret = 0, i; + uint32_t cmd_ctx_nums = 0; + void *cmd_ctx_buffer = NULL; + pid_t cur_pid = task_tgid_nr(current); + struct vpsp_cmd_ctx **load_cmd_ctx = NULL; + + pr_info("migrate(%d) load: serialization length %d\n", cur_pid, ctl->buffer_len); + + if (!ctl->buffer_len) + return 0; + + cmd_ctx_buffer = kzalloc(ctl->buffer_len, GFP_KERNEL); + if (!cmd_ctx_buffer) + return -ENOMEM; + + if (copy_from_user(cmd_ctx_buffer, ctl->cmd_ctx_ptr, ctl->buffer_len)) { + ret = -EFAULT; + goto end; + } + + ret = vpsp_cmd_ctx_deserialize(NULL, &cmd_ctx_nums, cmd_ctx_buffer, ctl->buffer_len); + if (ret && ret != -ENOBUFS) { + pr_err("%s vpsp_cmd_ctx_deserialize failed %d\n", __func__, ret); + goto end; + } + + load_cmd_ctx = kzalloc(cmd_ctx_nums * sizeof(struct vpsp_cmd_ctx *), GFP_KERNEL); + if (!load_cmd_ctx) { + ret = -ENOMEM; + goto end; + } + + for (i = 0; i < cmd_ctx_nums; ++i) { + load_cmd_ctx[i] = kmem_cache_zalloc(vpsp_cmd_ctx_slab, GFP_KERNEL); + if (!load_cmd_ctx[i]) { + ret = -ENOMEM; + goto end; + } + load_cmd_ctx[i]->key2 = cur_pid; + refcount_set(&load_cmd_ctx[i]->ref, 1); + } + + ret = vpsp_cmd_ctx_deserialize(load_cmd_ctx, &cmd_ctx_nums, + cmd_ctx_buffer, ctl->buffer_len); + if (ret) { + pr_err("%s vpsp_cmd_ctx_deserialize failed %d\n", __func__, ret); + goto end; + } + + for (i = 0; i < cmd_ctx_nums; ++i) + vpsp_hashtable_add_cmd_ctx(load_cmd_ctx[i]); + + pr_info("migrate(%d) load: %d command contexts loaded\n", cur_pid, cmd_ctx_nums); + ret = 0; +end: + if (ret) { + for (i = 0; i < cmd_ctx_nums; ++i) { + if (load_cmd_ctx[i]) + kmem_cache_free(vpsp_cmd_ctx_slab, load_cmd_ctx[i]); + } + } + + kfree(load_cmd_ctx); + kfree(cmd_ctx_buffer); + return ret; +} diff --git a/drivers/crypto/ccp/hygon/vpsp.h b/drivers/crypto/ccp/hygon/vpsp.h index ccfb68e1d523720ac05bc97b6c609a071ab5ca95..6eca68c0574b214be886cd70b0edc76deed1f33f 100644 --- a/drivers/crypto/ccp/hygon/vpsp.h +++ b/drivers/crypto/ccp/hygon/vpsp.h @@ -55,6 +55,9 @@ struct vpsp_ret { #define VPSP_RET_SYS_FORMAT 1 #define VPSP_RET_PSP_FORMAT 0 +#define VPSP_MAGIC_NUM 0x56505350 /* "VPSP" */ +#define TKM_ERR_SIZE_SMALL 5 + #define PSP_2MB_MASK (2*1024*1024 - 1) #define PSP_HUGEPAGE_2MB (2*1024*1024) #define PSP_HUGEPAGE_NUM_MAX 128 @@ -77,6 +80,7 @@ struct vpsp_dev_ctx { // `vm_is_bound` indicates whether the binding operation has been performed u32 vm_is_bound; u32 vm_handle; // only for csv + atomic64_t locked; }; struct vpsp_cmd_ctx { @@ -101,14 +105,44 @@ struct vpsp_cmd_ctx { struct hlist_node node; }; +struct vpsp_ctx_serialized { + gpa_t gpa; + uint32_t statval; + uint32_t data_size; + uint32_t data_offset; +} __packed; + +#define VPSP_SERIALIZED_VERSION 1 +struct vpsp_serialized_header { + uint32_t magic; + uint32_t buffer_len; + uint32_t version; + uint32_t ctx_count; + struct vpsp_ctx_serialized ctx_meta[]; +} __packed; + enum VPSP_DEV_CTRL_OPCODE { VPSP_OP_VID_ADD, VPSP_OP_VID_DEL, VPSP_OP_SET_DEFAULT_VID_PERMISSION, VPSP_OP_GET_DEFAULT_VID_PERMISSION, VPSP_OP_SET_GPA, + VPSP_OP_BACKUP_KEY, + VPSP_OP_RESTORE_KEY, + VPSP_OP_BACKUP_CTX, + VPSP_OP_RESTORE_CTX, }; +struct key_img_ctl { + unsigned int img_len; + void __user *key_img_ptr; +} __packed; + +struct cmd_ctx_ctl { + unsigned int buffer_len; + void __user *cmd_ctx_ptr; +} __packed; + struct vpsp_dev_ctrl { unsigned char op; /** @@ -124,6 +158,10 @@ struct vpsp_dev_ctrl { u64 gpa_start; u64 gpa_end; } gpa; + + struct key_img_ctl key_img_ctl; + struct cmd_ctx_ctl cmd_ctx_ctl; + unsigned char reserved[128]; } __packed data; };