From 44cbf45d2696f210a1fa05bf6babea30bcec7616 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 24 Jan 2016 03:40:44 +0300 Subject: [PATCH 01/82] eval: Split out typval_T dumping functions to nvim/encode.c --- src/nvim/encode.c | 954 ++++++++++++++++++++++++++++++++++++++++ src/nvim/encode.h | 57 +++ src/nvim/eval.c | 966 ++--------------------------------------- src/nvim/eval.h | 37 +- src/nvim/globals.h | 12 +- src/nvim/macros.h | 11 + src/nvim/option_defs.h | 1 + src/nvim/shada.c | 13 +- 8 files changed, 1091 insertions(+), 960 deletions(-) create mode 100644 src/nvim/encode.c create mode 100644 src/nvim/encode.h diff --git a/src/nvim/encode.c b/src/nvim/encode.c new file mode 100644 index 0000000000..da93ae98ad --- /dev/null +++ b/src/nvim/encode.c @@ -0,0 +1,954 @@ +/// @file encode.c +/// +/// File containing functions for encoding and decoding VimL values. +/// +/// Split out from eval.c. + +// TODO(ZyX-I): Move this to src/nvim/viml or src/nvim/eval + +#include + +#include "nvim/encode.h" +#include "nvim/eval.h" +#include "nvim/garray.h" +#include "nvim/message.h" +#include "nvim/macros.h" +#include "nvim/ascii.h" +#include "nvim/vim.h" // For _() +#include "nvim/lib/kvec.h" + +/// Structure representing current VimL to messagepack conversion state +typedef struct { + enum { + kMPConvDict, ///< Convert dict_T *dictionary. + kMPConvList, ///< Convert list_T *list. + kMPConvPairs, ///< Convert mapping represented as a list_T* of pairs. + } type; + union { + struct { + dict_T *dict; ///< Currently converted dictionary. + hashitem_T *hi; ///< Currently converted dictionary item. + size_t todo; ///< Amount of items left to process. + } d; ///< State of dictionary conversion. + struct { + list_T *list; ///< Currently converted list. + listitem_T *li; ///< Currently converted list item. + } l; ///< State of list or generic mapping conversion. + } data; ///< Data to convert. +} MPConvStackVal; + +/// Stack used to convert VimL values to messagepack. +typedef kvec_t(MPConvStackVal) MPConvStack; + +#ifdef INCLUDE_GENERATED_DECLARATIONS +# include "encode.c.generated.h" +#endif + +/// Msgpack callback for writing to readfile()-style list +int msgpack_list_write(void *data, const char *buf, size_t len) +{ + if (len == 0) { + return 0; + } + list_T *const list = (list_T *) data; + const char *const end = buf + len; + const char *line_end = buf; + if (list->lv_last == NULL) { + list_append_string(list, NULL, 0); + } + listitem_T *li = list->lv_last; + do { + const char *line_start = line_end; + line_end = xmemscan(line_start, NL, (size_t) (end - line_start)); + if (line_end == line_start) { + list_append_allocated_string(list, NULL); + } else { + const size_t line_length = (size_t) (line_end - line_start); + char *str; + if (li == NULL) { + str = xmemdupz(line_start, line_length); + } else { + const size_t li_len = (li->li_tv.vval.v_string == NULL + ? 0 + : STRLEN(li->li_tv.vval.v_string)); + li->li_tv.vval.v_string = xrealloc(li->li_tv.vval.v_string, + li_len + line_length + 1); + str = (char *) li->li_tv.vval.v_string + li_len; + memmove(str, line_start, line_length); + str[line_length] = 0; + } + for (size_t i = 0; i < line_length; i++) { + if (str[i] == NUL) { + str[i] = NL; + } + } + if (li == NULL) { + list_append_allocated_string(list, str); + } else { + li = NULL; + } + if (line_end == end - 1) { + list_append_allocated_string(list, NULL); + } + } + line_end++; + } while (line_end < end); + return 0; +} + +/// Abort conversion to string after a recursion error. +static bool did_echo_string_emsg = false; + +/// Show a error message when converting to msgpack value +/// +/// @param[in] msg Error message to dump. Must contain exactly two %s that +/// will be replaced with what was being dumped: first with +/// something like “F” or “function argument”, second with path +/// to the failed value. +/// @param[in] mpstack Path to the failed value. +/// @param[in] objname Dumped object name. +/// +/// @return FAIL. +static int conv_error(const char *const msg, const MPConvStack *const mpstack, + const char *const objname) + FUNC_ATTR_NONNULL_ALL +{ + garray_T msg_ga; + ga_init(&msg_ga, (int)sizeof(char), 80); + char *const key_msg = _("key %s"); + char *const key_pair_msg = _("key %s at index %i from special map"); + char *const idx_msg = _("index %i"); + for (size_t i = 0; i < kv_size(*mpstack); i++) { + if (i != 0) { + ga_concat(&msg_ga, (char_u *) ", "); + } + MPConvStackVal v = kv_A(*mpstack, i); + switch (v.type) { + case kMPConvDict: { + typval_T key_tv = { + .v_type = VAR_STRING, + .vval = { .v_string = (v.data.d.hi == NULL + ? v.data.d.dict->dv_hashtab.ht_array + : (v.data.d.hi - 1))->hi_key }, + }; + char *const key = encode_tv2string(&key_tv, NULL); + vim_snprintf((char *) IObuff, IOSIZE, key_msg, key); + xfree(key); + ga_concat(&msg_ga, IObuff); + break; + } + case kMPConvPairs: + case kMPConvList: { + int idx = 0; + const listitem_T *li; + for (li = v.data.l.list->lv_first; + li != NULL && li->li_next != v.data.l.li; + li = li->li_next) { + idx++; + } + if (v.type == kMPConvList + || li == NULL + || (li->li_tv.v_type != VAR_LIST + && li->li_tv.vval.v_list->lv_len <= 0)) { + vim_snprintf((char *) IObuff, IOSIZE, idx_msg, idx); + ga_concat(&msg_ga, IObuff); + } else { + typval_T key_tv = li->li_tv.vval.v_list->lv_first->li_tv; + char *const key = encode_tv2echo(&key_tv, NULL); + vim_snprintf((char *) IObuff, IOSIZE, key_pair_msg, key, idx); + xfree(key); + ga_concat(&msg_ga, IObuff); + } + break; + } + } + } + EMSG3(msg, objname, (kv_size(*mpstack) == 0 + ? _("itself") + : (char *) msg_ga.ga_data)); + ga_clear(&msg_ga); + return FAIL; +} + +/// Convert readfile()-style list to a char * buffer with length +/// +/// @param[in] list Converted list. +/// @param[out] ret_len Resulting buffer length. +/// @param[out] ret_buf Allocated buffer with the result or NULL if ret_len is +/// zero. +/// +/// @return true in case of success, false in case of failure. +static inline bool vim_list_to_buf(const list_T *const list, + size_t *const ret_len, char **const ret_buf) + FUNC_ATTR_NONNULL_ARG(2,3) FUNC_ATTR_WARN_UNUSED_RESULT +{ + size_t len = 0; + if (list != NULL) { + for (const listitem_T *li = list->lv_first; + li != NULL; + li = li->li_next) { + if (li->li_tv.v_type != VAR_STRING) { + return false; + } + len++; + if (li->li_tv.vval.v_string != 0) { + len += STRLEN(li->li_tv.vval.v_string); + } + } + if (len) { + len--; + } + } + *ret_len = len; + if (len == 0) { + *ret_buf = NULL; + return true; + } + ListReaderState lrstate = encode_init_lrstate(list); + char *const buf = xmalloc(len); + size_t read_bytes; + if (encode_read_from_list(&lrstate, buf, len, &read_bytes) != OK) { + assert(false); + } + assert(len == read_bytes); + *ret_buf = buf; + return true; +} + +/// Read bytes from list +/// +/// @param[in,out] state Structure describing position in list from which +/// reading should start. Is updated to reflect position +/// at which reading ended. +/// @param[out] buf Buffer to write to. +/// @param[in] nbuf Buffer length. +/// @param[out] read_bytes Is set to amount of bytes read. +/// +/// @return OK when reading was finished, FAIL in case of error (i.e. list item +/// was not a string), NOTDONE if reading was successfull, but there are +/// more bytes to read. +int encode_read_from_list(ListReaderState *const state, char *const buf, + const size_t nbuf, size_t *const read_bytes) + FUNC_ATTR_NONNULL_ALL FUNC_ATTR_WARN_UNUSED_RESULT +{ + char *const buf_end = buf + nbuf; + char *p = buf; + while (p < buf_end) { + for (size_t i = state->offset; i < state->li_length && p < buf_end; i++) { + const char ch = (char) state->li->li_tv.vval.v_string[state->offset++]; + *p++ = (ch == NL ? NUL : ch); + } + if (p < buf_end) { + state->li = state->li->li_next; + if (state->li == NULL) { + *read_bytes = (size_t) (p - buf); + return OK; + } + *p++ = NL; + if (state->li->li_tv.v_type != VAR_STRING) { + *read_bytes = (size_t) (p - buf); + return FAIL; + } + state->offset = 0; + state->li_length = (state->li->li_tv.vval.v_string == NULL + ? 0 + : STRLEN(state->li->li_tv.vval.v_string)); + } + } + *read_bytes = nbuf; + return (state->offset < state->li_length || state->li->li_next != NULL + ? NOTDONE + : OK); +} + +/// Code for checking whether container references itself +/// +/// @param[in,out] val Container to check. +/// @param copyID_attr Name of the container attribute that holds copyID. +/// After checking whether value of this attribute is +/// copyID (variable) it is set to copyID. +#define CHECK_SELF_REFERENCE(val, copyID_attr, conv_type) \ + do { \ + if ((val)->copyID_attr == copyID) { \ + CONV_RECURSE((val), conv_type); \ + } \ + (val)->copyID_attr = copyID; \ + } while (0) + +/// Define functions which convert VimL value to something else +/// +/// Creates function `vim_to_{name}(firstargtype firstargname, typval_T *const +/// tv)` which returns OK or FAIL and helper functions. +/// +/// @param firstargtype Type of the first argument. It will be used to return +/// the results. +/// @param firstargname Name of the first argument. +/// @param name Name of the target converter. +#define DEFINE_VIML_CONV_FUNCTIONS(scope, name, firstargtype, firstargname) \ +static int name##_convert_one_value(firstargtype firstargname, \ + MPConvStack *const mpstack, \ + typval_T *const tv, \ + const int copyID, \ + const char *const objname) \ + FUNC_ATTR_NONNULL_ALL FUNC_ATTR_WARN_UNUSED_RESULT \ +{ \ + switch (tv->v_type) { \ + case VAR_STRING: { \ + CONV_STRING(tv->vval.v_string, STRLEN(tv->vval.v_string)); \ + break; \ + } \ + case VAR_NUMBER: { \ + CONV_NUMBER(tv->vval.v_number); \ + break; \ + } \ + case VAR_FLOAT: { \ + CONV_FLOAT(tv->vval.v_float); \ + break; \ + } \ + case VAR_FUNC: { \ + CONV_FUNC(tv->vval.v_string); \ + break; \ + } \ + case VAR_LIST: { \ + if (tv->vval.v_list == NULL || tv->vval.v_list->lv_len == 0) { \ + CONV_EMPTY_LIST(); \ + break; \ + } \ + CHECK_SELF_REFERENCE(tv->vval.v_list, lv_copyID, kMPConvList); \ + CONV_LIST_START(tv->vval.v_list); \ + kv_push( \ + MPConvStackVal, \ + *mpstack, \ + ((MPConvStackVal) { \ + .type = kMPConvList, \ + .data = { \ + .l = { \ + .list = tv->vval.v_list, \ + .li = tv->vval.v_list->lv_first, \ + }, \ + }, \ + })); \ + break; \ + } \ + case VAR_DICT: { \ + if (tv->vval.v_dict == NULL \ + || tv->vval.v_dict->dv_hashtab.ht_used == 0) { \ + CONV_EMPTY_DICT(); \ + break; \ + } \ + const dictitem_T *type_di; \ + const dictitem_T *val_di; \ + if (CONV_ALLOW_SPECIAL \ + && tv->vval.v_dict->dv_hashtab.ht_used == 2 \ + && (type_di = dict_find((dict_T *) tv->vval.v_dict, \ + (char_u *) "_TYPE", -1)) != NULL \ + && type_di->di_tv.v_type == VAR_LIST \ + && (val_di = dict_find((dict_T *) tv->vval.v_dict, \ + (char_u *) "_VAL", -1)) != NULL) { \ + size_t i; \ + for (i = 0; i < ARRAY_SIZE(eval_msgpack_type_lists); i++) { \ + if (type_di->di_tv.vval.v_list == eval_msgpack_type_lists[i]) { \ + break; \ + } \ + } \ + if (i == ARRAY_SIZE(eval_msgpack_type_lists)) { \ + goto name##_convert_one_value_regular_dict; \ + } \ + switch ((MessagePackType) i) { \ + case kMPNil: { \ + CONV_SPECIAL_NIL(); \ + break; \ + } \ + case kMPBoolean: { \ + if (val_di->di_tv.v_type != VAR_NUMBER) { \ + goto name##_convert_one_value_regular_dict; \ + } \ + CONV_SPECIAL_BOOL(val_di->di_tv.vval.v_number); \ + break; \ + } \ + case kMPInteger: { \ + const list_T *val_list; \ + varnumber_T sign; \ + varnumber_T highest_bits; \ + varnumber_T high_bits; \ + varnumber_T low_bits; \ + /* List of 4 integers; first is signed (should be 1 or -1, but */ \ + /* this is not checked), second is unsigned and have at most */ \ + /* one (sign is -1) or two (sign is 1) non-zero bits (number of */ \ + /* bits is not checked), other unsigned and have at most 31 */ \ + /* non-zero bits (number of bits is not checked).*/ \ + if (val_di->di_tv.v_type != VAR_LIST \ + || (val_list = val_di->di_tv.vval.v_list) == NULL \ + || val_list->lv_len != 4 \ + || val_list->lv_first->li_tv.v_type != VAR_NUMBER \ + || (sign = val_list->lv_first->li_tv.vval.v_number) == 0 \ + || val_list->lv_first->li_next->li_tv.v_type != VAR_NUMBER \ + || (highest_bits = \ + val_list->lv_first->li_next->li_tv.vval.v_number) < 0 \ + || val_list->lv_last->li_prev->li_tv.v_type != VAR_NUMBER \ + || (high_bits = \ + val_list->lv_last->li_prev->li_tv.vval.v_number) < 0 \ + || val_list->lv_last->li_tv.v_type != VAR_NUMBER \ + || (low_bits = val_list->lv_last->li_tv.vval.v_number) < 0) { \ + goto name##_convert_one_value_regular_dict; \ + } \ + uint64_t number = ((uint64_t) (((uint64_t) highest_bits) << 62) \ + | (uint64_t) (((uint64_t) high_bits) << 31) \ + | (uint64_t) low_bits); \ + if (sign > 0) { \ + CONV_UNSIGNED_NUMBER(number); \ + } else { \ + CONV_NUMBER(-number); \ + } \ + break; \ + } \ + case kMPFloat: { \ + if (val_di->di_tv.v_type != VAR_FLOAT) { \ + goto name##_convert_one_value_regular_dict; \ + } \ + CONV_FLOAT(val_di->di_tv.vval.v_float); \ + break; \ + } \ + case kMPString: \ + case kMPBinary: { \ + const bool is_string = ((MessagePackType) i == kMPString); \ + if (val_di->di_tv.v_type != VAR_LIST) { \ + goto name##_convert_one_value_regular_dict; \ + } \ + size_t len; \ + char *buf; \ + if (!vim_list_to_buf(val_di->di_tv.vval.v_list, &len, &buf)) { \ + goto name##_convert_one_value_regular_dict; \ + } \ + if (is_string) { \ + CONV_STR_STRING(buf, len); \ + } else { \ + CONV_STRING(buf, len); \ + } \ + xfree(buf); \ + break; \ + } \ + case kMPArray: { \ + if (val_di->di_tv.v_type != VAR_LIST) { \ + goto name##_convert_one_value_regular_dict; \ + } \ + CHECK_SELF_REFERENCE(val_di->di_tv.vval.v_list, lv_copyID, \ + kMPConvList); \ + CONV_LIST_START(val_di->di_tv.vval.v_list); \ + kv_push(MPConvStackVal, *mpstack, ((MPConvStackVal) { \ + .type = kMPConvList, \ + .data = { \ + .l = { \ + .list = val_di->di_tv.vval.v_list, \ + .li = val_di->di_tv.vval.v_list->lv_first, \ + }, \ + }, \ + })); \ + break; \ + } \ + case kMPMap: { \ + if (val_di->di_tv.v_type != VAR_LIST) { \ + goto name##_convert_one_value_regular_dict; \ + } \ + if (val_di->di_tv.vval.v_list == NULL) { \ + CONV_EMPTY_DICT(); \ + break; \ + } \ + list_T *const val_list = val_di->di_tv.vval.v_list; \ + for (const listitem_T *li = val_list->lv_first; li != NULL; \ + li = li->li_next) { \ + if (li->li_tv.v_type != VAR_LIST \ + || li->li_tv.vval.v_list->lv_len != 2) { \ + goto name##_convert_one_value_regular_dict; \ + } \ + } \ + CHECK_SELF_REFERENCE(val_list, lv_copyID, kMPConvPairs); \ + CONV_SPECIAL_MAP_START(val_list); \ + kv_push(MPConvStackVal, *mpstack, ((MPConvStackVal) { \ + .type = kMPConvPairs, \ + .data = { \ + .l = { \ + .list = val_list, \ + .li = val_list->lv_first, \ + }, \ + }, \ + })); \ + break; \ + } \ + case kMPExt: { \ + const list_T *val_list; \ + varnumber_T type; \ + if (val_di->di_tv.v_type != VAR_LIST \ + || (val_list = val_di->di_tv.vval.v_list) == NULL \ + || val_list->lv_len != 2 \ + || (val_list->lv_first->li_tv.v_type != VAR_NUMBER) \ + || (type = val_list->lv_first->li_tv.vval.v_number) > INT8_MAX \ + || type < INT8_MIN \ + || (val_list->lv_last->li_tv.v_type != VAR_LIST)) { \ + goto name##_convert_one_value_regular_dict; \ + } \ + size_t len; \ + char *buf; \ + if (!vim_list_to_buf(val_list->lv_last->li_tv.vval.v_list, \ + &len, &buf)) { \ + goto name##_convert_one_value_regular_dict; \ + } \ + CONV_EXT_STRING(buf, len, type); \ + xfree(buf); \ + break; \ + } \ + } \ + break; \ + } \ +name##_convert_one_value_regular_dict: \ + CHECK_SELF_REFERENCE(tv->vval.v_dict, dv_copyID, kMPConvDict); \ + CONV_DICT_START(tv->vval.v_dict); \ + kv_push(MPConvStackVal, *mpstack, ((MPConvStackVal) { \ + .type = kMPConvDict, \ + .data = { \ + .d = { \ + .dict = tv->vval.v_dict, \ + .hi = tv->vval.v_dict->dv_hashtab.ht_array, \ + .todo = tv->vval.v_dict->dv_hashtab.ht_used, \ + }, \ + }, \ + })); \ + break; \ + } \ + default: { \ + EMSG2(_(e_intern2), #name "_convert_one_value()"); \ + return FAIL; \ + } \ + } \ + return OK; \ +} \ +\ +scope int encode_vim_to_##name(firstargtype firstargname, typval_T *const tv, \ + const char *const objname) \ + FUNC_ATTR_WARN_UNUSED_RESULT \ +{ \ + current_copyID += COPYID_INC; \ + const int copyID = current_copyID; \ + MPConvStack mpstack; \ + kv_init(mpstack); \ + if (name##_convert_one_value(firstargname, &mpstack, tv, copyID, objname) \ + == FAIL) { \ + goto encode_vim_to_##name##_error_ret; \ + } \ + while (kv_size(mpstack)) { \ + MPConvStackVal *cur_mpsv = &kv_A(mpstack, kv_size(mpstack) - 1); \ + typval_T *cur_tv = NULL; \ + switch (cur_mpsv->type) { \ + case kMPConvDict: { \ + if (!cur_mpsv->data.d.todo) { \ + (void) kv_pop(mpstack); \ + cur_mpsv->data.d.dict->dv_copyID = copyID - 1; \ + CONV_DICT_END(cur_mpsv->data.d.dict); \ + continue; \ + } else if (cur_mpsv->data.d.todo \ + != cur_mpsv->data.d.dict->dv_hashtab.ht_used) { \ + CONV_DICT_BETWEEN_ITEMS(cur_mpsv->data.d.dict); \ + } \ + while (HASHITEM_EMPTY(cur_mpsv->data.d.hi)) { \ + cur_mpsv->data.d.hi++; \ + } \ + dictitem_T *const di = HI2DI(cur_mpsv->data.d.hi); \ + cur_mpsv->data.d.todo--; \ + cur_mpsv->data.d.hi++; \ + CONV_STR_STRING(&di->di_key[0], STRLEN(&di->di_key[0])); \ + CONV_DICT_AFTER_KEY(cur_mpsv->data.d.dict); \ + cur_tv = &di->di_tv; \ + break; \ + } \ + case kMPConvList: { \ + if (cur_mpsv->data.l.li == NULL) { \ + (void) kv_pop(mpstack); \ + cur_mpsv->data.l.list->lv_copyID = copyID - 1; \ + CONV_LIST_END(cur_mpsv->data.l.list); \ + continue; \ + } else if (cur_mpsv->data.l.li != cur_mpsv->data.l.list->lv_first) { \ + CONV_LIST_BETWEEN_ITEMS(cur_mpsv->data.l.list); \ + } \ + cur_tv = &cur_mpsv->data.l.li->li_tv; \ + cur_mpsv->data.l.li = cur_mpsv->data.l.li->li_next; \ + break; \ + } \ + case kMPConvPairs: { \ + if (cur_mpsv->data.l.li == NULL) { \ + (void) kv_pop(mpstack); \ + cur_mpsv->data.l.list->lv_copyID = copyID - 1; \ + continue; \ + } \ + const list_T *const kv_pair = cur_mpsv->data.l.li->li_tv.vval.v_list; \ + if (name##_convert_one_value(firstargname, &mpstack, \ + &kv_pair->lv_first->li_tv, copyID, \ + objname) == FAIL) { \ + goto encode_vim_to_##name##_error_ret; \ + } \ + cur_tv = &kv_pair->lv_last->li_tv; \ + cur_mpsv->data.l.li = cur_mpsv->data.l.li->li_next; \ + break; \ + } \ + } \ + if (name##_convert_one_value(firstargname, &mpstack, cur_tv, copyID, \ + objname) == FAIL) { \ + goto encode_vim_to_##name##_error_ret; \ + } \ + } \ + kv_destroy(mpstack); \ + return OK; \ +encode_vim_to_##name##_error_ret: \ + kv_destroy(mpstack); \ + return FAIL; \ +} + +#define CONV_STRING(buf, len) \ + do { \ + const char *const buf_ = (const char *) buf; \ + if (buf == NULL) { \ + ga_concat(gap, (char_u *) "''"); \ + } else { \ + const size_t len_ = (len); \ + size_t num_quotes = 0; \ + for (size_t i = 0; i < len_; i++) { \ + if (buf_[i] == '\'') { \ + num_quotes++; \ + } \ + } \ + ga_grow(gap, (int) (2 + len_ + num_quotes)); \ + ga_append(gap, '\''); \ + for (size_t i = 0; i < len_; i++) { \ + if (buf_[i] == '\'') { \ + num_quotes++; \ + ga_append(gap, '\''); \ + } \ + ga_append(gap, buf_[i]); \ + } \ + ga_append(gap, '\''); \ + } \ + } while (0) + +#define CONV_STR_STRING(buf, len) \ + CONV_STRING(buf, len) + +#define CONV_EXT_STRING(buf, len, type) + +#define CONV_NUMBER(num) \ + do { \ + char numbuf[NUMBUFLEN]; \ + vim_snprintf(numbuf, NUMBUFLEN - 1, "%" PRId64, (int64_t) (num)); \ + ga_concat(gap, (char_u *) numbuf); \ + } while (0) + +#define CONV_FLOAT(flt) \ + do { \ + const float_T flt_ = (flt); \ + switch (fpclassify(flt_)) { \ + case FP_NAN: { \ + ga_concat(gap, (char_u *) "str2float('nan')"); \ + break; \ + } \ + case FP_INFINITE: { \ + if (flt_ < 0) { \ + ga_append(gap, '-'); \ + } \ + ga_concat(gap, (char_u *) "str2float('inf')"); \ + break; \ + } \ + default: { \ + char numbuf[NUMBUFLEN]; \ + vim_snprintf(numbuf, NUMBUFLEN - 1, "%g", flt_); \ + ga_concat(gap, (char_u *) numbuf); \ + } \ + } \ + } while (0) + +#define CONV_FUNC(fun) \ + do { \ + ga_concat(gap, (char_u *) "function("); \ + CONV_STRING(fun, STRLEN(fun)); \ + ga_append(gap, ')'); \ + } while (0) + +#define CONV_EMPTY_LIST() \ + ga_concat(gap, (char_u *) "[]") + +#define CONV_LIST_START(lst) \ + ga_append(gap, '[') + +#define CONV_EMPTY_DICT() \ + ga_concat(gap, (char_u *) "{}") + +#define CONV_SPECIAL_NIL() + +#define CONV_SPECIAL_BOOL(num) + +#define CONV_UNSIGNED_NUMBER(num) + +#define CONV_SPECIAL_MAP_START(lst) + +#define CONV_DICT_START(dct) \ + ga_append(gap, '{') + +#define CONV_DICT_END(dct) \ + ga_append(gap, '}') + +#define CONV_DICT_AFTER_KEY(dct) \ + ga_concat(gap, (char_u *) ": ") + +#define CONV_DICT_BETWEEN_ITEMS(dct) \ + ga_concat(gap, (char_u *) ", ") + +#define CONV_LIST_END(lst) \ + ga_append(gap, ']') + +#define CONV_LIST_BETWEEN_ITEMS(lst) \ + CONV_DICT_BETWEEN_ITEMS(NULL) + +#define CONV_RECURSE(val, conv_type) \ + do { \ + if (!did_echo_string_emsg) { \ + /* Only give this message once for a recursive call to avoid */ \ + /* flooding the user with errors. */ \ + did_echo_string_emsg = true; \ + EMSG(_("E724: unable to correctly dump variable " \ + "with self-referencing container")); \ + } \ + char ebuf[NUMBUFLEN + 7]; \ + size_t backref = 0; \ + for (; backref < kv_size(*mpstack); backref++) { \ + const MPConvStackVal mpval = kv_a(MPConvStackVal, *mpstack, backref); \ + if (mpval.type == conv_type) { \ + if (conv_type == kMPConvDict) { \ + if ((void *) mpval.data.d.dict == (void *) (val)) { \ + break; \ + } \ + } else if (conv_type == kMPConvList) { \ + if ((void *) mpval.data.l.list == (void *) (val)) { \ + break; \ + } \ + } \ + } \ + } \ + vim_snprintf(ebuf, NUMBUFLEN + 6, "{E724@%zu}", backref); \ + ga_concat(gap, (char_u *) &ebuf[0]); \ + return OK; \ + } while (0) + +#define CONV_ALLOW_SPECIAL false + +DEFINE_VIML_CONV_FUNCTIONS(static, string, garray_T *const, gap) + +#undef CONV_RECURSE +#define CONV_RECURSE(val, conv_type) \ + do { \ + char ebuf[NUMBUFLEN + 7]; \ + size_t backref = 0; \ + for (; backref < kv_size(*mpstack); backref++) { \ + const MPConvStackVal mpval = kv_a(MPConvStackVal, *mpstack, backref); \ + if (mpval.type == conv_type) { \ + if (conv_type == kMPConvDict) { \ + if ((void *) mpval.data.d.dict == (void *) val) { \ + break; \ + } \ + } else if (conv_type == kMPConvList) { \ + if ((void *) mpval.data.l.list == (void *) val) { \ + break; \ + } \ + } \ + } \ + } \ + if (conv_type == kMPConvDict) { \ + vim_snprintf(ebuf, NUMBUFLEN + 6, "{...@%zu}", backref); \ + } else { \ + vim_snprintf(ebuf, NUMBUFLEN + 6, "[...@%zu]", backref); \ + } \ + ga_concat(gap, (char_u *) &ebuf[0]); \ + return OK; \ + } while (0) + +DEFINE_VIML_CONV_FUNCTIONS(, echo, garray_T *const, gap) + +#undef CONV_STRING +#undef CONV_STR_STRING +#undef CONV_EXT_STRING +#undef CONV_NUMBER +#undef CONV_FLOAT +#undef CONV_FUNC +#undef CONV_EMPTY_LIST +#undef CONV_LIST_START +#undef CONV_EMPTY_DICT +#undef CONV_SPECIAL_NIL +#undef CONV_SPECIAL_BOOL +#undef CONV_UNSIGNED_NUMBER +#undef CONV_SPECIAL_MAP_START +#undef CONV_DICT_START +#undef CONV_DICT_END +#undef CONV_DICT_AFTER_KEY +#undef CONV_DICT_BETWEEN_ITEMS +#undef CONV_LIST_END +#undef CONV_LIST_BETWEEN_ITEMS +#undef CONV_RECURSE +#undef CONV_ALLOW_SPECIAL + +/// Return a string with the string representation of a variable. +/// Puts quotes around strings, so that they can be parsed back by eval(). +/// +/// @param[in] tv typval_T to convert. +/// @param[out] len Location where length of the result will be saved. +/// +/// @return String representation of the variable or NULL. +char *encode_tv2string(typval_T *tv, size_t *len) + FUNC_ATTR_NONNULL_ARG(1) FUNC_ATTR_MALLOC +{ + garray_T ga; + ga_init(&ga, (int)sizeof(char), 80); + encode_vim_to_string(&ga, tv, "encode_tv2string() argument"); + did_echo_string_emsg = false; + if (len != NULL) { + *len = (size_t) ga.ga_len; + } + ga_append(&ga, '\0'); + return (char *) ga.ga_data; +} + +/// Return a string with the string representation of a variable. +/// Does not put quotes around strings, as ":echo" displays values. +/// +/// @param[in] tv typval_T to convert. +/// @param[out] len Location where length of the result will be saved. +/// +/// @return String representation of the variable or NULL. +char *encode_tv2echo(typval_T *tv, size_t *len) + FUNC_ATTR_NONNULL_ARG(1) FUNC_ATTR_MALLOC +{ + garray_T ga; + ga_init(&ga, (int)sizeof(char), 80); + if (tv->v_type == VAR_STRING || tv->v_type == VAR_FUNC) { + if (tv->vval.v_string != NULL) { + ga_concat(&ga, tv->vval.v_string); + } + } else { + encode_vim_to_echo(&ga, tv, ":echo argument"); + } + if (len != NULL) { + *len = (size_t) ga.ga_len; + } + ga_append(&ga, '\0'); + return (char *) ga.ga_data; +} + +#define CONV_STRING(buf, len) \ + do { \ + if (buf == NULL) { \ + msgpack_pack_bin(packer, 0); \ + } else { \ + const size_t len_ = (len); \ + msgpack_pack_bin(packer, len_); \ + msgpack_pack_bin_body(packer, buf, len_); \ + } \ + } while (0) + +#define CONV_STR_STRING(buf, len) \ + do { \ + if (buf == NULL) { \ + msgpack_pack_str(packer, 0); \ + } else { \ + const size_t len_ = (len); \ + msgpack_pack_str(packer, len_); \ + msgpack_pack_str_body(packer, buf, len_); \ + } \ + } while (0) + +#define CONV_EXT_STRING(buf, len, type) \ + do { \ + if (buf == NULL) { \ + msgpack_pack_ext(packer, 0, (int8_t) type); \ + } else { \ + const size_t len_ = (len); \ + msgpack_pack_ext(packer, len_, (int8_t) type); \ + msgpack_pack_ext_body(packer, buf, len_); \ + } \ + } while (0) + +#define CONV_NUMBER(num) \ + msgpack_pack_int64(packer, (int64_t) (num)) + +#define CONV_FLOAT(flt) \ + msgpack_pack_double(packer, (double) (flt)) + +#define CONV_FUNC(fun) \ + return conv_error(_("E951: Error while dumping %s, %s: " \ + "attempt to dump function reference"), \ + mpstack, objname) + +#define CONV_EMPTY_LIST() \ + msgpack_pack_array(packer, 0) + +#define CONV_LIST_START(lst) \ + msgpack_pack_array(packer, (size_t) (lst)->lv_len) + +#define CONV_EMPTY_DICT() \ + msgpack_pack_map(packer, 0) + +#define CONV_SPECIAL_NIL() \ + msgpack_pack_nil(packer) + +#define CONV_SPECIAL_BOOL(num) \ + do { \ + if ((num)) { \ + msgpack_pack_true(packer); \ + } else { \ + msgpack_pack_false(packer); \ + } \ + } while (0) + +#define CONV_UNSIGNED_NUMBER(num) \ + msgpack_pack_uint64(packer, (num)) + +#define CONV_SPECIAL_MAP_START(lst) \ + msgpack_pack_map(packer, (size_t) (lst)->lv_len) + +#define CONV_DICT_START(dct) \ + msgpack_pack_map(packer, (dct)->dv_hashtab.ht_used) + +#define CONV_DICT_END(dct) + +#define CONV_DICT_AFTER_KEY(dct) + +#define CONV_DICT_BETWEEN_ITEMS(dct) + +#define CONV_LIST_END(lst) + +#define CONV_LIST_BETWEEN_ITEMS(lst) + +#define CONV_RECURSE(val, conv_type) \ + return conv_error(_("E952: Unable to dump %s: " \ + "container references itself in %s"), \ + mpstack, objname) + +#define CONV_ALLOW_SPECIAL true + +DEFINE_VIML_CONV_FUNCTIONS(, msgpack, msgpack_packer *const, packer) + +#undef CONV_STRING +#undef CONV_STR_STRING +#undef CONV_EXT_STRING +#undef CONV_NUMBER +#undef CONV_FLOAT +#undef CONV_FUNC +#undef CONV_EMPTY_LIST +#undef CONV_LIST_START +#undef CONV_EMPTY_DICT +#undef CONV_SPECIAL_NIL +#undef CONV_SPECIAL_BOOL +#undef CONV_UNSIGNED_NUMBER +#undef CONV_SPECIAL_MAP_START +#undef CONV_DICT_START +#undef CONV_DICT_END +#undef CONV_DICT_AFTER_KEY +#undef CONV_DICT_BETWEEN_ITEMS +#undef CONV_LIST_END +#undef CONV_LIST_BETWEEN_ITEMS +#undef CONV_RECURSE +#undef CONV_ALLOW_SPECIAL diff --git a/src/nvim/encode.h b/src/nvim/encode.h new file mode 100644 index 0000000000..799850aab9 --- /dev/null +++ b/src/nvim/encode.h @@ -0,0 +1,57 @@ +#ifndef NVIM_ENCODE_H +#define NVIM_ENCODE_H + +#include + +#include + +#include "nvim/eval.h" +#include "nvim/garray.h" +#include "nvim/vim.h" // For STRLEN + +/// Convert VimL value to msgpack string +/// +/// @param[out] packer Packer to save results in. +/// @param[in] tv Dumped value. +/// @param[in] objname Object name, used for error message. +/// +/// @return OK in case of success, FAIL otherwise. +int encode_vim_to_msgpack(msgpack_packer *const packer, + typval_T *const tv, + const char *const objname); + +/// Convert VimL value to :echo output +/// +/// @param[out] packer Packer to save results in. +/// @param[in] tv Dumped value. +/// @param[in] objname Object name, used for error message. +/// +/// @return OK in case of success, FAIL otherwise. +int encode_vim_to_echo(garray_T *const packer, + typval_T *const tv, + const char *const objname); + +/// Structure defining state for read_from_list() +typedef struct { + const listitem_T *li; ///< Item currently read. + size_t offset; ///< Byte offset inside the read item. + size_t li_length; ///< Length of the string inside the read item. +} ListReaderState; + +/// Initialize ListReaderState structure +static inline ListReaderState encode_init_lrstate(const list_T *const list) + FUNC_ATTR_NONNULL_ALL +{ + return (ListReaderState) { + .li = list->lv_first, + .offset = 0, + .li_length = (list->lv_first->li_tv.vval.v_string == NULL + ? 0 + : STRLEN(list->lv_first->li_tv.vval.v_string)), + }; +} + +#ifdef INCLUDE_GENERATED_DECLARATIONS +# include "encode.h.generated.h" +#endif +#endif // NVIM_ENCODE_H diff --git a/src/nvim/eval.c b/src/nvim/eval.c index 0d6e3d3ca3..b633bfb1b1 100644 --- a/src/nvim/eval.c +++ b/src/nvim/eval.c @@ -73,6 +73,7 @@ #include "nvim/undo.h" #include "nvim/version.h" #include "nvim/window.h" +#include "nvim/encode.h" #include "nvim/os/os.h" #include "nvim/event/libuv_process.h" #include "nvim/event/pty_process.h" @@ -87,7 +88,6 @@ #include "nvim/os/dl.h" #include "nvim/os/input.h" #include "nvim/event/loop.h" -#include "nvim/lib/kvec.h" #include "nvim/lib/queue.h" #define DICT_MAXNEST 100 /* maximum nesting of lists and dicts */ @@ -142,13 +142,6 @@ typedef struct lval_S { char_u *ll_newkey; /* New key for Dict in alloc. mem or NULL. */ } lval_T; -/// Structure defining state for read_from_list() -typedef struct { - const listitem_T *li; ///< Item currently read. - size_t offset; ///< Byte offset inside the read item. - size_t li_length; ///< Length of the string inside the read item. -} ListReaderState; - static char *e_letunexp = N_("E18: Unexpected characters in :let"); static char *e_listidx = N_("E684: list index out of range: %" PRId64); @@ -185,17 +178,9 @@ static dictitem_T globvars_var; /* variable used for g: */ */ static hashtab_T compat_hashtab; -/* - * When recursively copying lists and dicts we need to remember which ones we - * have done to avoid endless recursiveness. This unique ID is used for that. - * The last bit is used for previous_funccal, ignored when comparing. - */ -static int current_copyID = 0; -#define COPYID_INC 2 -#define COPYID_MASK (~0x1) +int current_copyID = 0; -/// Abort conversion to string after a recursion error. -static bool did_echo_string_emsg = false; +hashtab_T func_hashtab; /* * Array to hold the hashtab with variables local to each sourced script. @@ -419,29 +404,6 @@ typedef struct dict_watcher { bool busy; // prevent recursion if the dict is changed in the callback } DictWatcher; -/// Structure representing current VimL to messagepack conversion state -typedef struct { - enum { - kMPConvDict, ///< Convert dict_T *dictionary. - kMPConvList, ///< Convert list_T *list. - kMPConvPairs, ///< Convert mapping represented as a list_T* of pairs. - } type; - union { - struct { - dict_T *dict; ///< Currently converted dictionary. - hashitem_T *hi; ///< Currently converted dictionary item. - size_t todo; ///< Amount of items left to process. - } d; ///< State of dictionary conversion. - struct { - list_T *list; ///< Currently converted list. - listitem_T *li; ///< Currently converted list item. - } l; ///< State of list or generic mapping conversion. - } data; ///< Data to convert. -} MPConvStackVal; - -/// Stack used to convert VimL values to messagepack. -typedef kvec_t(MPConvStackVal) MPConvStack; - typedef struct { TerminalJobData *data; ufunc_T *callback; @@ -460,17 +422,6 @@ typedef struct { static uint64_t current_job_id = 1; static PMap(uint64_t) *jobs = NULL; -typedef enum { - kMPNil, - kMPBoolean, - kMPInteger, - kMPFloat, - kMPString, - kMPBinary, - kMPArray, - kMPMap, - kMPExt, -} MessagePackType; static const char *const msgpack_type_names[] = { [kMPNil] = "nil", [kMPBoolean] = "boolean", @@ -482,7 +433,7 @@ static const char *const msgpack_type_names[] = { [kMPMap] = "map", [kMPExt] = "ext", }; -static const list_T *msgpack_type_lists[] = { +const list_T *eval_msgpack_type_lists[] = { [kMPNil] = NULL, [kMPBoolean] = NULL, [kMPInteger] = NULL, @@ -538,7 +489,7 @@ void eval_init(void) .v_type = VAR_LIST, .vval = { .v_list = type_list, }, }; - msgpack_type_lists[i] = type_list; + eval_msgpack_type_lists[i] = type_list; if (dict_add(msgpack_types_dict, di) == FAIL) { // There must not be duplicate items in this dictionary by definition. assert(false); @@ -1733,7 +1684,7 @@ static char_u *list_arg_vars(exarg_T *eap, char_u *arg, int *first) } else { int c; - char_u *s = (char_u *) echo_string(&tv, NULL); + char_u *s = (char_u *) encode_tv2echo(&tv, NULL); c = *arg; *arg = NUL; list_one_var_a((char_u *)"", @@ -5493,7 +5444,7 @@ static int list_join_inner(garray_T *const gap, list_T *const l, for (item = l->lv_first; item != NULL && !got_int; item = item->li_next) { char *s; size_t len; - s = echo_string(&item->li_tv, &len); + s = encode_tv2echo(&item->li_tv, &len); if (s == NULL) return FAIL; @@ -5503,9 +5454,6 @@ static int list_join_inner(garray_T *const gap, list_T *const l, p->tofree = p->s = (char_u *) s; line_breakcheck(); - if (did_echo_string_emsg) { // recursion error, bail out - break; - } } /* Allocate result buffer with its total size, avoid re-allocation and @@ -6529,578 +6477,6 @@ failret: return OK; } -#define CHECK_SELF_REFERENCE(val, copyID_attr, conv_type) \ - do { \ - if ((val)->copyID_attr == copyID) { \ - CONV_RECURSE((val), conv_type); \ - } \ - (val)->copyID_attr = copyID; \ - } while (0) - -/// Define functions which convert VimL value to something else -/// -/// Creates function `vim_to_{name}(firstargtype firstargname, typval_T *const -/// tv)` which returns OK or FAIL and helper functions. -/// -/// @param firstargtype Type of the first argument. It will be used to return -/// the results. -/// @param firstargname Name of the first argument. -/// @param name Name of the target converter. -#define DEFINE_VIML_CONV_FUNCTIONS(scope, name, firstargtype, firstargname) \ -static int name##_convert_one_value(firstargtype firstargname, \ - MPConvStack *const mpstack, \ - typval_T *const tv, \ - const int copyID, \ - const char *const objname) \ - FUNC_ATTR_NONNULL_ALL FUNC_ATTR_WARN_UNUSED_RESULT \ -{ \ - switch (tv->v_type) { \ - case VAR_STRING: { \ - CONV_STRING(tv->vval.v_string, STRLEN(tv->vval.v_string)); \ - break; \ - } \ - case VAR_NUMBER: { \ - CONV_NUMBER(tv->vval.v_number); \ - break; \ - } \ - case VAR_FLOAT: { \ - CONV_FLOAT(tv->vval.v_float); \ - break; \ - } \ - case VAR_FUNC: { \ - CONV_FUNC(tv->vval.v_string); \ - break; \ - } \ - case VAR_LIST: { \ - if (tv->vval.v_list == NULL || tv->vval.v_list->lv_len == 0) { \ - CONV_EMPTY_LIST(); \ - break; \ - } \ - CHECK_SELF_REFERENCE(tv->vval.v_list, lv_copyID, kMPConvList); \ - CONV_LIST_START(tv->vval.v_list); \ - kv_push( \ - MPConvStackVal, \ - *mpstack, \ - ((MPConvStackVal) { \ - .type = kMPConvList, \ - .data = { \ - .l = { \ - .list = tv->vval.v_list, \ - .li = tv->vval.v_list->lv_first, \ - }, \ - }, \ - })); \ - break; \ - } \ - case VAR_DICT: { \ - if (tv->vval.v_dict == NULL \ - || tv->vval.v_dict->dv_hashtab.ht_used == 0) { \ - CONV_EMPTY_DICT(); \ - break; \ - } \ - const dictitem_T *type_di; \ - const dictitem_T *val_di; \ - if (CONV_ALLOW_SPECIAL \ - && tv->vval.v_dict->dv_hashtab.ht_used == 2 \ - && (type_di = dict_find((dict_T *) tv->vval.v_dict, \ - (char_u *) "_TYPE", -1)) != NULL \ - && type_di->di_tv.v_type == VAR_LIST \ - && (val_di = dict_find((dict_T *) tv->vval.v_dict, \ - (char_u *) "_VAL", -1)) != NULL) { \ - size_t i; \ - for (i = 0; i < ARRAY_SIZE(msgpack_type_lists); i++) { \ - if (type_di->di_tv.vval.v_list == msgpack_type_lists[i]) { \ - break; \ - } \ - } \ - if (i == ARRAY_SIZE(msgpack_type_lists)) { \ - goto name##_convert_one_value_regular_dict; \ - } \ - switch ((MessagePackType) i) { \ - case kMPNil: { \ - CONV_SPECIAL_NIL(); \ - break; \ - } \ - case kMPBoolean: { \ - if (val_di->di_tv.v_type != VAR_NUMBER) { \ - goto name##_convert_one_value_regular_dict; \ - } \ - CONV_SPECIAL_BOOL(val_di->di_tv.vval.v_number); \ - break; \ - } \ - case kMPInteger: { \ - const list_T *val_list; \ - varnumber_T sign; \ - varnumber_T highest_bits; \ - varnumber_T high_bits; \ - varnumber_T low_bits; \ - /* List of 4 integers; first is signed (should be 1 or -1, but */ \ - /* this is not checked), second is unsigned and have at most */ \ - /* one (sign is -1) or two (sign is 1) non-zero bits (number of */ \ - /* bits is not checked), other unsigned and have at most 31 */ \ - /* non-zero bits (number of bits is not checked).*/ \ - if (val_di->di_tv.v_type != VAR_LIST \ - || (val_list = val_di->di_tv.vval.v_list) == NULL \ - || val_list->lv_len != 4 \ - || val_list->lv_first->li_tv.v_type != VAR_NUMBER \ - || (sign = val_list->lv_first->li_tv.vval.v_number) == 0 \ - || val_list->lv_first->li_next->li_tv.v_type != VAR_NUMBER \ - || (highest_bits = \ - val_list->lv_first->li_next->li_tv.vval.v_number) < 0 \ - || val_list->lv_last->li_prev->li_tv.v_type != VAR_NUMBER \ - || (high_bits = \ - val_list->lv_last->li_prev->li_tv.vval.v_number) < 0 \ - || val_list->lv_last->li_tv.v_type != VAR_NUMBER \ - || (low_bits = val_list->lv_last->li_tv.vval.v_number) < 0) { \ - goto name##_convert_one_value_regular_dict; \ - } \ - uint64_t number = ((uint64_t) (((uint64_t) highest_bits) << 62) \ - | (uint64_t) (((uint64_t) high_bits) << 31) \ - | (uint64_t) low_bits); \ - if (sign > 0) { \ - CONV_UNSIGNED_NUMBER(number); \ - } else { \ - CONV_NUMBER(-number); \ - } \ - break; \ - } \ - case kMPFloat: { \ - if (val_di->di_tv.v_type != VAR_FLOAT) { \ - goto name##_convert_one_value_regular_dict; \ - } \ - CONV_FLOAT(val_di->di_tv.vval.v_float); \ - break; \ - } \ - case kMPString: \ - case kMPBinary: { \ - const bool is_string = ((MessagePackType) i == kMPString); \ - if (val_di->di_tv.v_type != VAR_LIST) { \ - goto name##_convert_one_value_regular_dict; \ - } \ - size_t len; \ - char *buf; \ - if (!vim_list_to_buf(val_di->di_tv.vval.v_list, &len, &buf)) { \ - goto name##_convert_one_value_regular_dict; \ - } \ - if (is_string) { \ - CONV_STR_STRING(buf, len); \ - } else { \ - CONV_STRING(buf, len); \ - } \ - xfree(buf); \ - break; \ - } \ - case kMPArray: { \ - if (val_di->di_tv.v_type != VAR_LIST) { \ - goto name##_convert_one_value_regular_dict; \ - } \ - CHECK_SELF_REFERENCE(val_di->di_tv.vval.v_list, lv_copyID, \ - kMPConvList); \ - CONV_LIST_START(val_di->di_tv.vval.v_list); \ - kv_push(MPConvStackVal, *mpstack, ((MPConvStackVal) { \ - .type = kMPConvList, \ - .data = { \ - .l = { \ - .list = val_di->di_tv.vval.v_list, \ - .li = val_di->di_tv.vval.v_list->lv_first, \ - }, \ - }, \ - })); \ - break; \ - } \ - case kMPMap: { \ - if (val_di->di_tv.v_type != VAR_LIST) { \ - goto name##_convert_one_value_regular_dict; \ - } \ - if (val_di->di_tv.vval.v_list == NULL) { \ - CONV_EMPTY_DICT(); \ - break; \ - } \ - list_T *const val_list = val_di->di_tv.vval.v_list; \ - for (const listitem_T *li = val_list->lv_first; li != NULL; \ - li = li->li_next) { \ - if (li->li_tv.v_type != VAR_LIST \ - || li->li_tv.vval.v_list->lv_len != 2) { \ - goto name##_convert_one_value_regular_dict; \ - } \ - } \ - CHECK_SELF_REFERENCE(val_list, lv_copyID, kMPConvPairs); \ - CONV_SPECIAL_MAP_START(val_list); \ - kv_push(MPConvStackVal, *mpstack, ((MPConvStackVal) { \ - .type = kMPConvPairs, \ - .data = { \ - .l = { \ - .list = val_list, \ - .li = val_list->lv_first, \ - }, \ - }, \ - })); \ - break; \ - } \ - case kMPExt: { \ - const list_T *val_list; \ - varnumber_T type; \ - if (val_di->di_tv.v_type != VAR_LIST \ - || (val_list = val_di->di_tv.vval.v_list) == NULL \ - || val_list->lv_len != 2 \ - || (val_list->lv_first->li_tv.v_type != VAR_NUMBER) \ - || (type = val_list->lv_first->li_tv.vval.v_number) > INT8_MAX \ - || type < INT8_MIN \ - || (val_list->lv_last->li_tv.v_type != VAR_LIST)) { \ - goto name##_convert_one_value_regular_dict; \ - } \ - size_t len; \ - char *buf; \ - if (!vim_list_to_buf(val_list->lv_last->li_tv.vval.v_list, \ - &len, &buf)) { \ - goto name##_convert_one_value_regular_dict; \ - } \ - CONV_EXT_STRING(buf, len, type); \ - xfree(buf); \ - break; \ - } \ - } \ - break; \ - } \ -name##_convert_one_value_regular_dict: \ - CHECK_SELF_REFERENCE(tv->vval.v_dict, dv_copyID, kMPConvDict); \ - CONV_DICT_START(tv->vval.v_dict); \ - kv_push(MPConvStackVal, *mpstack, ((MPConvStackVal) { \ - .type = kMPConvDict, \ - .data = { \ - .d = { \ - .dict = tv->vval.v_dict, \ - .hi = tv->vval.v_dict->dv_hashtab.ht_array, \ - .todo = tv->vval.v_dict->dv_hashtab.ht_used, \ - }, \ - }, \ - })); \ - break; \ - } \ - default: { \ - EMSG2(_(e_intern2), #name "_convert_one_value()"); \ - return FAIL; \ - } \ - } \ - return OK; \ -} \ -\ -scope int vim_to_##name(firstargtype firstargname, typval_T *const tv, \ - const char *const objname) \ - FUNC_ATTR_WARN_UNUSED_RESULT \ -{ \ - current_copyID += COPYID_INC; \ - const int copyID = current_copyID; \ - MPConvStack mpstack; \ - kv_init(mpstack); \ - if (name##_convert_one_value(firstargname, &mpstack, tv, copyID, objname) \ - == FAIL) { \ - goto vim_to_msgpack_error_ret; \ - } \ - while (kv_size(mpstack)) { \ - MPConvStackVal *cur_mpsv = &kv_A(mpstack, kv_size(mpstack) - 1); \ - typval_T *cur_tv = NULL; \ - switch (cur_mpsv->type) { \ - case kMPConvDict: { \ - if (!cur_mpsv->data.d.todo) { \ - (void) kv_pop(mpstack); \ - cur_mpsv->data.d.dict->dv_copyID = copyID - 1; \ - CONV_DICT_END(cur_mpsv->data.d.dict); \ - continue; \ - } else if (cur_mpsv->data.d.todo \ - != cur_mpsv->data.d.dict->dv_hashtab.ht_used) { \ - CONV_DICT_BETWEEN_ITEMS(cur_mpsv->data.d.dict); \ - } \ - while (HASHITEM_EMPTY(cur_mpsv->data.d.hi)) { \ - cur_mpsv->data.d.hi++; \ - } \ - dictitem_T *const di = HI2DI(cur_mpsv->data.d.hi); \ - cur_mpsv->data.d.todo--; \ - cur_mpsv->data.d.hi++; \ - CONV_STR_STRING(&di->di_key[0], STRLEN(&di->di_key[0])); \ - CONV_DICT_AFTER_KEY(cur_mpsv->data.d.dict); \ - cur_tv = &di->di_tv; \ - break; \ - } \ - case kMPConvList: { \ - if (cur_mpsv->data.l.li == NULL) { \ - (void) kv_pop(mpstack); \ - cur_mpsv->data.l.list->lv_copyID = copyID - 1; \ - CONV_LIST_END(cur_mpsv->data.l.list); \ - continue; \ - } else if (cur_mpsv->data.l.li != cur_mpsv->data.l.list->lv_first) { \ - CONV_LIST_BETWEEN_ITEMS(cur_mpsv->data.l.list); \ - } \ - cur_tv = &cur_mpsv->data.l.li->li_tv; \ - cur_mpsv->data.l.li = cur_mpsv->data.l.li->li_next; \ - break; \ - } \ - case kMPConvPairs: { \ - if (cur_mpsv->data.l.li == NULL) { \ - (void) kv_pop(mpstack); \ - cur_mpsv->data.l.list->lv_copyID = copyID - 1; \ - continue; \ - } \ - const list_T *const kv_pair = cur_mpsv->data.l.li->li_tv.vval.v_list; \ - if (name##_convert_one_value(firstargname, &mpstack, \ - &kv_pair->lv_first->li_tv, copyID, \ - objname) == FAIL) { \ - goto vim_to_msgpack_error_ret; \ - } \ - cur_tv = &kv_pair->lv_last->li_tv; \ - cur_mpsv->data.l.li = cur_mpsv->data.l.li->li_next; \ - break; \ - } \ - } \ - if (name##_convert_one_value(firstargname, &mpstack, cur_tv, copyID, \ - objname) == FAIL) { \ - goto vim_to_msgpack_error_ret; \ - } \ - } \ - kv_destroy(mpstack); \ - return OK; \ -vim_to_msgpack_error_ret: \ - kv_destroy(mpstack); \ - return FAIL; \ -} - -#define CONV_STRING(buf, len) \ - do { \ - const char *const buf_ = (const char *) buf; \ - if (buf == NULL) { \ - ga_concat(gap, (char_u *) "''"); \ - } else { \ - const size_t len_ = (len); \ - size_t num_quotes = 0; \ - for (size_t i = 0; i < len_; i++) { \ - if (buf_[i] == '\'') { \ - num_quotes++; \ - } \ - } \ - ga_grow(gap, 2 + len_ + num_quotes); \ - ga_append(gap, '\''); \ - for (size_t i = 0; i < len_; i++) { \ - if (buf_[i] == '\'') { \ - num_quotes++; \ - ga_append(gap, '\''); \ - } \ - ga_append(gap, buf_[i]); \ - } \ - ga_append(gap, '\''); \ - } \ - } while (0) - -#define CONV_STR_STRING(buf, len) \ - CONV_STRING(buf, len) - -#define CONV_EXT_STRING(buf, len, type) - -#define CONV_NUMBER(num) \ - do { \ - char numbuf[NUMBUFLEN]; \ - vim_snprintf(numbuf, NUMBUFLEN - 1, "%" PRId64, (int64_t) (num)); \ - ga_concat(gap, (char_u *) numbuf); \ - } while (0) - -#define CONV_FLOAT(flt) \ - do { \ - const float_T flt_ = (flt); \ - switch (fpclassify(flt_)) { \ - case FP_NAN: { \ - ga_concat(gap, (char_u *) "str2float('nan')"); \ - break; \ - } \ - case FP_INFINITE: { \ - if (flt_ < 0) { \ - ga_append(gap, '-'); \ - } \ - ga_concat(gap, (char_u *) "str2float('inf')"); \ - break; \ - } \ - default: { \ - char numbuf[NUMBUFLEN]; \ - vim_snprintf(numbuf, NUMBUFLEN - 1, "%g", flt_); \ - ga_concat(gap, (char_u *) numbuf); \ - } \ - } \ - } while (0) - -#define CONV_FUNC(fun) \ - do { \ - ga_concat(gap, (char_u *) "function("); \ - CONV_STRING(fun, STRLEN(fun)); \ - ga_append(gap, ')'); \ - } while (0) - -#define CONV_EMPTY_LIST() \ - ga_concat(gap, (char_u *) "[]") - -#define CONV_LIST_START(lst) \ - ga_append(gap, '[') - -#define CONV_EMPTY_DICT() \ - ga_concat(gap, (char_u *) "{}") - -#define CONV_SPECIAL_NIL() - -#define CONV_SPECIAL_BOOL(num) - -#define CONV_UNSIGNED_NUMBER(num) - -#define CONV_SPECIAL_MAP_START(lst) - -#define CONV_DICT_START(dct) \ - ga_append(gap, '{') - -#define CONV_DICT_END(dct) \ - ga_append(gap, '}') - -#define CONV_DICT_AFTER_KEY(dct) \ - ga_concat(gap, (char_u *) ": ") - -#define CONV_DICT_BETWEEN_ITEMS(dct) \ - ga_concat(gap, (char_u *) ", ") - -#define CONV_LIST_END(lst) \ - ga_append(gap, ']') - -#define CONV_LIST_BETWEEN_ITEMS(lst) \ - CONV_DICT_BETWEEN_ITEMS(NULL) - -#define CONV_RECURSE(val, conv_type) \ - do { \ - if (!did_echo_string_emsg) { \ - /* Only give this message once for a recursive call to avoid */ \ - /* flooding the user with errors. */ \ - did_echo_string_emsg = true; \ - EMSG(_("E724: unable to correctly dump variable " \ - "with self-referencing container")); \ - } \ - char ebuf[NUMBUFLEN + 7]; \ - size_t backref = 0; \ - for (; backref < kv_size(*mpstack); backref++) { \ - const MPConvStackVal mpval = kv_a(MPConvStackVal, *mpstack, backref); \ - if (mpval.type == conv_type) { \ - if (conv_type == kMPConvDict) { \ - if ((void *) mpval.data.d.dict == (void *) (val)) { \ - break; \ - } \ - } else if (conv_type == kMPConvList) { \ - if ((void *) mpval.data.l.list == (void *) (val)) { \ - break; \ - } \ - } \ - } \ - } \ - vim_snprintf(ebuf, NUMBUFLEN + 6, "{E724@%zu}", backref); \ - ga_concat(gap, (char_u *) &ebuf[0]); \ - return OK; \ - } while (0) - -#define CONV_ALLOW_SPECIAL false - -DEFINE_VIML_CONV_FUNCTIONS(static, string, garray_T *const, gap) - -#undef CONV_RECURSE -#define CONV_RECURSE(val, conv_type) \ - do { \ - char ebuf[NUMBUFLEN + 7]; \ - size_t backref = 0; \ - for (; backref < kv_size(*mpstack); backref++) { \ - const MPConvStackVal mpval = kv_a(MPConvStackVal, *mpstack, backref); \ - if (mpval.type == conv_type) { \ - if (conv_type == kMPConvDict) { \ - if ((void *) mpval.data.d.dict == (void *) val) { \ - break; \ - } \ - } else if (conv_type == kMPConvList) { \ - if ((void *) mpval.data.l.list == (void *) val) { \ - break; \ - } \ - } \ - } \ - } \ - if (conv_type == kMPConvDict) { \ - vim_snprintf(ebuf, NUMBUFLEN + 6, "{...@%zu}", backref); \ - } else { \ - vim_snprintf(ebuf, NUMBUFLEN + 6, "[...@%zu]", backref); \ - } \ - ga_concat(gap, (char_u *) &ebuf[0]); \ - return OK; \ - } while (0) - -DEFINE_VIML_CONV_FUNCTIONS(static, echo, garray_T *const, gap) - -#undef CONV_STRING -#undef CONV_STR_STRING -#undef CONV_EXT_STRING -#undef CONV_NUMBER -#undef CONV_FLOAT -#undef CONV_FUNC -#undef CONV_EMPTY_LIST -#undef CONV_LIST_START -#undef CONV_EMPTY_DICT -#undef CONV_SPECIAL_NIL -#undef CONV_SPECIAL_BOOL -#undef CONV_UNSIGNED_NUMBER -#undef CONV_SPECIAL_MAP_START -#undef CONV_DICT_START -#undef CONV_DICT_END -#undef CONV_DICT_AFTER_KEY -#undef CONV_DICT_BETWEEN_ITEMS -#undef CONV_LIST_END -#undef CONV_LIST_BETWEEN_ITEMS -#undef CONV_RECURSE -#undef CONV_ALLOW_SPECIAL - -/// Return a string with the string representation of a variable. -/// Puts quotes around strings, so that they can be parsed back by eval(). -/// -/// @param[in] tv typval_T to convert. -/// @param[out] len Location where length of the result will be saved. -/// -/// @return String representation of the variable or NULL. -static char *tv2string(typval_T *tv, size_t *len) - FUNC_ATTR_NONNULL_ARG(1) FUNC_ATTR_MALLOC -{ - garray_T ga; - ga_init(&ga, (int)sizeof(char), 80); - vim_to_string(&ga, tv, "tv2string() argument"); - did_echo_string_emsg = false; - if (len != NULL) { - *len = (size_t) ga.ga_len; - } - ga_append(&ga, '\0'); - return (char *) ga.ga_data; -} - -/// Return a string with the string representation of a variable. -/// Does not put quotes around strings, as ":echo" displays values. -/// -/// @param[in] tv typval_T to convert. -/// @param[out] len Location where length of the result will be saved. -/// -/// @return String representation of the variable or NULL. -static char *echo_string(typval_T *tv, size_t *len) - FUNC_ATTR_NONNULL_ARG(1) FUNC_ATTR_MALLOC -{ - garray_T ga; - ga_init(&ga, (int)sizeof(char), 80); - if (tv->v_type == VAR_STRING || tv->v_type == VAR_FUNC) { - if (tv->vval.v_string != NULL) { - ga_concat(&ga, tv->vval.v_string); - } - } else { - vim_to_echo(&ga, tv, ":echo argument"); - did_echo_string_emsg = false; - } - if (len != NULL) { - *len = (size_t) ga.ga_len; - } - ga_append(&ga, '\0'); - return (char *) ga.ga_data; -} - /* * Convert the string "text" to a floating point number. * This uses strtod(). setlocale(LC_NUMERIC, "C") has been used to make sure @@ -8102,19 +7478,19 @@ static void fill_assert_error(garray_T *gap, typval_T *opt_msg_tv, char_u *tofree; if (opt_msg_tv->v_type != VAR_UNKNOWN) { - tofree = (char_u *) tv2string(opt_msg_tv, NULL); + tofree = (char_u *) encode_tv2string(opt_msg_tv, NULL); ga_concat(gap, tofree); xfree(tofree); } else { ga_concat(gap, (char_u *)"Expected "); if (exp_str == NULL) { - tofree = (char_u *) tv2string(exp_tv, NULL); + tofree = (char_u *) encode_tv2string(exp_tv, NULL); ga_concat(gap, tofree); xfree(tofree); } else { ga_concat(gap, exp_str); } - tofree = (char_u *) tv2string(got_tv, NULL); + tofree = (char_u *) encode_tv2string(got_tv, NULL); ga_concat(gap, (char_u *)" but got "); ga_concat(gap, tofree); xfree(tofree); @@ -12482,7 +11858,7 @@ static void find_some_match(typval_T *argvars, typval_T *rettv, int type) break; } xfree(tofree); - tofree = str = (char_u *) echo_string(&li->li_tv, NULL); + tofree = str = (char_u *) encode_tv2echo(&li->li_tv, NULL); if (str == NULL) break; } @@ -12934,237 +12310,6 @@ static int msgpack_list_write(void *data, const char *buf, size_t len) return 0; } -/// Convert readfile()-style list to a char * buffer with length -/// -/// @param[in] list Converted list. -/// @param[out] ret_len Resulting buffer length. -/// @param[out] ret_buf Allocated buffer with the result or NULL if ret_len is -/// zero. -/// -/// @return true in case of success, false in case of failure. -static inline bool vim_list_to_buf(const list_T *const list, - size_t *const ret_len, char **const ret_buf) - FUNC_ATTR_NONNULL_ARG(2,3) FUNC_ATTR_WARN_UNUSED_RESULT -{ - size_t len = 0; - if (list != NULL) { - for (const listitem_T *li = list->lv_first; - li != NULL; - li = li->li_next) { - if (li->li_tv.v_type != VAR_STRING) { - return false; - } - len++; - if (li->li_tv.vval.v_string != 0) { - len += STRLEN(li->li_tv.vval.v_string); - } - } - if (len) { - len--; - } - } - *ret_len = len; - if (len == 0) { - *ret_buf = NULL; - return true; - } - ListReaderState lrstate = init_lrstate(list); - char *const buf = xmalloc(len); - size_t read_bytes; - if (read_from_list(&lrstate, buf, len, &read_bytes) != OK) { - assert(false); - } - assert(len == read_bytes); - *ret_buf = buf; - return true; -} - -/// Show a error message when converting to msgpack value -/// -/// @param[in] msg Error message to dump. Must contain exactly two %s that -/// will be replaced with what was being dumped: first with -/// something like “F” or “function argument”, second with path -/// to the failed value. -/// @param[in] mpstack Path to the failed value. -/// @param[in] objname Dumped object name. -/// -/// @return FAIL. -static int conv_error(const char *const msg, const MPConvStack *const mpstack, - const char *const objname) - FUNC_ATTR_NONNULL_ALL -{ - garray_T msg_ga; - ga_init(&msg_ga, (int)sizeof(char), 80); - char *const key_msg = _("key %s"); - char *const key_pair_msg = _("key %s at index %i from special map"); - char *const idx_msg = _("index %i"); - for (size_t i = 0; i < kv_size(*mpstack); i++) { - if (i != 0) { - ga_concat(&msg_ga, (char_u *) ", "); - } - MPConvStackVal v = kv_A(*mpstack, i); - switch (v.type) { - case kMPConvDict: { - typval_T key_tv = { - .v_type = VAR_STRING, - .vval = { .v_string = (v.data.d.hi == NULL - ? v.data.d.dict->dv_hashtab.ht_array - : (v.data.d.hi - 1))->hi_key }, - }; - char *const key = tv2string(&key_tv, NULL); - vim_snprintf((char *) IObuff, IOSIZE, key_msg, key); - xfree(key); - ga_concat(&msg_ga, IObuff); - break; - } - case kMPConvPairs: - case kMPConvList: { - int idx = 0; - const listitem_T *li; - for (li = v.data.l.list->lv_first; - li != NULL && li->li_next != v.data.l.li; - li = li->li_next) { - idx++; - } - if (v.type == kMPConvList - || li == NULL - || (li->li_tv.v_type != VAR_LIST - && li->li_tv.vval.v_list->lv_len <= 0)) { - vim_snprintf((char *) IObuff, IOSIZE, idx_msg, idx); - ga_concat(&msg_ga, IObuff); - } else { - typval_T key_tv = li->li_tv.vval.v_list->lv_first->li_tv; - char *const key = echo_string(&key_tv, NULL); - vim_snprintf((char *) IObuff, IOSIZE, key_pair_msg, key, idx); - xfree(key); - ga_concat(&msg_ga, IObuff); - } - break; - } - } - } - EMSG3(msg, objname, (kv_size(*mpstack) == 0 - ? _("itself") - : (char *) msg_ga.ga_data)); - ga_clear(&msg_ga); - return FAIL; -} - -#define CONV_STRING(buf, len) \ - do { \ - if (buf == NULL) { \ - msgpack_pack_bin(packer, 0); \ - } else { \ - const size_t len_ = (len); \ - msgpack_pack_bin(packer, len_); \ - msgpack_pack_bin_body(packer, buf, len_); \ - } \ - } while (0) - -#define CONV_STR_STRING(buf, len) \ - do { \ - if (buf == NULL) { \ - msgpack_pack_str(packer, 0); \ - } else { \ - const size_t len_ = (len); \ - msgpack_pack_str(packer, len_); \ - msgpack_pack_str_body(packer, buf, len_); \ - } \ - } while (0) - -#define CONV_EXT_STRING(buf, len, type) \ - do { \ - if (buf == NULL) { \ - msgpack_pack_ext(packer, 0, type); \ - } else { \ - const size_t len_ = (len); \ - msgpack_pack_ext(packer, len_, (int8_t) type); \ - msgpack_pack_ext_body(packer, buf, len_); \ - } \ - } while (0) - -#define CONV_NUMBER(num) \ - msgpack_pack_int64(packer, (int64_t) (num)) - -#define CONV_FLOAT(flt) \ - msgpack_pack_double(packer, (double) (flt)) - -#define CONV_FUNC(fun) \ - return conv_error(_("E951: Error while dumping %s, %s: " \ - "attempt to dump function reference"), \ - mpstack, objname) - -#define CONV_EMPTY_LIST() \ - msgpack_pack_array(packer, 0) - -#define CONV_LIST_START(lst) \ - msgpack_pack_array(packer, (lst)->lv_len) - -#define CONV_EMPTY_DICT() \ - msgpack_pack_map(packer, 0) - -#define CONV_SPECIAL_NIL() \ - msgpack_pack_nil(packer) - -#define CONV_SPECIAL_BOOL(num) \ - do { \ - if ((num)) { \ - msgpack_pack_true(packer); \ - } else { \ - msgpack_pack_false(packer); \ - } \ - } while (0) - -#define CONV_UNSIGNED_NUMBER(num) \ - msgpack_pack_uint64(packer, (num)) - -#define CONV_SPECIAL_MAP_START(lst) \ - msgpack_pack_map(packer, (lst)->lv_len) - -#define CONV_DICT_START(dct) \ - msgpack_pack_map(packer, (dct)->dv_hashtab.ht_used) - -#define CONV_DICT_END(dct) - -#define CONV_DICT_AFTER_KEY(dct) - -#define CONV_DICT_BETWEEN_ITEMS(dct) - -#define CONV_LIST_END(lst) - -#define CONV_LIST_BETWEEN_ITEMS(lst) - -#define CONV_RECURSE(val, conv_type) \ - return conv_error(_("E952: Unable to dump %s: " \ - "container references itself in %s"), \ - mpstack, objname) - -#define CONV_ALLOW_SPECIAL true - -DEFINE_VIML_CONV_FUNCTIONS(, msgpack, msgpack_packer *const, packer) - -#undef CONV_STRING -#undef CONV_STR_STRING -#undef CONV_EXT_STRING -#undef CONV_NUMBER -#undef CONV_FLOAT -#undef CONV_FUNC -#undef CONV_EMPTY_LIST -#undef CONV_LIST_START -#undef CONV_EMPTY_DICT -#undef CONV_SPECIAL_NIL -#undef CONV_SPECIAL_BOOL -#undef CONV_UNSIGNED_NUMBER -#undef CONV_SPECIAL_MAP_START -#undef CONV_DICT_START -#undef CONV_DICT_END -#undef CONV_DICT_AFTER_KEY -#undef CONV_DICT_BETWEEN_ITEMS -#undef CONV_LIST_END -#undef CONV_LIST_BETWEEN_ITEMS -#undef CONV_RECURSE -#undef CONV_ALLOW_SPECIAL - /// "msgpackdump()" function static void f_msgpackdump(typval_T *argvars, typval_T *rettv) FUNC_ATTR_NONNULL_ALL @@ -13186,72 +12331,13 @@ static void f_msgpackdump(typval_T *argvars, typval_T *rettv) for (listitem_T *li = list->lv_first; li != NULL; li = li->li_next) { vim_snprintf(msgbuf, sizeof(msgbuf), (char *) msg, idx); idx++; - if (vim_to_msgpack(lpacker, &li->li_tv, msgbuf) == FAIL) { + if (encode_vim_to_msgpack(lpacker, &li->li_tv, msgbuf) == FAIL) { break; } } msgpack_packer_free(lpacker); } -/// Read bytes from list -/// -/// @param[in,out] state Structure describing position in list from which -/// reading should start. Is updated to reflect position -/// at which reading ended. -/// @param[out] buf Buffer to write to. -/// @param[in] nbuf Buffer length. -/// @param[out] read_bytes Is set to amount of bytes read. -/// -/// @return OK when reading was finished, FAIL in case of error (i.e. list item -/// was not a string), NOTDONE if reading was successfull, but there are -/// more bytes to read. -static int read_from_list(ListReaderState *const state, char *const buf, - const size_t nbuf, size_t *const read_bytes) - FUNC_ATTR_NONNULL_ALL FUNC_ATTR_WARN_UNUSED_RESULT -{ - char *const buf_end = buf + nbuf; - char *p = buf; - while (p < buf_end) { - for (size_t i = state->offset; i < state->li_length && p < buf_end; i++) { - const char ch = state->li->li_tv.vval.v_string[state->offset++]; - *p++ = (ch == NL ? NUL : ch); - } - if (p < buf_end) { - state->li = state->li->li_next; - if (state->li == NULL) { - *read_bytes = (size_t) (p - buf); - return OK; - } - *p++ = NL; - if (state->li->li_tv.v_type != VAR_STRING) { - *read_bytes = (size_t) (p - buf); - return FAIL; - } - state->offset = 0; - state->li_length = (state->li->li_tv.vval.v_string == NULL - ? 0 - : STRLEN(state->li->li_tv.vval.v_string)); - } - } - *read_bytes = nbuf; - return (state->offset < state->li_length || state->li->li_next != NULL - ? NOTDONE - : OK); -} - -/// Initialize ListReaderState structure -static inline ListReaderState init_lrstate(const list_T *const list) - FUNC_ATTR_NONNULL_ALL -{ - return (ListReaderState) { - .li = list->lv_first, - .offset = 0, - .li_length = (list->lv_first->li_tv.vval.v_string == NULL - ? 0 - : STRLEN(list->lv_first->li_tv.vval.v_string)), - }; -} - /// Convert msgpack object to a VimL one int msgpack_to_vim(const msgpack_object mobj, typval_T *const rettv) FUNC_ATTR_NONNULL_ALL FUNC_ATTR_WARN_UNUSED_RESULT @@ -13262,7 +12348,7 @@ int msgpack_to_vim(const msgpack_object mobj, typval_T *const rettv) dictitem_T *const type_di = dictitem_alloc((char_u *) "_TYPE"); \ type_di->di_tv.v_type = VAR_LIST; \ type_di->di_tv.v_lock = 0; \ - type_di->di_tv.vval.v_list = (list_T *) msgpack_type_lists[type]; \ + type_di->di_tv.vval.v_list = (list_T *) eval_msgpack_type_lists[type]; \ type_di->di_tv.vval.v_list->lv_refcount++; \ dict_add(dict, type_di); \ dictitem_T *const val_di = dictitem_alloc((char_u *) "_VAL"); \ @@ -13504,7 +12590,7 @@ static void f_msgpackparse(typval_T *argvars, typval_T *rettv) EMSG2(_(e_invarg2), "List item is not a string"); return; } - ListReaderState lrstate = init_lrstate(list); + ListReaderState lrstate = encode_init_lrstate(list); msgpack_unpacker *const unpacker = msgpack_unpacker_new(IOSIZE); if (unpacker == NULL) { EMSG(_(e_outofmem)); @@ -13518,7 +12604,7 @@ static void f_msgpackparse(typval_T *argvars, typval_T *rettv) goto f_msgpackparse_exit; } size_t read_bytes; - const int rlret = read_from_list( + const int rlret = encode_read_from_list( &lrstate, msgpack_unpacker_buffer(unpacker), IOSIZE, &read_bytes); if (rlret == FAIL) { EMSG2(_(e_invarg2), "List item is not a string"); @@ -15779,9 +14865,9 @@ static int item_compare(const void *s1, const void *s2, bool keep_zero) si2 = (sortItem_T *)s2; typval_T *tv1 = &si1->item->li_tv; typval_T *tv2 = &si2->item->li_tv; - // tv2string() puts quotes around a string and allocates memory. Don't do - // that for string variables. Use a single quote when comparing with a - // non-string to do what the docs promise. + // encode_tv2string() puts quotes around a string and allocates memory. Don't + // do that for string variables. Use a single quote when comparing with + // a non-string to do what the docs promise. if (tv1->v_type == VAR_STRING) { if (tv2->v_type != VAR_STRING || item_compare_numeric) { p1 = (char_u *)"'"; @@ -15789,7 +14875,7 @@ static int item_compare(const void *s1, const void *s2, bool keep_zero) p1 = tv1->vval.v_string; } } else { - tofree1 = p1 = (char_u *) tv2string(tv1, NULL); + tofree1 = p1 = (char_u *) encode_tv2string(tv1, NULL); } if (tv2->v_type == VAR_STRING) { if (tv1->v_type != VAR_STRING || item_compare_numeric) { @@ -15798,7 +14884,7 @@ static int item_compare(const void *s1, const void *s2, bool keep_zero) p2 = tv2->vval.v_string; } } else { - tofree2 = p2 = (char_u *) tv2string(tv2, NULL); + tofree2 = p2 = (char_u *) encode_tv2string(tv2, NULL); } if (p1 == NULL) p1 = (char_u *)""; @@ -16365,7 +15451,7 @@ static void f_stridx(typval_T *argvars, typval_T *rettv) static void f_string(typval_T *argvars, typval_T *rettv) { rettv->v_type = VAR_STRING; - rettv->vval.v_string = (char_u *) tv2string(&argvars[0], NULL); + rettv->vval.v_string = (char_u *) encode_tv2string(&argvars[0], NULL); } /* @@ -18970,7 +18056,7 @@ static void delete_var(hashtab_T *ht, hashitem_T *hi) */ static void list_one_var(dictitem_T *v, char_u *prefix, int *first) { - char_u *s = (char_u *) echo_string(&v->di_tv, NULL); + char_u *s = (char_u *) encode_tv2echo(&v->di_tv, NULL); list_one_var_a(prefix, v->di_key, v->di_tv.v_type, s == NULL ? (char_u *)"" : s, first); xfree(s); @@ -19424,7 +18510,7 @@ void ex_echo(exarg_T *eap) } } else if (eap->cmdidx == CMD_echo) msg_puts_attr((char_u *)" ", echo_attr); - char_u *tofree = p = (char_u *) echo_string(&rettv, NULL); + char_u *tofree = p = (char_u *) encode_tv2echo(&rettv, NULL); if (p != NULL) { for (; *p != NUL && !got_int; ++p) { if (*p == '\n' || *p == '\r' || *p == TAB) { @@ -21077,7 +20163,7 @@ call_user_func ( } else { // Do not want errors such as E724 here. ++emsg_off; - char_u *s = (char_u *) tv2string(&argvars[i], NULL); + char_u *s = (char_u *) encode_tv2string(&argvars[i], NULL); char_u *tofree = s; --emsg_off; if (s != NULL) { @@ -21171,7 +20257,7 @@ call_user_func ( // have some idea how it starts and ends. smsg() would always // truncate it at the end. Don't want errors such as E724 here. ++emsg_off; - char_u *s = (char_u *) tv2string(fc->rettv, NULL); + char_u *s = (char_u *) encode_tv2string(fc->rettv, NULL); char_u *tofree = s; --emsg_off; if (s != NULL) { @@ -21444,7 +20530,7 @@ char_u *get_return_cmd(void *rettv) char_u *tofree = NULL; if (rettv != NULL) { - tofree = s = (char_u *) echo_string((typval_T *) rettv, NULL); + tofree = s = (char_u *) encode_tv2echo((typval_T *) rettv, NULL); } if (s == NULL) { s = (char_u *)""; diff --git a/src/nvim/eval.h b/src/nvim/eval.h index f51b0f4921..e8b5964775 100644 --- a/src/nvim/eval.h +++ b/src/nvim/eval.h @@ -4,9 +4,22 @@ #include #include "nvim/profile.h" +#include "nvim/hashtab.h" // For hashtab_T +#include "nvim/garray.h" // For garray_T +#include "nvim/buffer_defs.h" // For scid_T +#include "nvim/ex_cmds_defs.h" // For exarg_T + +#define COPYID_INC 2 +#define COPYID_MASK (~0x1) // All user-defined functions are found in this hashtable. -EXTERN hashtab_T func_hashtab; +extern hashtab_T func_hashtab; + +/// CopyID for recursively traversing lists and dicts +/// +/// This value is needed to avoid endless recursiveness. Last bit is used for +/// previous_funccal and normally ignored when comparing. +extern int current_copyID; // Structure to hold info for a user function. typedef struct ufunc ufunc_T; @@ -117,12 +130,28 @@ enum { VV_LEN, // number of v: vars }; +/// All recognized msgpack types +typedef enum { + kMPNil, + kMPBoolean, + kMPInteger, + kMPFloat, + kMPString, + kMPBinary, + kMPArray, + kMPMap, + kMPExt, +#define LAST_MSGPACK_TYPE kMPExt +} MessagePackType; + +/// Array mapping values from MessagePackType to corresponding list pointers +extern const list_T *eval_msgpack_type_lists[LAST_MSGPACK_TYPE + 1]; + +#undef LAST_MSGPACK_TYPE + /// Maximum number of function arguments #define MAX_FUNC_ARGS 20 -int vim_to_msgpack(msgpack_packer *const, typval_T *const, - const char *const objname); - #ifdef INCLUDE_GENERATED_DECLARATIONS # include "eval.h.generated.h" #endif diff --git a/src/nvim/globals.h b/src/nvim/globals.h index 40b5718071..49d1de21d9 100644 --- a/src/nvim/globals.h +++ b/src/nvim/globals.h @@ -4,17 +4,7 @@ #include #include -// EXTERN is only defined in main.c. That's where global variables are -// actually defined and initialized. -#ifndef EXTERN -# define EXTERN extern -# define INIT(...) -#else -# ifndef INIT -# define INIT(...) __VA_ARGS__ -# endif -#endif - +#include "nvim/macros.h" #include "nvim/ex_eval.h" #include "nvim/iconv.h" #include "nvim/mbyte.h" diff --git a/src/nvim/macros.h b/src/nvim/macros.h index 26ab5a7de7..5f69fa2f6a 100644 --- a/src/nvim/macros.h +++ b/src/nvim/macros.h @@ -1,6 +1,17 @@ #ifndef NVIM_MACROS_H #define NVIM_MACROS_H +// EXTERN is only defined in main.c. That's where global variables are +// actually defined and initialized. +#ifndef EXTERN +# define EXTERN extern +# define INIT(...) +#else +# ifndef INIT +# define INIT(...) __VA_ARGS__ +# endif +#endif + #ifndef MIN # define MIN(X, Y) ((X) < (Y) ? (X) : (Y)) #endif diff --git a/src/nvim/option_defs.h b/src/nvim/option_defs.h index 8e74e5036e..bbe40873ec 100644 --- a/src/nvim/option_defs.h +++ b/src/nvim/option_defs.h @@ -4,6 +4,7 @@ #include #include "nvim/types.h" +#include "nvim/macros.h" // For EXTERN // option_defs.h: definition of global variables for settable options diff --git a/src/nvim/shada.c b/src/nvim/shada.c index def2de9b1a..22767dbe35 100644 --- a/src/nvim/shada.c +++ b/src/nvim/shada.c @@ -39,6 +39,7 @@ #include "nvim/fileio.h" #include "nvim/strings.h" #include "nvim/quickfix.h" +#include "nvim/encode.h" #include "nvim/lib/khash.h" #include "nvim/lib/kvec.h" @@ -1687,8 +1688,9 @@ static ShaDaWriteResult shada_pack_entry(msgpack_packer *const packer, do { \ if ((src) != NULL) { \ for (listitem_T *li = (src)->lv_first; li != NULL; li = li->li_next) { \ - if (vim_to_msgpack(spacker, &li->li_tv, \ - _("additional elements of ShaDa " what)) == FAIL) { \ + if (encode_vim_to_msgpack(spacker, &li->li_tv, \ + _("additional elements of ShaDa " what)) \ + == FAIL) { \ goto shada_pack_entry_error; \ } \ } \ @@ -1706,8 +1708,9 @@ static ShaDaWriteResult shada_pack_entry(msgpack_packer *const packer, const size_t key_len = strlen((const char *) hi->hi_key); \ msgpack_pack_str(spacker, key_len); \ msgpack_pack_str_body(spacker, (const char *) hi->hi_key, key_len); \ - if (vim_to_msgpack(spacker, &di->di_tv, \ - _("additional data of ShaDa " what)) == FAIL) { \ + if (encode_vim_to_msgpack(spacker, &di->di_tv, \ + _("additional data of ShaDa " what)) \ + == FAIL) { \ goto shada_pack_entry_error; \ } \ } \ @@ -1757,7 +1760,7 @@ static ShaDaWriteResult shada_pack_entry(msgpack_packer *const packer, char vardesc[256] = "variable g:"; memcpy(&vardesc[sizeof("variable g:") - 1], varname.data, varname.size + 1); - if (vim_to_msgpack(spacker, &entry.data.global_var.value, vardesc) + if (encode_vim_to_msgpack(spacker, &entry.data.global_var.value, vardesc) == FAIL) { ret = kSDWriteIgnError; EMSG2(_(WERR "Failed to write variable %s"), From f21cb425fbae47713fd524bfad98fcbc229b1971 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sat, 30 Jan 2016 21:53:55 +0300 Subject: [PATCH 02/82] functests(msgpack): Fix test names --- test/functional/eval/msgpack_functions_spec.lua | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/functional/eval/msgpack_functions_spec.lua b/test/functional/eval/msgpack_functions_spec.lua index 41b0faf76c..0443424c0d 100644 --- a/test/functional/eval/msgpack_functions_spec.lua +++ b/test/functional/eval/msgpack_functions_spec.lua @@ -512,29 +512,29 @@ describe('msgpackdump() function', function() eq({'\129\128\128'}, eval('msgpackdump([todump])')) end) - it('can dump generic mapping with ext', function() + it('can dump special ext mapping', function() execute('let todump = {"_TYPE": v:msgpack_types.ext, "_VAL": [5, ["",""]]}') eq({'\212\005', ''}, eval('msgpackdump([todump])')) end) - it('can dump generic mapping with array', function() + it('can dump special array mapping', function() execute('let todump = {"_TYPE": v:msgpack_types.array, "_VAL": [5, [""]]}') eq({'\146\005\145\196\n'}, eval('msgpackdump([todump])')) end) - it('can dump generic mapping with UINT64_MAX', function() + it('can dump special UINT64_MAX mapping', function() execute('let todump = {"_TYPE": v:msgpack_types.integer}') execute('let todump._VAL = [1, 3, 0x7FFFFFFF, 0x7FFFFFFF]') eq({'\207\255\255\255\255\255\255\255\255'}, eval('msgpackdump([todump])')) end) - it('can dump generic mapping with INT64_MIN', function() + it('can dump special INT64_MIN mapping', function() execute('let todump = {"_TYPE": v:msgpack_types.integer}') execute('let todump._VAL = [-1, 2, 0, 0]') eq({'\211\128\n\n\n\n\n\n\n'}, eval('msgpackdump([todump])')) end) - it('dump and restore generic mapping with floating-point value', function() + it('dump and restore special mapping with floating-point value', function() execute('let todump = {"_TYPE": v:msgpack_types.float, "_VAL": 0.125}') eq({0.125}, eval('msgpackparse(msgpackdump([todump]))')) end) From c3efad5398da01fcdb0fd40822bedff4957c7797 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sat, 30 Jan 2016 21:54:57 +0300 Subject: [PATCH 03/82] functests(msgpack): Fix location of one of the tests --- test/functional/eval/msgpack_functions_spec.lua | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/functional/eval/msgpack_functions_spec.lua b/test/functional/eval/msgpack_functions_spec.lua index 0443424c0d..fc0aad7902 100644 --- a/test/functional/eval/msgpack_functions_spec.lua +++ b/test/functional/eval/msgpack_functions_spec.lua @@ -382,6 +382,11 @@ describe('msgpack*() functions', function() eq({"\n"}, eval('parsed')) eq(1, eval('dumped ==# dumped2')) end) + + it('dump and restore special mapping with floating-point value', function() + execute('let todump = {"_TYPE": v:msgpack_types.float, "_VAL": 0.125}') + eq({0.125}, eval('msgpackparse(msgpackdump([todump]))')) + end) end) describe('msgpackparse() function', function() @@ -534,11 +539,6 @@ describe('msgpackdump() function', function() eq({'\211\128\n\n\n\n\n\n\n'}, eval('msgpackdump([todump])')) end) - it('dump and restore special mapping with floating-point value', function() - execute('let todump = {"_TYPE": v:msgpack_types.float, "_VAL": 0.125}') - eq({0.125}, eval('msgpackparse(msgpackdump([todump]))')) - end) - it('fails to dump a function reference', function() execute('let Todump = function("tr")') eq('Vim(call):E951: Error while dumping msgpackdump() argument, index 0, itself: attempt to dump function reference', From 68e58444b48fc34c9a7c262883750778fbd935d7 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sat, 30 Jan 2016 22:25:21 +0300 Subject: [PATCH 04/82] eval: Add jsonencode() function Ref #3471 --- runtime/doc/eval.txt | 14 + src/nvim/encode.c | 370 +++++++++++++++++-- src/nvim/eval.c | 8 + src/nvim/garray.c | 17 +- test/functional/eval/json_functions_spec.lua | 213 +++++++++++ 5 files changed, 583 insertions(+), 39 deletions(-) create mode 100644 test/functional/eval/json_functions_spec.lua diff --git a/runtime/doc/eval.txt b/runtime/doc/eval.txt index cf1394d799..2d08afff4f 100644 --- a/runtime/doc/eval.txt +++ b/runtime/doc/eval.txt @@ -1932,6 +1932,7 @@ jobstart( {cmd}[, {opts}]) Number Spawns {cmd} as a job jobstop( {job}) Number Stops a job jobwait( {ids}[, {timeout}]) Number Wait for a set of jobs join( {list} [, {sep}]) String join {list} items into one String +jsonencode( {expr}) String Convert {expr} to JSON keys( {dict}) List keys in {dict} len( {expr}) Number the length of {expr} libcall( {lib}, {func}, {arg}) String call {func} in library {lib} with {arg} @@ -4291,6 +4292,19 @@ join({list} [, {sep}]) *join()* converted into a string like with |string()|. The opposite function is |split()|. +jsonencode({expr}) *jsonencode()* + Convert {expr} into a JSON string. Accepts + |msgpack-special-dict| as the input. Will not convert + |Funcref|s, mappings with non-string keys (can be created as + |msgpack-special-dict|), values with self-referencing + containers, strings which contain non-UTF-8 characters, + pseudo-UTF-8 strings which contain codepoints reserved for + surrogate pairs (such strings are not valid UTF-8 strings). + When converting 'encoding' is taken into account, if it is not + "utf-8", then conversion is performed before encoding strings. + Non-printable characters are converted into "\u1234" escapes + or special escapes like "\t", other are dumped as-is. + keys({dict}) *keys()* Return a |List| with all the keys of {dict}. The |List| is in arbitrary order. diff --git a/src/nvim/encode.c b/src/nvim/encode.c index da93ae98ad..84aaed53cb 100644 --- a/src/nvim/encode.c +++ b/src/nvim/encode.c @@ -7,16 +7,29 @@ // TODO(ZyX-I): Move this to src/nvim/viml or src/nvim/eval #include +#include #include "nvim/encode.h" +#include "nvim/buffer_defs.h" // vimconv_T #include "nvim/eval.h" #include "nvim/garray.h" +#include "nvim/mbyte.h" #include "nvim/message.h" +#include "nvim/charset.h" // vim_isprintc() #include "nvim/macros.h" #include "nvim/ascii.h" #include "nvim/vim.h" // For _() #include "nvim/lib/kvec.h" +#define ga_concat(a, b) ga_concat(a, (char_u *)b) +#define utf_ptr2char(b) utf_ptr2char((char_u *)b) +#define utf_ptr2len(b) ((size_t)utf_ptr2len((char_u *)b)) +#define utf_char2len(b) ((size_t)utf_char2len(b)) +#define string_convert(a, b, c) \ + ((char *)string_convert((vimconv_T *)a, (char_u *)b, c)) +#define convert_setup(vcp, from, to) \ + (convert_setup(vcp, (char_u *)from, (char_u *)to)) + /// Structure representing current VimL to messagepack conversion state typedef struct { enum { @@ -120,7 +133,7 @@ static int conv_error(const char *const msg, const MPConvStack *const mpstack, char *const idx_msg = _("index %i"); for (size_t i = 0; i < kv_size(*mpstack); i++) { if (i != 0) { - ga_concat(&msg_ga, (char_u *) ", "); + ga_concat(&msg_ga, ", "); } MPConvStackVal v = kv_A(*mpstack, i); switch (v.type) { @@ -450,11 +463,11 @@ static int name##_convert_one_value(firstargtype firstargname, \ if (val_di->di_tv.v_type != VAR_LIST) { \ goto name##_convert_one_value_regular_dict; \ } \ - if (val_di->di_tv.vval.v_list == NULL) { \ + list_T *const val_list = val_di->di_tv.vval.v_list; \ + if (val_list == NULL || val_list->lv_len == 0) { \ CONV_EMPTY_DICT(); \ break; \ } \ - list_T *const val_list = val_di->di_tv.vval.v_list; \ for (const listitem_T *li = val_list->lv_first; li != NULL; \ li = li->li_next) { \ if (li->li_tv.v_type != VAR_LIST \ @@ -463,7 +476,7 @@ static int name##_convert_one_value(firstargtype firstargname, \ } \ } \ CHECK_SELF_REFERENCE(val_list, lv_copyID, kMPConvPairs); \ - CONV_SPECIAL_MAP_START(val_list); \ + CONV_DICT_START(val_list->lv_len); \ kv_push(MPConvStackVal, *mpstack, ((MPConvStackVal) { \ .type = kMPConvPairs, \ .data = { \ @@ -502,7 +515,7 @@ static int name##_convert_one_value(firstargtype firstargname, \ } \ name##_convert_one_value_regular_dict: \ CHECK_SELF_REFERENCE(tv->vval.v_dict, dv_copyID, kMPConvDict); \ - CONV_DICT_START(tv->vval.v_dict); \ + CONV_DICT_START(tv->vval.v_dict->dv_hashtab.ht_used); \ kv_push(MPConvStackVal, *mpstack, ((MPConvStackVal) { \ .type = kMPConvDict, \ .data = { \ @@ -543,11 +556,11 @@ scope int encode_vim_to_##name(firstargtype firstargname, typval_T *const tv, \ if (!cur_mpsv->data.d.todo) { \ (void) kv_pop(mpstack); \ cur_mpsv->data.d.dict->dv_copyID = copyID - 1; \ - CONV_DICT_END(cur_mpsv->data.d.dict); \ + CONV_DICT_END(); \ continue; \ } else if (cur_mpsv->data.d.todo \ != cur_mpsv->data.d.dict->dv_hashtab.ht_used) { \ - CONV_DICT_BETWEEN_ITEMS(cur_mpsv->data.d.dict); \ + CONV_DICT_BETWEEN_ITEMS(); \ } \ while (HASHITEM_EMPTY(cur_mpsv->data.d.hi)) { \ cur_mpsv->data.d.hi++; \ @@ -556,7 +569,7 @@ scope int encode_vim_to_##name(firstargtype firstargname, typval_T *const tv, \ cur_mpsv->data.d.todo--; \ cur_mpsv->data.d.hi++; \ CONV_STR_STRING(&di->di_key[0], STRLEN(&di->di_key[0])); \ - CONV_DICT_AFTER_KEY(cur_mpsv->data.d.dict); \ + CONV_DICT_AFTER_KEY(); \ cur_tv = &di->di_tv; \ break; \ } \ @@ -567,7 +580,7 @@ scope int encode_vim_to_##name(firstargtype firstargname, typval_T *const tv, \ CONV_LIST_END(cur_mpsv->data.l.list); \ continue; \ } else if (cur_mpsv->data.l.li != cur_mpsv->data.l.list->lv_first) { \ - CONV_LIST_BETWEEN_ITEMS(cur_mpsv->data.l.list); \ + CONV_LIST_BETWEEN_ITEMS(); \ } \ cur_tv = &cur_mpsv->data.l.li->li_tv; \ cur_mpsv->data.l.li = cur_mpsv->data.l.li->li_next; \ @@ -577,14 +590,19 @@ scope int encode_vim_to_##name(firstargtype firstargname, typval_T *const tv, \ if (cur_mpsv->data.l.li == NULL) { \ (void) kv_pop(mpstack); \ cur_mpsv->data.l.list->lv_copyID = copyID - 1; \ + CONV_DICT_END(); \ continue; \ + } else if (cur_mpsv->data.l.li != cur_mpsv->data.l.list->lv_first) { \ + CONV_DICT_BETWEEN_ITEMS(); \ } \ const list_T *const kv_pair = cur_mpsv->data.l.li->li_tv.vval.v_list; \ + CONV_SPECIAL_DICT_KEY_CHECK(kv_pair); \ if (name##_convert_one_value(firstargname, &mpstack, \ &kv_pair->lv_first->li_tv, copyID, \ objname) == FAIL) { \ goto encode_vim_to_##name##_error_ret; \ } \ + CONV_DICT_AFTER_KEY(); \ cur_tv = &kv_pair->lv_last->li_tv; \ cur_mpsv->data.l.li = cur_mpsv->data.l.li->li_next; \ break; \ @@ -606,7 +624,7 @@ encode_vim_to_##name##_error_ret: \ do { \ const char *const buf_ = (const char *) buf; \ if (buf == NULL) { \ - ga_concat(gap, (char_u *) "''"); \ + ga_concat(gap, "''"); \ } else { \ const size_t len_ = (len); \ size_t num_quotes = 0; \ @@ -637,7 +655,7 @@ encode_vim_to_##name##_error_ret: \ do { \ char numbuf[NUMBUFLEN]; \ vim_snprintf(numbuf, NUMBUFLEN - 1, "%" PRId64, (int64_t) (num)); \ - ga_concat(gap, (char_u *) numbuf); \ + ga_concat(gap, numbuf); \ } while (0) #define CONV_FLOAT(flt) \ @@ -665,19 +683,19 @@ encode_vim_to_##name##_error_ret: \ #define CONV_FUNC(fun) \ do { \ - ga_concat(gap, (char_u *) "function("); \ + ga_concat(gap, "function("); \ CONV_STRING(fun, STRLEN(fun)); \ ga_append(gap, ')'); \ } while (0) #define CONV_EMPTY_LIST() \ - ga_concat(gap, (char_u *) "[]") + ga_concat(gap, "[]") #define CONV_LIST_START(lst) \ ga_append(gap, '[') #define CONV_EMPTY_DICT() \ - ga_concat(gap, (char_u *) "{}") + ga_concat(gap, "{}") #define CONV_SPECIAL_NIL() @@ -685,25 +703,25 @@ encode_vim_to_##name##_error_ret: \ #define CONV_UNSIGNED_NUMBER(num) -#define CONV_SPECIAL_MAP_START(lst) - -#define CONV_DICT_START(dct) \ +#define CONV_DICT_START(len) \ ga_append(gap, '{') -#define CONV_DICT_END(dct) \ +#define CONV_DICT_END() \ ga_append(gap, '}') -#define CONV_DICT_AFTER_KEY(dct) \ - ga_concat(gap, (char_u *) ": ") +#define CONV_DICT_AFTER_KEY() \ + ga_concat(gap, ": ") -#define CONV_DICT_BETWEEN_ITEMS(dct) \ - ga_concat(gap, (char_u *) ", ") +#define CONV_DICT_BETWEEN_ITEMS() \ + ga_concat(gap, ", ") + +#define CONV_SPECIAL_DICT_KEY_CHECK(kv_pair) #define CONV_LIST_END(lst) \ ga_append(gap, ']') -#define CONV_LIST_BETWEEN_ITEMS(lst) \ - CONV_DICT_BETWEEN_ITEMS(NULL) +#define CONV_LIST_BETWEEN_ITEMS() \ + CONV_DICT_BETWEEN_ITEMS() #define CONV_RECURSE(val, conv_type) \ do { \ @@ -731,7 +749,7 @@ encode_vim_to_##name##_error_ret: \ } \ } \ vim_snprintf(ebuf, NUMBUFLEN + 6, "{E724@%zu}", backref); \ - ga_concat(gap, (char_u *) &ebuf[0]); \ + ga_concat(gap, &ebuf[0]); \ return OK; \ } while (0) @@ -763,12 +781,272 @@ DEFINE_VIML_CONV_FUNCTIONS(static, string, garray_T *const, gap) } else { \ vim_snprintf(ebuf, NUMBUFLEN + 6, "[...@%zu]", backref); \ } \ - ga_concat(gap, (char_u *) &ebuf[0]); \ + ga_concat(gap, &ebuf[0]); \ return OK; \ } while (0) DEFINE_VIML_CONV_FUNCTIONS(, echo, garray_T *const, gap) +#undef CONV_RECURSE +#define CONV_RECURSE(val, conv_type) \ + do { \ + if (!did_echo_string_emsg) { \ + /* Only give this message once for a recursive call to avoid */ \ + /* flooding the user with errors. */ \ + did_echo_string_emsg = true; \ + EMSG(_("E724: unable to correctly dump variable " \ + "with self-referencing container")); \ + } \ + return OK; \ + } while (0) + +#undef CONV_ALLOW_SPECIAL +#define CONV_ALLOW_SPECIAL true + +#undef CONV_SPECIAL_NIL +#define CONV_SPECIAL_NIL() \ + ga_concat(gap, "null") + +#undef CONV_SPECIAL_BOOL +#define CONV_SPECIAL_BOOL(num) \ + ga_concat(gap, ((num)? "true": "false")) + +#undef CONV_UNSIGNED_NUMBER +#define CONV_UNSIGNED_NUMBER(num) \ + do { \ + char numbuf[NUMBUFLEN]; \ + vim_snprintf(numbuf, sizeof(numbuf), "%" PRIu64, (num)); \ + ga_concat(gap, numbuf); \ + } while (0) + +#undef CONV_FLOAT +#define CONV_FLOAT(flt) \ + do { \ + char numbuf[NUMBUFLEN]; \ + vim_snprintf(numbuf, NUMBUFLEN - 1, "%g", (flt)); \ + ga_concat(gap, numbuf); \ + } while (0) + +/// Last used p_enc value +/// +/// Generic pointer: it is not used as a string, only pointer comparisons are +/// performed. Must not be freed. +static const void *last_p_enc = NULL; + +/// Conversion setup for converting from last_p_enc to UTF-8 +static vimconv_T p_enc_conv = { + .vc_type = CONV_NONE, +}; + +/// Escape sequences used in JSON +static const char escapes[][3] = { + [BS] = "\\b", + [TAB] = "\\t", + [NL] = "\\n", + [CAR] = "\\r", + ['"'] = "\\\"", + ['\\'] = "\\\\", +}; + +static const char xdigits[] = "0123456789ABCDEF"; + +/// Convert given string to JSON string +/// +/// @param[out] gap Garray where result will be saved. +/// @param[in] buf Converted string. +/// @param[in] len Converted string length. +/// +/// @return OK in case of success, FAIL otherwise. +static inline int convert_to_json_string(garray_T *const gap, + const char *const buf, + const size_t len) + FUNC_ATTR_NONNULL_ARG(1) FUNC_ATTR_ALWAYS_INLINE +{ + const char *buf_ = buf; + if (buf_ == NULL) { + ga_concat(gap, "\"\""); + } else { + size_t len_ = len; + char *tofree = NULL; + if (last_p_enc != (const void *) p_enc) { + convert_setup(&p_enc_conv, p_enc, "utf-8"); + p_enc_conv.vc_fail = true; + last_p_enc = p_enc; + } + if (p_enc_conv.vc_type != CONV_NONE) { + tofree = string_convert(&p_enc_conv, buf_, &len_); + if (tofree == NULL) { + EMSG2(_("E474: Failed to convert string \"%s\" to UTF-8"), buf_); + return FAIL; + } + buf_ = tofree; + } + size_t str_len = 0; + for (size_t i = 0; i < len_;) { + const int ch = utf_ptr2char(buf + i); + const size_t shift = (ch == 0? 1: utf_ptr2len(buf + i)); + assert(shift > 0); + i += shift; + switch (ch) { + case BS: + case TAB: + case NL: + case FF: + case CAR: + case '"': + case '\\': { + str_len += 2; + break; + } + default: { + if (ch > 0x7F && shift == 1) { + EMSG2(_("E474: String \"%s\" contains byte that does not start any " + "UTF-8 character"), buf_); + return FAIL; + } else if ((0xD800 <= ch && ch <= 0xDB7F) + || (0xDC00 <= ch && ch <= 0xDFFF)) { + EMSG2(_("E474: UTF-8 string contains code point which belongs " + "to surrogate pairs"), buf_); + return FAIL; + } else if (vim_isprintc(ch)) { + str_len += shift; + } else { + str_len += ((sizeof("\\u1234") - 1) * (1 + (ch > 0xFFFF))); + } + break; + } + } + } + ga_append(gap, '"'); + ga_grow(gap, (int) str_len); + for (size_t i = 0; i < len_;) { + const int ch = utf_ptr2char(buf + i); + const size_t shift = (ch == 0? 1: utf_char2len(ch)); + assert(shift > 0); + // Is false on invalid unicode, but this should already be handled. + assert(ch == 0 || shift == utf_ptr2len(buf + i)); + switch (ch) { + case BS: + case TAB: + case NL: + case FF: + case CAR: + case '"': + case '\\': { + ga_concat_len(gap, escapes[ch], 2); + break; + } + default: { + if (vim_isprintc(ch)) { + ga_concat_len(gap, buf + i, shift); + } else if (ch <= 0xFFFF) { + ga_concat_len(gap, ((const char []) { + '\\', 'u', + xdigits[(ch >> (4 * 3)) & 0xF], + xdigits[(ch >> (4 * 2)) & 0xF], + xdigits[(ch >> (4 * 1)) & 0xF], + xdigits[(ch >> (4 * 0)) & 0xF], + }), sizeof("\\u1234") - 1); + } else { + uint32_t tmp = (uint32_t) ch - 0x010000; + uint16_t hi = 0xD800 + ((tmp >> 10) & 0x03FF); + uint16_t lo = 0xDC00 + ((tmp >> 0) & 0x03FF); + ga_concat_len(gap, ((const char []) { + '\\', 'u', + xdigits[(hi >> (4 * 3)) & 0xF], + xdigits[(hi >> (4 * 2)) & 0xF], + xdigits[(hi >> (4 * 1)) & 0xF], + xdigits[(hi >> (4 * 0)) & 0xF], + '\\', 'u', + xdigits[(lo >> (4 * 3)) & 0xF], + xdigits[(lo >> (4 * 2)) & 0xF], + xdigits[(lo >> (4 * 1)) & 0xF], + xdigits[(lo >> (4 * 0)) & 0xF], + }), (sizeof("\\u1234") - 1) * 2); + } + break; + } + } + i += shift; + } + ga_append(gap, '"'); + xfree(tofree); + } + return OK; +} + +#undef CONV_STRING +#define CONV_STRING(buf, len) \ + do { \ + if (convert_to_json_string(gap, (const char *) (buf), (len)) != OK) { \ + return FAIL; \ + } \ + } while (0) + +#undef CONV_EXT_STRING +#define CONV_EXT_STRING(buf, len, type) \ + do { \ + xfree(buf); \ + EMSG(_("E474: Unable to convert EXT string to JSON")); \ + return FAIL; \ + } while (0) + +#undef CONV_FUNC +#define CONV_FUNC(fun) \ + return conv_error(_("E474: Error while dumping %s, %s: " \ + "attempt to dump function reference"), \ + mpstack, objname) + +/// Check whether given key can be used in jsonencode() +/// +/// @param[in] tv Key to check. +static inline bool check_json_key(const typval_T *const tv) + FUNC_ATTR_NONNULL_ALL FUNC_ATTR_WARN_UNUSED_RESULT FUNC_ATTR_PURE + FUNC_ATTR_ALWAYS_INLINE +{ + if (tv->v_type == VAR_STRING) { + return true; + } + if (tv->v_type != VAR_DICT) { + return false; + } + const dict_T *const spdict = tv->vval.v_dict; + if (spdict->dv_hashtab.ht_used != 2) { + return false; + } + const dictitem_T *type_di; + const dictitem_T *val_di; + if ((type_di = dict_find((dict_T *) spdict, (char_u *) "_TYPE", -1)) == NULL + || type_di->di_tv.v_type != VAR_LIST + || (type_di->di_tv.vval.v_list != eval_msgpack_type_lists[kMPString] + && type_di->di_tv.vval.v_list != eval_msgpack_type_lists[kMPBinary]) + || (val_di = dict_find((dict_T *) spdict, (char_u *) "_VAL", -1)) == NULL + || val_di->di_tv.v_type != VAR_LIST) { + return false; + } + if (val_di->di_tv.vval.v_list == NULL) { + return true; + } + for (const listitem_T *li = val_di->di_tv.vval.v_list->lv_first; + li != NULL; li = li->li_next) { + if (li->li_tv.v_type != VAR_STRING) { + return false; + } + } + return true; +} + +#undef CONV_SPECIAL_DICT_KEY_CHECK +#define CONV_SPECIAL_DICT_KEY_CHECK(kv_pair) \ + do { \ + if (!check_json_key(&kv_pair->lv_first->li_tv)) { \ + EMSG(_("E474: Invalid key in special dictionary")); \ + return FAIL; \ + } \ + } while (0) + +DEFINE_VIML_CONV_FUNCTIONS(static, json, garray_T *const, gap) + #undef CONV_STRING #undef CONV_STR_STRING #undef CONV_EXT_STRING @@ -781,11 +1059,11 @@ DEFINE_VIML_CONV_FUNCTIONS(, echo, garray_T *const, gap) #undef CONV_SPECIAL_NIL #undef CONV_SPECIAL_BOOL #undef CONV_UNSIGNED_NUMBER -#undef CONV_SPECIAL_MAP_START #undef CONV_DICT_START #undef CONV_DICT_END #undef CONV_DICT_AFTER_KEY #undef CONV_DICT_BETWEEN_ITEMS +#undef CONV_SPECIAL_DICT_KEY_CHECK #undef CONV_LIST_END #undef CONV_LIST_BETWEEN_ITEMS #undef CONV_RECURSE @@ -838,6 +1116,27 @@ char *encode_tv2echo(typval_T *tv, size_t *len) return (char *) ga.ga_data; } +/// Return a string with the string representation of a variable. +/// Puts quotes around strings, so that they can be parsed back by eval(). +/// +/// @param[in] tv typval_T to convert. +/// @param[out] len Location where length of the result will be saved. +/// +/// @return String representation of the variable or NULL. +char *encode_tv2json(typval_T *tv, size_t *len) + FUNC_ATTR_NONNULL_ARG(1) FUNC_ATTR_MALLOC +{ + garray_T ga; + ga_init(&ga, (int)sizeof(char), 80); + encode_vim_to_json(&ga, tv, "encode_tv2json() argument"); + did_echo_string_emsg = false; + if (len != NULL) { + *len = (size_t) ga.ga_len; + } + ga_append(&ga, '\0'); + return (char *) ga.ga_data; +} + #define CONV_STRING(buf, len) \ do { \ if (buf == NULL) { \ @@ -906,21 +1205,20 @@ char *encode_tv2echo(typval_T *tv, size_t *len) #define CONV_UNSIGNED_NUMBER(num) \ msgpack_pack_uint64(packer, (num)) -#define CONV_SPECIAL_MAP_START(lst) \ - msgpack_pack_map(packer, (size_t) (lst)->lv_len) +#define CONV_DICT_START(len) \ + msgpack_pack_map(packer, (size_t) (len)) -#define CONV_DICT_START(dct) \ - msgpack_pack_map(packer, (dct)->dv_hashtab.ht_used) +#define CONV_DICT_END() -#define CONV_DICT_END(dct) +#define CONV_DICT_AFTER_KEY() -#define CONV_DICT_AFTER_KEY(dct) +#define CONV_DICT_BETWEEN_ITEMS() -#define CONV_DICT_BETWEEN_ITEMS(dct) +#define CONV_SPECIAL_DICT_KEY_CHECK(kv_pair) #define CONV_LIST_END(lst) -#define CONV_LIST_BETWEEN_ITEMS(lst) +#define CONV_LIST_BETWEEN_ITEMS() #define CONV_RECURSE(val, conv_type) \ return conv_error(_("E952: Unable to dump %s: " \ @@ -943,11 +1241,11 @@ DEFINE_VIML_CONV_FUNCTIONS(, msgpack, msgpack_packer *const, packer) #undef CONV_SPECIAL_NIL #undef CONV_SPECIAL_BOOL #undef CONV_UNSIGNED_NUMBER -#undef CONV_SPECIAL_MAP_START #undef CONV_DICT_START #undef CONV_DICT_END #undef CONV_DICT_AFTER_KEY #undef CONV_DICT_BETWEEN_ITEMS +#undef CONV_SPECIAL_DICT_KEY_CHECK #undef CONV_LIST_END #undef CONV_LIST_BETWEEN_ITEMS #undef CONV_RECURSE diff --git a/src/nvim/eval.c b/src/nvim/eval.c index b633bfb1b1..9e0698b104 100644 --- a/src/nvim/eval.c +++ b/src/nvim/eval.c @@ -6704,6 +6704,7 @@ static struct fst { { "jobstop", 1, 1, f_jobstop }, { "jobwait", 1, 2, f_jobwait }, { "join", 1, 2, f_join }, + { "jsonencode", 1, 1, f_jsonencode }, { "keys", 1, 1, f_keys }, { "last_buffer_nr", 0, 0, f_last_buffer_nr }, // obsolete { "len", 1, 1, f_len }, @@ -11494,6 +11495,13 @@ static void f_join(typval_T *argvars, typval_T *rettv) rettv->vval.v_string = NULL; } +/// jsonencode() function +static void f_jsonencode(typval_T *argvars, typval_T *rettv) +{ + rettv->v_type = VAR_STRING; + rettv->vval.v_string = (char_u *) encode_tv2json(&argvars[0], NULL); +} + /* * "keys()" function */ diff --git a/src/nvim/garray.c b/src/nvim/garray.c index e6cbd9332b..47a370b18c 100644 --- a/src/nvim/garray.c +++ b/src/nvim/garray.c @@ -188,11 +188,22 @@ void ga_concat(garray_T *gap, const char_u *restrict s) return; } - int len = (int)strlen((char *) s); + ga_concat_len(gap, (const char *restrict) s, strlen((char *) s)); +} + +/// Concatenate a string to a growarray which contains characters +/// +/// @param[out] gap Growarray to modify. +/// @param[in] s String to concatenate. +/// @param[in] len String length. +void ga_concat_len(garray_T *const gap, const char *restrict s, + const size_t len) + FUNC_ATTR_NONNULL_ALL +{ if (len) { - ga_grow(gap, len); + ga_grow(gap, (int) len); char *data = gap->ga_data; - memcpy(data + gap->ga_len, s, (size_t)len); + memcpy(data + gap->ga_len, s, len); gap->ga_len += len; } } diff --git a/test/functional/eval/json_functions_spec.lua b/test/functional/eval/json_functions_spec.lua new file mode 100644 index 0000000000..5e1c6a3984 --- /dev/null +++ b/test/functional/eval/json_functions_spec.lua @@ -0,0 +1,213 @@ +local helpers = require('test.functional.helpers') +local clear = helpers.clear +local funcs = helpers.funcs +local eq = helpers.eq +local eval = helpers.eval +local execute = helpers.execute +local exc_exec = helpers.exc_exec + +describe('jsonencode() function', function() + before_each(clear) + + it('dumps strings', function() + eq('"Test"', funcs.jsonencode('Test')) + eq('""', funcs.jsonencode('')) + eq('"\\t"', funcs.jsonencode('\t')) + eq('"\\n"', funcs.jsonencode('\n')) + eq('"\\u001B"', funcs.jsonencode('\27')) + end) + + it('dumps numbers', function() + eq('0', funcs.jsonencode(0)) + eq('10', funcs.jsonencode(10)) + eq('-10', funcs.jsonencode(-10)) + end) + + it('dumps floats', function() + eq('0.0', eval('jsonencode(0.0)')) + eq('10.5', funcs.jsonencode(10.5)) + eq('-10.5', funcs.jsonencode(-10.5)) + eq('-1.0e-5', funcs.jsonencode(-1e-5)) + eq('1.0e50', eval('jsonencode(1.0e50)')) + end) + + it('dumps lists', function() + eq('[]', funcs.jsonencode({})) + eq('[[]]', funcs.jsonencode({{}})) + eq('[[], []]', funcs.jsonencode({{}, {}})) + end) + + it('dumps dictionaries', function() + eq('{}', eval('jsonencode({})')) + eq('{"d": []}', funcs.jsonencode({d={}})) + eq('{"d": [], "e": []}', funcs.jsonencode({d={}, e={}})) + end) + + it('cannot dump generic mapping with generic mapping keys and values', + function() + execute('let todump = {"_TYPE": v:msgpack_types.map, "_VAL": []}') + execute('let todumpv1 = {"_TYPE": v:msgpack_types.map, "_VAL": []}') + execute('let todumpv2 = {"_TYPE": v:msgpack_types.map, "_VAL": []}') + execute('call add(todump._VAL, [todumpv1, todumpv2])') + eq('Vim(call):E474: Invalid key in special dictionary', exc_exec('call jsonencode(todump)')) + end) + + it('cannot dump generic mapping with ext key', function() + execute('let todump = {"_TYPE": v:msgpack_types.ext, "_VAL": [5, ["",""]]}') + execute('let todump = {"_TYPE": v:msgpack_types.map, "_VAL": [[todump, 1]]}') + eq('Vim(call):E474: Invalid key in special dictionary', exc_exec('call jsonencode(todump)')) + end) + + it('cannot dump generic mapping with array key', function() + execute('let todump = {"_TYPE": v:msgpack_types.array, "_VAL": [5, [""]]}') + execute('let todump = {"_TYPE": v:msgpack_types.map, "_VAL": [[todump, 1]]}') + eq('Vim(call):E474: Invalid key in special dictionary', exc_exec('call jsonencode(todump)')) + end) + + it('cannot dump generic mapping with UINT64_MAX key', function() + execute('let todump = {"_TYPE": v:msgpack_types.integer}') + execute('let todump._VAL = [1, 3, 0x7FFFFFFF, 0x7FFFFFFF]') + execute('let todump = {"_TYPE": v:msgpack_types.map, "_VAL": [[todump, 1]]}') + eq('Vim(call):E474: Invalid key in special dictionary', exc_exec('call jsonencode(todump)')) + end) + + it('cannot dump generic mapping with floating-point key', function() + execute('let todump = {"_TYPE": v:msgpack_types.float, "_VAL": 0.125}') + execute('let todump = {"_TYPE": v:msgpack_types.map, "_VAL": [[todump, 1]]}') + eq('Vim(call):E474: Invalid key in special dictionary', exc_exec('call jsonencode(todump)')) + end) + + it('can dump generic mapping with STR special key and NUL', function() + execute('let todump = {"_TYPE": v:msgpack_types.string, "_VAL": ["\\n"]}') + execute('let todump = {"_TYPE": v:msgpack_types.map, "_VAL": [[todump, 1]]}') + eq('{"\\u0000": 1}', eval('jsonencode(todump)')) + end) + + it('can dump generic mapping with BIN special key and NUL', function() + execute('let todump = {"_TYPE": v:msgpack_types.binary, "_VAL": ["\\n"]}') + execute('let todump = {"_TYPE": v:msgpack_types.map, "_VAL": [[todump, 1]]}') + eq('{"\\u0000": 1}', eval('jsonencode(todump)')) + end) + + it('can dump STR special mapping with NUL and NL', function() + execute('let todump = {"_TYPE": v:msgpack_types.string, "_VAL": ["\\n", ""]}') + eq('"\\u0000\\n"', eval('jsonencode(todump)')) + end) + + it('can dump BIN special mapping with NUL and NL', function() + execute('let todump = {"_TYPE": v:msgpack_types.binary, "_VAL": ["\\n", ""]}') + eq('"\\u0000\\n"', eval('jsonencode(todump)')) + end) + + it('cannot dump special ext mapping', function() + execute('let todump = {"_TYPE": v:msgpack_types.ext, "_VAL": [5, ["",""]]}') + eq('Vim(call):E474: Unable to convert EXT string to JSON', exc_exec('call jsonencode(todump)')) + end) + + it('can dump special array mapping', function() + execute('let todump = {"_TYPE": v:msgpack_types.array, "_VAL": [5, [""]]}') + eq('[5, [""]]', eval('jsonencode(todump)')) + end) + + it('can dump special UINT64_MAX mapping', function() + execute('let todump = {"_TYPE": v:msgpack_types.integer}') + execute('let todump._VAL = [1, 3, 0x7FFFFFFF, 0x7FFFFFFF]') + eq('18446744073709551615', eval('jsonencode(todump)')) + end) + + it('can dump special INT64_MIN mapping', function() + execute('let todump = {"_TYPE": v:msgpack_types.integer}') + execute('let todump._VAL = [-1, 2, 0, 0]') + eq('-9223372036854775808', eval('jsonencode(todump)')) + end) + + it('can dump special BOOLEAN true mapping', function() + execute('let todump = {"_TYPE": v:msgpack_types.boolean, "_VAL": 1}') + eq('true', eval('jsonencode(todump)')) + end) + + it('can dump special BOOLEAN false mapping', function() + execute('let todump = {"_TYPE": v:msgpack_types.boolean, "_VAL": 0}') + eq('false', eval('jsonencode(todump)')) + end) + + it('can dump special NIL mapping', function() + execute('let todump = {"_TYPE": v:msgpack_types.nil, "_VAL": 0}') + eq('null', eval('jsonencode(todump)')) + end) + + it('fails to dump a function reference', function() + eq('Vim(call):E474: Error while dumping encode_tv2json() argument, itself: attempt to dump function reference', + exc_exec('call jsonencode(function("tr"))')) + end) + + it('fails to dump a function reference in a list', function() + eq('Vim(call):E474: Error while dumping encode_tv2json() argument, index 0: attempt to dump function reference', + exc_exec('call jsonencode([function("tr")])')) + end) + + it('fails to dump a recursive list', function() + execute('let todump = [[[]]]') + execute('call add(todump[0][0], todump)') + eq('Vim(call):E724: unable to correctly dump variable with self-referencing container', + exc_exec('call jsonencode(todump)')) + end) + + it('fails to dump a recursive dict', function() + execute('let todump = {"d": {"d": {}}}') + execute('call extend(todump.d.d, {"d": todump})') + eq('Vim(call):E724: unable to correctly dump variable with self-referencing container', + exc_exec('call jsonencode([todump])')) + end) + + it('can dump dict with two same dicts inside', function() + execute('let inter = {}') + execute('let todump = {"a": inter, "b": inter}') + eq('{"a": {}, "b": {}}', eval('jsonencode(todump)')) + end) + + it('can dump list with two same lists inside', function() + execute('let inter = []') + execute('let todump = [inter, inter]') + eq('[[], []]', eval('jsonencode(todump)')) + end) + + it('fails to dump a recursive list in a special dict', function() + execute('let todump = {"_TYPE": v:msgpack_types.array, "_VAL": []}') + execute('call add(todump._VAL, todump)') + eq('Vim(call):E724: unable to correctly dump variable with self-referencing container', + exc_exec('call jsonencode(todump)')) + end) + + it('fails to dump a recursive (val) map in a special dict', function() + execute('let todump = {"_TYPE": v:msgpack_types.map, "_VAL": []}') + execute('call add(todump._VAL, ["", todump])') + eq('Vim(call):E724: unable to correctly dump variable with self-referencing container', + exc_exec('call jsonencode([todump])')) + end) + + it('fails to dump a recursive (val) map in a special dict, _VAL reference', function() + execute('let todump = {"_TYPE": v:msgpack_types.map, "_VAL": [["", []]]}') + execute('call add(todump._VAL[0][1], todump._VAL)') + eq('Vim(call):E724: unable to correctly dump variable with self-referencing container', + exc_exec('call jsonencode(todump)')) + end) + + it('fails to dump a recursive (val) special list in a special dict', + function() + execute('let todump = {"_TYPE": v:msgpack_types.array, "_VAL": []}') + execute('call add(todump._VAL, ["", todump._VAL])') + eq('Vim(call):E724: unable to correctly dump variable with self-referencing container', + exc_exec('call jsonencode(todump)')) + end) + + it('fails when called with no arguments', function() + eq('Vim(call):E119: Not enough arguments for function: jsonencode', + exc_exec('call jsonencode()')) + end) + + it('fails when called with two arguments', function() + eq('Vim(call):E118: Too many arguments for function: jsonencode', + exc_exec('call jsonencode(["", ""], 1)')) + end) +end) From 256a5d25226505c56c067d35d715f1a184cd05ee Mon Sep 17 00:00:00 2001 From: ZyX Date: Sat, 30 Jan 2016 23:22:29 +0300 Subject: [PATCH 05/82] encode: Add a hint for static analyzer that cur_tv is not NULL --- src/nvim/encode.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/nvim/encode.c b/src/nvim/encode.c index 84aaed53cb..fded609483 100644 --- a/src/nvim/encode.c +++ b/src/nvim/encode.c @@ -608,6 +608,7 @@ scope int encode_vim_to_##name(firstargtype firstargname, typval_T *const tv, \ break; \ } \ } \ + assert(cur_tv != NULL); \ if (name##_convert_one_value(firstargname, &mpstack, cur_tv, copyID, \ objname) == FAIL) { \ goto encode_vim_to_##name##_error_ret; \ From 18903bd9b88ec960cb36b1ddd2b5062aad4bac2e Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 31 Jan 2016 00:06:46 +0300 Subject: [PATCH 06/82] eval: Add special variable type --- src/nvim/api/private/helpers.c | 8 +-- src/nvim/encode.c | 59 ++++++++++++++++----- src/nvim/eval.c | 44 +++++++++------- src/nvim/eval_defs.h | 61 ++++++++++++++-------- src/nvim/version.c | 8 +-- test/functional/eval/special_vars_spec.lua | 36 +++++++++++++ 6 files changed, 152 insertions(+), 64 deletions(-) create mode 100644 test/functional/eval/special_vars_spec.lua diff --git a/src/nvim/api/private/helpers.c b/src/nvim/api/private/helpers.c index 7a0b5191d7..c770618ce4 100644 --- a/src/nvim/api/private/helpers.c +++ b/src/nvim/api/private/helpers.c @@ -397,13 +397,13 @@ bool object_to_vim(Object obj, typval_T *tv, Error *err) switch (obj.type) { case kObjectTypeNil: - tv->v_type = VAR_NUMBER; - tv->vval.v_number = 0; + tv->v_type = VAR_SPECIAL; + tv->vval.v_special = kSpecialVarNull; break; case kObjectTypeBoolean: - tv->v_type = VAR_NUMBER; - tv->vval.v_number = obj.data.boolean; + tv->v_type = VAR_SPECIAL; + tv->vval.v_special = obj.data.boolean? kSpecialVarTrue: kSpecialVarFalse; break; case kObjectTypeBuffer: diff --git a/src/nvim/encode.c b/src/nvim/encode.c index fded609483..6fdbe67ec8 100644 --- a/src/nvim/encode.c +++ b/src/nvim/encode.c @@ -343,6 +343,24 @@ static int name##_convert_one_value(firstargtype firstargname, \ })); \ break; \ } \ + case VAR_SPECIAL: { \ + switch (tv->vval.v_special) { \ + case kSpecialVarNull: { \ + CONV_NIL(); \ + break; \ + } \ + case kSpecialVarTrue: \ + case kSpecialVarFalse: { \ + CONV_BOOL(tv->vval.v_special == kSpecialVarTrue); \ + break; \ + } \ + case kSpecialVarNone: { \ + CONV_NONE(); \ + break; \ + } \ + } \ + break; \ + } \ case VAR_DICT: { \ if (tv->vval.v_dict == NULL \ || tv->vval.v_dict->dv_hashtab.ht_used == 0) { \ @@ -369,14 +387,14 @@ static int name##_convert_one_value(firstargtype firstargname, \ } \ switch ((MessagePackType) i) { \ case kMPNil: { \ - CONV_SPECIAL_NIL(); \ + CONV_NIL(); \ break; \ } \ case kMPBoolean: { \ if (val_di->di_tv.v_type != VAR_NUMBER) { \ goto name##_convert_one_value_regular_dict; \ } \ - CONV_SPECIAL_BOOL(val_di->di_tv.vval.v_number); \ + CONV_BOOL(val_di->di_tv.vval.v_number); \ break; \ } \ case kMPInteger: { \ @@ -698,9 +716,14 @@ encode_vim_to_##name##_error_ret: \ #define CONV_EMPTY_DICT() \ ga_concat(gap, "{}") -#define CONV_SPECIAL_NIL() +#define CONV_NIL() \ + ga_append(gap, "v:null") -#define CONV_SPECIAL_BOOL(num) +#define CONV_BOOL(num) \ + ga_append(gap, ((num)? "v:true": "v:false")) + +#define CONV_NONE() \ + ga_append(gap, "v:none") #define CONV_UNSIGNED_NUMBER(num) @@ -804,12 +827,12 @@ DEFINE_VIML_CONV_FUNCTIONS(, echo, garray_T *const, gap) #undef CONV_ALLOW_SPECIAL #define CONV_ALLOW_SPECIAL true -#undef CONV_SPECIAL_NIL -#define CONV_SPECIAL_NIL() \ +#undef CONV_NIL +#define CONV_NIL() \ ga_concat(gap, "null") -#undef CONV_SPECIAL_BOOL -#define CONV_SPECIAL_BOOL(num) \ +#undef CONV_BOOL +#define CONV_BOOL(num) \ ga_concat(gap, ((num)? "true": "false")) #undef CONV_UNSIGNED_NUMBER @@ -1046,6 +1069,9 @@ static inline bool check_json_key(const typval_T *const tv) } \ } while (0) +#undef CONV_NONE +#define CONV_NONE() + DEFINE_VIML_CONV_FUNCTIONS(static, json, garray_T *const, gap) #undef CONV_STRING @@ -1057,8 +1083,9 @@ DEFINE_VIML_CONV_FUNCTIONS(static, json, garray_T *const, gap) #undef CONV_EMPTY_LIST #undef CONV_LIST_START #undef CONV_EMPTY_DICT -#undef CONV_SPECIAL_NIL -#undef CONV_SPECIAL_BOOL +#undef CONV_NIL +#undef CONV_BOOL +#undef CONV_NONE #undef CONV_UNSIGNED_NUMBER #undef CONV_DICT_START #undef CONV_DICT_END @@ -1191,10 +1218,14 @@ char *encode_tv2json(typval_T *tv, size_t *len) #define CONV_EMPTY_DICT() \ msgpack_pack_map(packer, 0) -#define CONV_SPECIAL_NIL() \ +#define CONV_NIL() \ msgpack_pack_nil(packer) -#define CONV_SPECIAL_BOOL(num) \ +#define CONV_NONE() \ + return conv_error(_("E953: Attempt to convert v:none in %s, %s"), \ + mpstack, objname) + +#define CONV_BOOL(num) \ do { \ if ((num)) { \ msgpack_pack_true(packer); \ @@ -1239,8 +1270,8 @@ DEFINE_VIML_CONV_FUNCTIONS(, msgpack, msgpack_packer *const, packer) #undef CONV_EMPTY_LIST #undef CONV_LIST_START #undef CONV_EMPTY_DICT -#undef CONV_SPECIAL_NIL -#undef CONV_SPECIAL_BOOL +#undef CONV_NIL +#undef CONV_BOOL #undef CONV_UNSIGNED_NUMBER #undef CONV_DICT_START #undef CONV_DICT_END diff --git a/src/nvim/eval.c b/src/nvim/eval.c index 9e0698b104..114368d621 100644 --- a/src/nvim/eval.c +++ b/src/nvim/eval.c @@ -8474,6 +8474,9 @@ static void f_empty(typval_T *argvars, typval_T *rettv) n = argvars[0].vval.v_dict == NULL || argvars[0].vval.v_dict->dv_hashtab.ht_used == 0; break; + case VAR_SPECIAL: + n = argvars[0].vval.v_special != kSpecialVarTrue; + break; default: EMSG2(_(e_intern2), "f_empty()"); n = 0; @@ -17571,26 +17574,27 @@ handle_subscript ( void free_tv(typval_T *varp) { if (varp != NULL) { - switch (varp->v_type) { - case VAR_FUNC: - func_unref(varp->vval.v_string); - /*FALLTHROUGH*/ - case VAR_STRING: - xfree(varp->vval.v_string); - break; - case VAR_LIST: - list_unref(varp->vval.v_list); - break; - case VAR_DICT: - dict_unref(varp->vval.v_dict); - break; - case VAR_NUMBER: - case VAR_FLOAT: - case VAR_UNKNOWN: - break; - default: - EMSG2(_(e_intern2), "free_tv()"); - break; + switch ((VarType) varp->v_type) { + case VAR_FUNC: + func_unref(varp->vval.v_string); + /*FALLTHROUGH*/ + case VAR_STRING: + xfree(varp->vval.v_string); + break; + case VAR_LIST: + list_unref(varp->vval.v_list); + break; + case VAR_DICT: + dict_unref(varp->vval.v_dict); + break; + case VAR_SPECIAL: + case VAR_NUMBER: + case VAR_FLOAT: + case VAR_UNKNOWN: + break; + default: + EMSG2(_(e_intern2), "free_tv()"); + break; } xfree(varp); } diff --git a/src/nvim/eval_defs.h b/src/nvim/eval_defs.h index cdad1f3197..56833f97d8 100644 --- a/src/nvim/eval_defs.h +++ b/src/nvim/eval_defs.h @@ -16,38 +16,55 @@ typedef double float_T; typedef struct listvar_S list_T; typedef struct dictvar_S dict_T; -/* - * Structure to hold an internal variable without a name. - */ +/// Special variable values +typedef enum { + kSpecialVarNull, ///< v:null + kSpecialVarNone, ///< v:none + kSpecialVarFalse, ///< v:false + kSpecialVarTrue, ///< v:true +} SpecialVarValue; + +/// Structure that holds an internal variable value typedef struct { - char v_type; /* see below: VAR_NUMBER, VAR_STRING, etc. */ - char v_lock; /* see below: VAR_LOCKED, VAR_FIXED */ + VarType v_type; ///< Variable type. + VarLockStatus v_lock; ///< Variable lock status. union { - varnumber_T v_number; /* number value */ - float_T v_float; /* floating number value */ - char_u *v_string; /* string value (can be NULL!) */ - list_T *v_list; /* list value (can be NULL!) */ - dict_T *v_dict; /* dict value (can be NULL!) */ - } vval; + varnumber_T v_number; ///< Number, for VAR_NUMBER. + SpecialVarValue v_special; ///< Special value, for VAR_SPECIAL. + float_T v_float; ///< Floating-point number, for VAR_FLOAT. + char_u *v_string; ///< String, for VAR_STRING and VAR_FUNC, can be NULL. + list_T *v_list; ///< List for VAR_LIST, can be NULL. + dict_T *v_dict; ///< Dictionary for VAR_DICT, can be NULL. + } vval; ///< Actual value. } typval_T; -/* Values for "v_type". */ -#define VAR_UNKNOWN 0 -#define VAR_NUMBER 1 /* "v_number" is used */ -#define VAR_STRING 2 /* "v_string" is used */ -#define VAR_FUNC 3 /* "v_string" is function name */ -#define VAR_LIST 4 /* "v_list" is used */ -#define VAR_DICT 5 /* "v_dict" is used */ -#define VAR_FLOAT 6 /* "v_float" is used */ +/// VimL variable types, for use in typval_T.v_type +/// +/// @warning Numbers are part of the user API (returned by type()), so they must +/// not be changed. +typedef enum { + VAR_UNKNOWN = 0, ///< Unknown (unspecified) value. + VAR_NUMBER = 1, ///< Number, .v_number is used. + VAR_STRING = 2, ///< String, .v_string is used. + VAR_FUNC = 3, ///< Function referene, .v_string is used for function name. + VAR_LIST = 4, ///< List, .v_list is used. + VAR_DICT = 5, ///< Dictionary, .v_dict is used. + VAR_FLOAT = 6, ///< Floating-point value, .v_float is used. + VAR_SPECIAL = 7, ///< Special value (true, false, null, none), .v_special + ///< is used. +} VarType; /* Values for "dv_scope". */ #define VAR_SCOPE 1 /* a:, v:, s:, etc. scope dictionaries */ #define VAR_DEF_SCOPE 2 /* l:, g: scope dictionaries: here funcrefs are not allowed to mask existing functions */ -/* Values for "v_lock". */ -#define VAR_LOCKED 1 /* locked with lock(), can use unlock() */ -#define VAR_FIXED 2 /* locked forever */ +/// Variable lock status for typval_T.v_lock +typedef enum { + VAR_UNLOCKED = 0, ///< Not locked. + VAR_LOCKED, ///< User lock, can be unlocked. + VAR_FIXED, ///< Locked forever. +} VarLockStatus; /* * Structure to hold an item of a list: an internal variable without a name. diff --git a/src/nvim/version.c b/src/nvim/version.c index 3b160d0d99..8f45620570 100644 --- a/src/nvim/version.c +++ b/src/nvim/version.c @@ -181,20 +181,20 @@ static int included_patches[] = { // 1184 NA // 1183 NA // 1182 NA - // 1181, + 1181, 1180, // 1179, - // 1178, + 1178, // 1177 NA // 1176 NA // 1175 NA // 1174 NA - // 1173, + 1173, // 1172 NA // 1171 NA // 1170 NA // 1169 NA - // 1168, + 1168, // 1167, // 1166, // 1165 NA diff --git a/test/functional/eval/special_vars_spec.lua b/test/functional/eval/special_vars_spec.lua new file mode 100644 index 0000000000..7adc529c88 --- /dev/null +++ b/test/functional/eval/special_vars_spec.lua @@ -0,0 +1,36 @@ +local helpers = require('test.functional.helpers') +local execute = helpers.execute +local funcs = helpers.funcs +local clear = helpers.clear +local eval = helpers.eval + +describe('Special values', function() + before_each(clear) + + it('do not cause error when freed', function() + execute([[ + function Test() + try + return v:true + finally + return 'something else' + endtry + endfunction + ]]) + eq(true, funcs.Test()) + end) + + it('work with empty()', function() + eq(0, funcs.empty(true)) + eq(1, funcs.empty(false)) + eq(1, funcs.empty(nil)) + eq(1, eval('empty(v:none)')) + end) + + it('can be stringified and eval’ed back', function() + eq(true, funcs.eval(funcs.string(true))) + eq(false, funcs.eval(funcs.string(false))) + eq(nil, funcs.eval(funcs.string(nil))) + eq(1, eval('eval(string(v:none)) is# v:none')) + end) +end) From d70a322c40e849f98ad573d2a37dc680c5616b26 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 31 Jan 2016 01:25:00 +0300 Subject: [PATCH 07/82] eval: Add special variables v:false, v:null, v:none --- runtime/doc/eval.txt | 32 ++- src/nvim/api/private/helpers.c | 20 ++ src/nvim/encode.c | 31 ++- src/nvim/encode.h | 3 + src/nvim/eval.c | 209 +++++++++++++----- src/nvim/eval.h | 12 +- src/nvim/eval_defs.h | 47 ++-- src/nvim/version.c | 14 +- .../eval/msgpack_functions_spec.lua | 26 +++ test/functional/eval/special_vars_spec.lua | 156 ++++++++++++- test/functional/eval/string_spec.lua | 7 + 11 files changed, 441 insertions(+), 116 deletions(-) diff --git a/runtime/doc/eval.txt b/runtime/doc/eval.txt index 2d08afff4f..572cf4c03f 100644 --- a/runtime/doc/eval.txt +++ b/runtime/doc/eval.txt @@ -1418,6 +1418,13 @@ v:exception The value of the exception most recently caught and not :endtry < Output: "caught oops". + *v:false* *false-variable* +v:false Special value used to put "false" in JSON and msgpack. See + |jsonencode()|. This value is converted to "false" when used + as a String (e.g. in |expr5| with string concatenation + operator) and to zero when used as a Number (e.g. in |expr5| + or |expr7| when used with numeric operators). + *v:fcs_reason* *fcs_reason-variable* v:fcs_reason The reason why the |FileChangedShell| event was triggered. Can be used in an autocommand to decide what to do and/or what @@ -1557,6 +1564,20 @@ v:msgpack_types Dictionary containing msgpack types used by |msgpackparse()| (not editable) empty lists. To check whether some list is one of msgpack types, use |is| operator. + *v:null* *null-variable* +v:null Special value used to put "null" in JSON and NIL in msgpack. + See |jsonencode()|. This value is converted to "null" when + used as a String (e.g. in |expr5| with string concatenation + operator) and to zero when used as a Number (e.g. in |expr5| + or |expr7| when used with numeric operators). + + *v:none* *none-variable* +v:none Special value used to put an empty item in JSON. See + |jsonencode()|. This value is converted to "none" when used + as a String (e.g. in |expr5| with string concatenation + operator) and to zero when used as a Number (e.g. in |expr5| + or |expr7| when used with numeric operators). + *v:oldfiles* *oldfiles-variable* v:oldfiles List of file names that is loaded from the |shada| file on startup. These are the files that Vim remembers marks for. @@ -1722,6 +1743,13 @@ v:throwpoint The point where the exception most recently caught and not :endtry < Output: "Exception from test.vim, line 2" + *v:true* *true-variable* +v:true Special value used to put "true" in JSON and msgpack. See + |jsonencode()|. This value is converted to "true" when used + as a String (e.g. in |expr5| with string concatenation + operator) and to one when used as a Number (e.g. in |expr5| or + |expr7| when used with numeric operators). + *v:val* *val-variable* v:val Value of the current item of a |List| or |Dictionary|. Only valid while evaluating the expression used with |map()| and @@ -4832,8 +4860,8 @@ msgpackdump({list}) {Nvim} *msgpackdump()* (dictionary with zero items is represented by 0x80 byte in messagepack). - Limitations: *E951* *E952* - 1. |Funcref|s cannot be dumped. + Limitations: *E951* *E952* *E953* + 1. |Funcref|s and |v:none| cannot be dumped. 2. Containers that reference themselves cannot be dumped. 3. Dictionary keys are always dumped as STR strings. 4. Other strings are always dumped as BIN strings. diff --git a/src/nvim/api/private/helpers.c b/src/nvim/api/private/helpers.c index c770618ce4..a8082655fd 100644 --- a/src/nvim/api/private/helpers.c +++ b/src/nvim/api/private/helpers.c @@ -651,6 +651,22 @@ static Object vim_to_object_rec(typval_T *obj, PMap(ptr_t) *lookup) } switch (obj->v_type) { + case VAR_SPECIAL: + switch (obj->vval.v_special) { + case kSpecialVarTrue: + case kSpecialVarFalse: { + rv.type = kObjectTypeBoolean; + rv.data.boolean = (obj->vval.v_special == kSpecialVarTrue); + break; + } + case kSpecialVarNull: + case kSpecialVarNone: { + rv.type = kObjectTypeNil; + break; + } + } + break; + case VAR_STRING: rv.type = kObjectTypeString; rv.data.string = cstr_to_string((char *) obj->vval.v_string); @@ -730,6 +746,10 @@ static Object vim_to_object_rec(typval_T *obj, PMap(ptr_t) *lookup) } } break; + + case VAR_UNKNOWN: + case VAR_FUNC: + break; } return rv; diff --git a/src/nvim/encode.c b/src/nvim/encode.c index 6fdbe67ec8..c80e9783e0 100644 --- a/src/nvim/encode.c +++ b/src/nvim/encode.c @@ -8,10 +8,12 @@ #include #include +#include #include "nvim/encode.h" #include "nvim/buffer_defs.h" // vimconv_T #include "nvim/eval.h" +#include "nvim/eval_defs.h" #include "nvim/garray.h" #include "nvim/mbyte.h" #include "nvim/message.h" @@ -53,6 +55,13 @@ typedef struct { /// Stack used to convert VimL values to messagepack. typedef kvec_t(MPConvStackVal) MPConvStack; +const char *const encode_special_var_names[] = { + [kSpecialVarNull] = "null", + [kSpecialVarNone] = "none", + [kSpecialVarTrue] = "true", + [kSpecialVarFalse] = "false", +}; + #ifdef INCLUDE_GENERATED_DECLARATIONS # include "encode.c.generated.h" #endif @@ -355,7 +364,7 @@ static int name##_convert_one_value(firstargtype firstargname, \ break; \ } \ case kSpecialVarNone: { \ - CONV_NONE(); \ + CONV_NONE_VAL(); \ break; \ } \ } \ @@ -558,8 +567,7 @@ scope int encode_vim_to_##name(firstargtype firstargname, typval_T *const tv, \ const char *const objname) \ FUNC_ATTR_WARN_UNUSED_RESULT \ { \ - current_copyID += COPYID_INC; \ - const int copyID = current_copyID; \ + const int copyID = get_copyID(); \ MPConvStack mpstack; \ kv_init(mpstack); \ if (name##_convert_one_value(firstargname, &mpstack, tv, copyID, objname) \ @@ -717,13 +725,13 @@ encode_vim_to_##name##_error_ret: \ ga_concat(gap, "{}") #define CONV_NIL() \ - ga_append(gap, "v:null") + ga_concat(gap, "v:null") #define CONV_BOOL(num) \ - ga_append(gap, ((num)? "v:true": "v:false")) + ga_concat(gap, ((num)? "v:true": "v:false")) -#define CONV_NONE() \ - ga_append(gap, "v:none") +#define CONV_NONE_VAL() \ + ga_concat(gap, "v:none") #define CONV_UNSIGNED_NUMBER(num) @@ -1069,8 +1077,8 @@ static inline bool check_json_key(const typval_T *const tv) } \ } while (0) -#undef CONV_NONE -#define CONV_NONE() +#undef CONV_NONE_VAL +#define CONV_NONE_VAL() DEFINE_VIML_CONV_FUNCTIONS(static, json, garray_T *const, gap) @@ -1085,7 +1093,7 @@ DEFINE_VIML_CONV_FUNCTIONS(static, json, garray_T *const, gap) #undef CONV_EMPTY_DICT #undef CONV_NIL #undef CONV_BOOL -#undef CONV_NONE +#undef CONV_NONE_VAL #undef CONV_UNSIGNED_NUMBER #undef CONV_DICT_START #undef CONV_DICT_END @@ -1221,7 +1229,7 @@ char *encode_tv2json(typval_T *tv, size_t *len) #define CONV_NIL() \ msgpack_pack_nil(packer) -#define CONV_NONE() \ +#define CONV_NONE_VAL() \ return conv_error(_("E953: Attempt to convert v:none in %s, %s"), \ mpstack, objname) @@ -1272,6 +1280,7 @@ DEFINE_VIML_CONV_FUNCTIONS(, msgpack, msgpack_packer *const, packer) #undef CONV_EMPTY_DICT #undef CONV_NIL #undef CONV_BOOL +#undef CONV_NONE_VAL #undef CONV_UNSIGNED_NUMBER #undef CONV_DICT_START #undef CONV_DICT_END diff --git a/src/nvim/encode.h b/src/nvim/encode.h index 799850aab9..5b81ed84dc 100644 --- a/src/nvim/encode.h +++ b/src/nvim/encode.h @@ -51,6 +51,9 @@ static inline ListReaderState encode_init_lrstate(const list_T *const list) }; } +/// Array mapping values from SpecialVarValue enum to names +extern const char *const encode_special_var_names[]; + #ifdef INCLUDE_GENERATED_DECLARATIONS # include "encode.h.generated.h" #endif diff --git a/src/nvim/eval.c b/src/nvim/eval.c index 114368d621..819b3059e2 100644 --- a/src/nvim/eval.c +++ b/src/nvim/eval.c @@ -178,8 +178,6 @@ static dictitem_T globvars_var; /* variable used for g: */ */ static hashtab_T compat_hashtab; -int current_copyID = 0; - hashtab_T func_hashtab; /* @@ -366,11 +364,16 @@ static struct vimvar { { VV_NAME("errors", VAR_LIST), 0 }, { VV_NAME("msgpack_types", VAR_DICT), VV_RO }, { VV_NAME("event", VAR_DICT), VV_RO }, + { VV_NAME("false", VAR_SPECIAL), VV_RO }, + { VV_NAME("true", VAR_SPECIAL), VV_RO }, + { VV_NAME("null", VAR_SPECIAL), VV_RO }, + { VV_NAME("none", VAR_SPECIAL), VV_RO }, }; /* shorthand */ #define vv_type vv_di.di_tv.v_type #define vv_nr vv_di.di_tv.vval.v_number +#define vv_special vv_di.di_tv.vval.v_special #define vv_float vv_di.di_tv.vval.v_float #define vv_str vv_di.di_tv.vval.v_string #define vv_list vv_di.di_tv.vval.v_list @@ -506,7 +509,13 @@ void eval_init(void) set_vim_var_list(VV_ERRORS, list_alloc()); set_vim_var_nr(VV_SEARCHFORWARD, 1L); set_vim_var_nr(VV_HLSEARCH, 1L); - set_reg_var(0); /* default for v:register is not 0 but '"' */ + + set_vim_var_special(VV_FALSE, kSpecialVarFalse); + set_vim_var_special(VV_TRUE, kSpecialVarTrue); + set_vim_var_special(VV_NONE, kSpecialVarNone); + set_vim_var_special(VV_NULL, kSpecialVarNull); + + set_reg_var(0); // default for v:register is not 0 but '"' } #if defined(EXITFREE) @@ -2368,11 +2377,12 @@ static int tv_op(typval_T *tv1, typval_T *tv2, char_u *op) char_u numbuf[NUMBUFLEN]; char_u *s; - /* Can't do anything with a Funcref or a Dict on the right. */ + // Can't do anything with a Funcref, a Dict or special value on the right. if (tv2->v_type != VAR_FUNC && tv2->v_type != VAR_DICT) { switch (tv1->v_type) { case VAR_DICT: case VAR_FUNC: + case VAR_SPECIAL: break; case VAR_LIST: @@ -2440,6 +2450,9 @@ static int tv_op(typval_T *tv1, typval_T *tv2, char_u *op) tv1->vval.v_float -= f; } return OK; + + case VAR_UNKNOWN: + assert(false); } } @@ -3077,6 +3090,15 @@ static void item_lock(typval_T *tv, int deep, int lock) } } } + break; + case VAR_NUMBER: + case VAR_FLOAT: + case VAR_STRING: + case VAR_FUNC: + case VAR_SPECIAL: + break; + case VAR_UNKNOWN: + assert(false); } --recurse; } @@ -4306,6 +4328,11 @@ eval_index ( if (verbose) EMSG(_(e_float_as_string)); return FAIL; + } else if (rettv->v_type == VAR_SPECIAL) { + if (verbose) { + EMSG(_("E15: Cannot index a special value")); + } + return FAIL; } init_tv(&var1); @@ -4496,6 +4523,11 @@ eval_index ( *rettv = var1; } break; + case VAR_FUNC: + case VAR_FLOAT: + case VAR_UNKNOWN: + case VAR_SPECIAL: + assert(false); } } @@ -5040,6 +5072,12 @@ tv_equal ( s1 = get_tv_string_buf(tv1, buf1); s2 = get_tv_string_buf(tv2, buf2); return (ic ? mb_stricmp(s1, s2) : STRCMP(s1, s2)) == 0; + + case VAR_SPECIAL: + return tv1->vval.v_special == tv2->vval.v_special; + + case VAR_UNKNOWN: + break; } EMSG2(_(e_intern2), "tv_equal()"); @@ -5505,6 +5543,22 @@ static int list_join(garray_T *const gap, list_T *const l, return retval; } +/// Get next (unique) copy ID +/// +/// Used for traversing nested structures e.g. when serializing them or garbage +/// collecting. +int get_copyID(void) + FUNC_ATTR_WARN_UNUSED_RESULT +{ + // CopyID for recursively traversing lists and dicts + // + // This value is needed to avoid endless recursiveness. Last bit is used for + // previous_funccal and normally ignored when comparing. + static int current_copyID = 0; + current_copyID += COPYID_INC; + return current_copyID; +} + /* * Garbage collection for lists and dictionaries. * @@ -5540,8 +5594,7 @@ bool garbage_collect(void) // We advance by two because we add one for items referenced through // previous_funccal. - current_copyID += COPYID_INC; - int copyID = current_copyID; + const int copyID = get_copyID(); // 1. Go through all accessible variables and mark all lists and dicts // with copyID. @@ -5886,6 +5939,15 @@ bool set_ref_in_item(typval_T *tv, int copyID, ht_stack_T **ht_stack, } break; } + + case VAR_FUNC: + case VAR_UNKNOWN: + case VAR_SPECIAL: + case VAR_FLOAT: + case VAR_NUMBER: + case VAR_STRING: { + break; + } } return abort; } @@ -8260,9 +8322,8 @@ static void f_deepcopy(typval_T *argvars, typval_T *rettv) if (noref < 0 || noref > 1) EMSG(_(e_invarg)); else { - current_copyID += COPYID_INC; var_item_copy(NULL, &argvars[0], rettv, true, (noref == 0 - ? current_copyID + ? get_copyID() : 0)); } } @@ -8477,7 +8538,7 @@ static void f_empty(typval_T *argvars, typval_T *rettv) case VAR_SPECIAL: n = argvars[0].vval.v_special != kSpecialVarTrue; break; - default: + case VAR_UNKNOWN: EMSG2(_(e_intern2), "f_empty()"); n = 0; } @@ -16386,13 +16447,28 @@ static void f_type(typval_T *argvars, typval_T *rettv) int n; switch (argvars[0].v_type) { - case VAR_NUMBER: n = 0; break; - case VAR_STRING: n = 1; break; - case VAR_FUNC: n = 2; break; - case VAR_LIST: n = 3; break; - case VAR_DICT: n = 4; break; - case VAR_FLOAT: n = 5; break; - default: EMSG2(_(e_intern2), "f_type()"); n = 0; break; + case VAR_NUMBER: n = 0; break; + case VAR_STRING: n = 1; break; + case VAR_FUNC: n = 2; break; + case VAR_LIST: n = 3; break; + case VAR_DICT: n = 4; break; + case VAR_FLOAT: n = 5; break; + case VAR_SPECIAL: { + switch (argvars[0].vval.v_special) { + case kSpecialVarTrue: + case kSpecialVarFalse: { + n = 6; + break; + } + case kSpecialVarNone: + case kSpecialVarNull: { + n = 7; + break; + } + } + break; + } + case VAR_UNKNOWN: EMSG2(_(e_intern2), "f_type()"); n = 0; break; } rettv->vval.v_number = n; } @@ -17226,6 +17302,12 @@ void set_vim_var_nr(int idx, long val) vimvars[idx].vv_nr = val; } +/// Set special v: variable to "val" +void set_vim_var_special(const int idx, const SpecialVarValue val) +{ + vimvars[idx].vv_special = val; +} + /* * Get number v: variable value. */ @@ -17574,7 +17656,7 @@ handle_subscript ( void free_tv(typval_T *varp) { if (varp != NULL) { - switch ((VarType) varp->v_type) { + switch (varp->v_type) { case VAR_FUNC: func_unref(varp->vval.v_string); /*FALLTHROUGH*/ @@ -17592,9 +17674,6 @@ void free_tv(typval_T *varp) case VAR_FLOAT: case VAR_UNKNOWN: break; - default: - EMSG2(_(e_intern2), "free_tv()"); - break; } xfree(varp); } @@ -17632,10 +17711,11 @@ void clear_tv(typval_T *varp) case VAR_FLOAT: varp->vval.v_float = 0.0; break; + case VAR_SPECIAL: + varp->vval.v_special = kSpecialVarFalse; + break; case VAR_UNKNOWN: break; - default: - EMSG2(_(e_intern2), "clear_tv()"); } varp->v_lock = 0; } @@ -17690,7 +17770,19 @@ long get_tv_number_chk(typval_T *varp, int *denote) case VAR_DICT: EMSG(_("E728: Using a Dictionary as a Number")); break; - default: + case VAR_SPECIAL: + switch (varp->vval.v_special) { + case kSpecialVarTrue: { + return 1; + } + case kSpecialVarFalse: + case kSpecialVarNone: + case kSpecialVarNull: { + return 0; + } + } + break; + case VAR_UNKNOWN: EMSG2(_(e_intern2), "get_tv_number()"); break; } @@ -17796,7 +17888,10 @@ static char_u *get_tv_string_buf_chk(const typval_T *varp, char_u *buf) if (varp->vval.v_string != NULL) return varp->vval.v_string; return (char_u *)""; - default: + case VAR_SPECIAL: + STRCPY(buf, encode_special_var_names[varp->vval.v_special]); + return buf; + case VAR_UNKNOWN: EMSG2(_(e_intern2), "get_tv_string_buf()"); break; } @@ -18345,42 +18440,34 @@ void copy_tv(typval_T *from, typval_T *to) { to->v_type = from->v_type; to->v_lock = 0; + memmove(&to->vval, &from->vval, sizeof(to->vval)); switch (from->v_type) { - case VAR_NUMBER: - to->vval.v_number = from->vval.v_number; - break; - case VAR_FLOAT: - to->vval.v_float = from->vval.v_float; - break; - case VAR_STRING: - case VAR_FUNC: - if (from->vval.v_string == NULL) - to->vval.v_string = NULL; - else { - to->vval.v_string = vim_strsave(from->vval.v_string); - if (from->v_type == VAR_FUNC) - func_ref(to->vval.v_string); - } - break; - case VAR_LIST: - if (from->vval.v_list == NULL) - to->vval.v_list = NULL; - else { - to->vval.v_list = from->vval.v_list; - ++to->vval.v_list->lv_refcount; - } - break; - case VAR_DICT: - if (from->vval.v_dict == NULL) - to->vval.v_dict = NULL; - else { - to->vval.v_dict = from->vval.v_dict; - ++to->vval.v_dict->dv_refcount; - } - break; - default: - EMSG2(_(e_intern2), "copy_tv()"); - break; + case VAR_NUMBER: + case VAR_FLOAT: + case VAR_SPECIAL: + break; + case VAR_STRING: + case VAR_FUNC: + if (from->vval.v_string != NULL) { + to->vval.v_string = vim_strsave(from->vval.v_string); + if (from->v_type == VAR_FUNC) { + func_ref(to->vval.v_string); + } + } + break; + case VAR_LIST: + if (from->vval.v_list != NULL) { + to->vval.v_list->lv_refcount++; + } + break; + case VAR_DICT: + if (from->vval.v_dict != NULL) { + to->vval.v_dict->dv_refcount++; + } + break; + case VAR_UNKNOWN: + EMSG2(_(e_intern2), "copy_tv()"); + break; } } @@ -18420,6 +18507,7 @@ int var_item_copy(const vimconv_T *const conv, case VAR_NUMBER: case VAR_FLOAT: case VAR_FUNC: + case VAR_SPECIAL: copy_tv(from, to); break; case VAR_STRING: @@ -18466,7 +18554,7 @@ int var_item_copy(const vimconv_T *const conv, if (to->vval.v_dict == NULL) ret = FAIL; break; - default: + case VAR_UNKNOWN: EMSG2(_(e_intern2), "var_item_copy()"); ret = FAIL; } @@ -21705,4 +21793,3 @@ static bool is_watched(dict_T *d) { return d && !QUEUE_EMPTY(&d->watchers); } - diff --git a/src/nvim/eval.h b/src/nvim/eval.h index e8b5964775..89aa263434 100644 --- a/src/nvim/eval.h +++ b/src/nvim/eval.h @@ -15,12 +15,6 @@ // All user-defined functions are found in this hashtable. extern hashtab_T func_hashtab; -/// CopyID for recursively traversing lists and dicts -/// -/// This value is needed to avoid endless recursiveness. Last bit is used for -/// previous_funccal and normally ignored when comparing. -extern int current_copyID; - // Structure to hold info for a user function. typedef struct ufunc ufunc_T; @@ -127,7 +121,11 @@ enum { VV_ERRORS, VV_MSGPACK_TYPES, VV_EVENT, - VV_LEN, // number of v: vars + VV_FALSE, + VV_TRUE, + VV_NULL, + VV_NONE, + VV_LEN, ///< Number of v: variables }; /// All recognized msgpack types diff --git a/src/nvim/eval_defs.h b/src/nvim/eval_defs.h index 56833f97d8..bcd9e80f9a 100644 --- a/src/nvim/eval_defs.h +++ b/src/nvim/eval_defs.h @@ -18,12 +18,32 @@ typedef struct dictvar_S dict_T; /// Special variable values typedef enum { - kSpecialVarNull, ///< v:null - kSpecialVarNone, ///< v:none kSpecialVarFalse, ///< v:false kSpecialVarTrue, ///< v:true + kSpecialVarNone, ///< v:none + kSpecialVarNull, ///< v:null } SpecialVarValue; +/// Variable lock status for typval_T.v_lock +typedef enum { + VAR_UNLOCKED = 0, ///< Not locked. + VAR_LOCKED, ///< User lock, can be unlocked. + VAR_FIXED, ///< Locked forever. +} VarLockStatus; + +/// VimL variable types, for use in typval_T.v_type +typedef enum { + VAR_UNKNOWN = 0, ///< Unknown (unspecified) value. + VAR_NUMBER, ///< Number, .v_number is used. + VAR_STRING, ///< String, .v_string is used. + VAR_FUNC, ///< Function referene, .v_string is used for function name. + VAR_LIST, ///< List, .v_list is used. + VAR_DICT, ///< Dictionary, .v_dict is used. + VAR_FLOAT, ///< Floating-point value, .v_float is used. + VAR_SPECIAL, ///< Special value (true, false, null, none), .v_special + ///< is used. +} VarType; + /// Structure that holds an internal variable value typedef struct { VarType v_type; ///< Variable type. @@ -38,34 +58,11 @@ typedef struct { } vval; ///< Actual value. } typval_T; -/// VimL variable types, for use in typval_T.v_type -/// -/// @warning Numbers are part of the user API (returned by type()), so they must -/// not be changed. -typedef enum { - VAR_UNKNOWN = 0, ///< Unknown (unspecified) value. - VAR_NUMBER = 1, ///< Number, .v_number is used. - VAR_STRING = 2, ///< String, .v_string is used. - VAR_FUNC = 3, ///< Function referene, .v_string is used for function name. - VAR_LIST = 4, ///< List, .v_list is used. - VAR_DICT = 5, ///< Dictionary, .v_dict is used. - VAR_FLOAT = 6, ///< Floating-point value, .v_float is used. - VAR_SPECIAL = 7, ///< Special value (true, false, null, none), .v_special - ///< is used. -} VarType; - /* Values for "dv_scope". */ #define VAR_SCOPE 1 /* a:, v:, s:, etc. scope dictionaries */ #define VAR_DEF_SCOPE 2 /* l:, g: scope dictionaries: here funcrefs are not allowed to mask existing functions */ -/// Variable lock status for typval_T.v_lock -typedef enum { - VAR_UNLOCKED = 0, ///< Not locked. - VAR_LOCKED, ///< User lock, can be unlocked. - VAR_FIXED, ///< Locked forever. -} VarLockStatus; - /* * Structure to hold an item of a list: an internal variable without a name. */ diff --git a/src/nvim/version.c b/src/nvim/version.c index 8f45620570..9ccd53f76b 100644 --- a/src/nvim/version.c +++ b/src/nvim/version.c @@ -195,18 +195,18 @@ static int included_patches[] = { // 1170 NA // 1169 NA 1168, - // 1167, - // 1166, + 1167, + 1166, // 1165 NA - // 1164, - // 1163, + 1164, + 1163, // 1162 NA // 1161, - // 1160, + 1160, // 1159 NA // 1158 NA - // 1157, - // 1156, + 1157, + // 1156 NA // 1155 NA // 1154, // 1153, diff --git a/test/functional/eval/msgpack_functions_spec.lua b/test/functional/eval/msgpack_functions_spec.lua index fc0aad7902..0c91471656 100644 --- a/test/functional/eval/msgpack_functions_spec.lua +++ b/test/functional/eval/msgpack_functions_spec.lua @@ -1,5 +1,6 @@ local helpers = require('test.functional.helpers') local clear = helpers.clear +local funcs = helpers.funcs local eval, eq = helpers.eval, helpers.eq local execute = helpers.execute local nvim = helpers.nvim @@ -517,6 +518,19 @@ describe('msgpackdump() function', function() eq({'\129\128\128'}, eval('msgpackdump([todump])')) end) + it('can dump v:true', function() + eq({'\195'}, funcs.msgpackdump({true})) + end) + + it('can dump v:false', function() + eq({'\194'}, funcs.msgpackdump({false})) + end) + + it('can v:null', function() + execute('let todump = v:null') + eq({'\192'}, eval('msgpackdump([todump])')) + end) + it('can dump special ext mapping', function() execute('let todump = {"_TYPE": v:msgpack_types.ext, "_VAL": [5, ["",""]]}') eq({'\212\005', ''}, eval('msgpackdump([todump])')) @@ -620,6 +634,11 @@ describe('msgpackdump() function', function() exc_exec('call msgpackdump([todump])')) end) + it('fails to dump v:none', function() + eq('Vim(call):E953: Attempt to convert v:none in msgpackdump() argument, index 0, itself', + exc_exec('call msgpackdump([v:none])')) + end) + it('fails when called with no arguments', function() eq('Vim(call):E119: Not enough arguments for function: msgpackdump', exc_exec('call msgpackdump()')) @@ -654,4 +673,11 @@ describe('msgpackdump() function', function() eq('Vim(call):E686: Argument of msgpackdump() must be a List', exc_exec('call msgpackdump(0.0)')) end) + + it('fails to dump special value', function() + for _, val in ipairs({'v:true', 'v:false', 'v:null', 'v:none'}) do + eq('Vim(call):E686: Argument of msgpackdump() must be a List', + exc_exec('call msgpackdump(' .. val .. ')')) + end + end) end) diff --git a/test/functional/eval/special_vars_spec.lua b/test/functional/eval/special_vars_spec.lua index 7adc529c88..c7df847946 100644 --- a/test/functional/eval/special_vars_spec.lua +++ b/test/functional/eval/special_vars_spec.lua @@ -1,8 +1,12 @@ local helpers = require('test.functional.helpers') +local exc_exec = helpers.exc_exec local execute = helpers.execute +local meths = helpers.meths local funcs = helpers.funcs +local meths = helpers.meths local clear = helpers.clear local eval = helpers.eval +local eq = helpers.eq describe('Special values', function() before_each(clear) @@ -17,20 +21,166 @@ describe('Special values', function() endtry endfunction ]]) - eq(true, funcs.Test()) + eq(0, exc_exec('call Test()')) end) it('work with empty()', function() eq(0, funcs.empty(true)) eq(1, funcs.empty(false)) - eq(1, funcs.empty(nil)) + eq(1, eval('empty(v:null)')) eq(1, eval('empty(v:none)')) end) it('can be stringified and eval’ed back', function() eq(true, funcs.eval(funcs.string(true))) eq(false, funcs.eval(funcs.string(false))) - eq(nil, funcs.eval(funcs.string(nil))) + eq(nil, eval('eval(string(v:null))')) eq(1, eval('eval(string(v:none)) is# v:none')) end) + + it('work with is/isnot properly', function() + eq(1, eval('v:none is v:none')) + eq(0, eval('v:none is v:null')) + eq(0, eval('v:none is v:true')) + eq(0, eval('v:none is v:false')) + eq(1, eval('v:null is v:null')) + eq(0, eval('v:null is v:true')) + eq(0, eval('v:null is v:false')) + eq(1, eval('v:true is v:true')) + eq(0, eval('v:true is v:false')) + eq(1, eval('v:false is v:false')) + + eq(0, eval('v:none is 0')) + eq(0, eval('v:null is 0')) + eq(0, eval('v:true is 0')) + eq(0, eval('v:false is 0')) + + eq(0, eval('v:none is 1')) + eq(0, eval('v:null is 1')) + eq(0, eval('v:true is 1')) + eq(0, eval('v:false is 1')) + + eq(0, eval('v:none is ""')) + eq(0, eval('v:null is ""')) + eq(0, eval('v:true is ""')) + eq(0, eval('v:false is ""')) + + eq(0, eval('v:none is "none"')) + eq(0, eval('v:null is "null"')) + eq(0, eval('v:true is "true"')) + eq(0, eval('v:false is "false"')) + + eq(0, eval('v:none is []')) + eq(0, eval('v:null is []')) + eq(0, eval('v:true is []')) + eq(0, eval('v:false is []')) + + eq(0, eval('v:none isnot v:none')) + eq(1, eval('v:none isnot v:null')) + eq(1, eval('v:none isnot v:true')) + eq(1, eval('v:none isnot v:false')) + eq(0, eval('v:null isnot v:null')) + eq(1, eval('v:null isnot v:true')) + eq(1, eval('v:null isnot v:false')) + eq(0, eval('v:true isnot v:true')) + eq(1, eval('v:true isnot v:false')) + eq(0, eval('v:false isnot v:false')) + + eq(1, eval('v:none isnot 0')) + eq(1, eval('v:null isnot 0')) + eq(1, eval('v:true isnot 0')) + eq(1, eval('v:false isnot 0')) + + eq(1, eval('v:none isnot 1')) + eq(1, eval('v:null isnot 1')) + eq(1, eval('v:true isnot 1')) + eq(1, eval('v:false isnot 1')) + + eq(1, eval('v:none isnot ""')) + eq(1, eval('v:null isnot ""')) + eq(1, eval('v:true isnot ""')) + eq(1, eval('v:false isnot ""')) + + eq(1, eval('v:none isnot "none"')) + eq(1, eval('v:null isnot "null"')) + eq(1, eval('v:true isnot "true"')) + eq(1, eval('v:false isnot "false"')) + + eq(1, eval('v:none isnot []')) + eq(1, eval('v:null isnot []')) + eq(1, eval('v:true isnot []')) + eq(1, eval('v:false isnot []')) + end) + + it('work with +/-/* properly', function() + eq(1, eval('0 + v:true')) + eq(0, eval('0 + v:none')) + eq(0, eval('0 + v:null')) + eq(0, eval('0 + v:false')) + + eq(-1, eval('0 - v:true')) + eq( 0, eval('0 - v:none')) + eq( 0, eval('0 - v:null')) + eq( 0, eval('0 - v:false')) + + eq(1, eval('1 * v:true')) + eq(0, eval('1 * v:none')) + eq(0, eval('1 * v:null')) + eq(0, eval('1 * v:false')) + end) + + it('does not work with +=/-=/.=', function() + meths.set_var('true', true) + meths.set_var('false', false) + execute('let none = v:none') + execute('let null = v:null') + + eq('Vim(let):E734: Wrong variable type for +=', exc_exec('let true += 1')) + eq('Vim(let):E734: Wrong variable type for +=', exc_exec('let false += 1')) + eq('Vim(let):E734: Wrong variable type for +=', exc_exec('let none += 1')) + eq('Vim(let):E734: Wrong variable type for +=', exc_exec('let null += 1')) + + eq('Vim(let):E734: Wrong variable type for -=', exc_exec('let true -= 1')) + eq('Vim(let):E734: Wrong variable type for -=', exc_exec('let false -= 1')) + eq('Vim(let):E734: Wrong variable type for -=', exc_exec('let none -= 1')) + eq('Vim(let):E734: Wrong variable type for -=', exc_exec('let null -= 1')) + + eq('Vim(let):E734: Wrong variable type for .=', exc_exec('let true .= 1')) + eq('Vim(let):E734: Wrong variable type for .=', exc_exec('let false .= 1')) + eq('Vim(let):E734: Wrong variable type for .=', exc_exec('let none .= 1')) + eq('Vim(let):E734: Wrong variable type for .=', exc_exec('let null .= 1')) + end) + + it('work with . (concat) properly', function() + eq("true", eval('"" . v:true')) + eq("none", eval('"" . v:none')) + eq("null", eval('"" . v:null')) + eq("false", eval('"" . v:false')) + end) + + it('work with type()', function() + eq(6, funcs.type(true)) + eq(6, funcs.type(false)) + eq(7, eval('type(v:null)')) + eq(7, eval('type(v:none)')) + end) + + it('work with copy() and deepcopy()', function() + eq(true, funcs.deepcopy(true)) + eq(false, funcs.deepcopy(false)) + eq(nil, eval('deepcopy(v:null)')) + eq(nil, eval('deepcopy(v:none)')) + + eq(true, funcs.copy(true)) + eq(false, funcs.copy(false)) + eq(nil, eval('copy(v:null)')) + eq(nil, eval('copy(v:none)')) + end) + + it('fails in index', function() + eq('Vim(echo):E15: Cannot index a special value', exc_exec('echo v:true[0]')) + eq('Vim(echo):E15: Cannot index a special value', exc_exec('echo v:false[0]')) + eq('Vim(echo):E15: Cannot index a special value', exc_exec('echo v:none[0]')) + eq('Vim(echo):E15: Cannot index a special value', exc_exec('echo v:null[0]')) + end) end) diff --git a/test/functional/eval/string_spec.lua b/test/functional/eval/string_spec.lua index f7f5dca70a..fe79708910 100644 --- a/test/functional/eval/string_spec.lua +++ b/test/functional/eval/string_spec.lua @@ -28,6 +28,13 @@ describe('string() function', function() eq('0.0', eval('string(0.0)')) end) + it('dumps special v: values', function() + eq('v:true', eval('string(v:true)')) + eq('v:false', eval('string(v:false)')) + eq('v:none', eval('string(v:none)')) + eq('v:null', eval('string(v:null)')) + end) + it('dumps values with at most six digits after the decimal point', function() eq('1.234568e-20', funcs.string(1.23456789123456789123456789e-020)) From 6e5498c3e32ecc7adfedc3f47b876f82de90fff8 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 31 Jan 2016 02:28:53 +0300 Subject: [PATCH 08/82] runtime/msgpack: Add support for special values --- runtime/autoload/msgpack.vim | 4 +++- test/functional/plugin/msgpack_spec.lua | 21 +++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/runtime/autoload/msgpack.vim b/runtime/autoload/msgpack.vim index 2bb7ec5b02..2e2697c57f 100644 --- a/runtime/autoload/msgpack.vim +++ b/runtime/autoload/msgpack.vim @@ -356,6 +356,8 @@ let s:MSGPACK_STANDARD_TYPES = { \type(''): 'binary', \type([]): 'array', \type({}): 'map', + \type(v:true): 'boolean', + \type(v:null): 'nil', \} "" @@ -379,7 +381,7 @@ endfunction "" " Dump boolean value. function s:msgpack_dump_boolean(v) abort - return a:v._VAL ? 'TRUE' : 'FALSE' + return (a:v is v:true || (a:v isnot v:false && a:v._VAL)) ? 'TRUE' : 'FALSE' endfunction "" diff --git a/test/functional/plugin/msgpack_spec.lua b/test/functional/plugin/msgpack_spec.lua index 90cc2af9c0..197a1a92e5 100644 --- a/test/functional/plugin/msgpack_spec.lua +++ b/test/functional/plugin/msgpack_spec.lua @@ -1,4 +1,5 @@ local helpers = require('test.functional.helpers') +local meths = helpers.meths local eq, nvim_eval, nvim_command, exc_exec = helpers.eq, helpers.eval, helpers.command, helpers.exc_exec local ok = helpers.ok @@ -409,6 +410,12 @@ describe('In autoload/msgpack.vim', function() string_eq('nan', '(1.0/0.0-1.0/0.0)') string_eq('nan', '-(1.0/0.0-1.0/0.0)') end) + + it('works for special v: values like v:true', function() + string_eq('TRUE', 'v:true') + string_eq('FALSE', 'v:false') + string_eq('NIL', 'v:null') + end) end) describe('function msgpack#deepcopy', function() @@ -523,6 +530,20 @@ describe('In autoload/msgpack.vim', function() eq(2.0, nvim_eval('flt2')) eq('abc', nvim_eval('bin2')) end) + + it('works for special v: values like v:true', function() + meths.set_var('true', true) + meths.set_var('false', false) + nvim_command('let nil = v:null') + + nvim_command('let true2 = msgpack#deepcopy(true)') + nvim_command('let false2 = msgpack#deepcopy(false)') + nvim_command('let nil2 = msgpack#deepcopy(nil)') + + eq(true, meths.get_var('true')) + eq(false, meths.get_var('false')) + eq(nil, meths.get_var('nil')) + end) end) describe('function msgpack#eval', function() From f5c35ba10988a5c75367e028516b4287b6eaa8f5 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 31 Jan 2016 02:32:46 +0300 Subject: [PATCH 09/82] functests/msgpack: Test dumping special nil and bool dicts --- test/functional/eval/msgpack_functions_spec.lua | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/test/functional/eval/msgpack_functions_spec.lua b/test/functional/eval/msgpack_functions_spec.lua index 0c91471656..2c73a144ca 100644 --- a/test/functional/eval/msgpack_functions_spec.lua +++ b/test/functional/eval/msgpack_functions_spec.lua @@ -528,6 +528,20 @@ describe('msgpackdump() function', function() it('can v:null', function() execute('let todump = v:null') + end) + + it('can dump special bool mapping (true)', function() + execute('let todump = {"_TYPE": v:msgpack_types.boolean, "_VAL": 1}') + eq({'\195'}, eval('msgpackdump([todump])')) + end) + + it('can dump special bool mapping (false)', function() + execute('let todump = {"_TYPE": v:msgpack_types.boolean, "_VAL": 0}') + eq({'\194'}, eval('msgpackdump([todump])')) + end) + + it('can dump special nil mapping', function() + execute('let todump = {"_TYPE": v:msgpack_types.nil, "_VAL": 0}') eq({'\192'}, eval('msgpackdump([todump])')) end) From 0f0e2bdfd4b99d678cbcd090505ae9ebf74233a2 Mon Sep 17 00:00:00 2001 From: ZyX Date: Tue, 2 Feb 2016 01:32:35 +0300 Subject: [PATCH 10/82] encode: Do not use default case for v_type --- src/nvim/encode.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nvim/encode.c b/src/nvim/encode.c index c80e9783e0..2968c262ff 100644 --- a/src/nvim/encode.c +++ b/src/nvim/encode.c @@ -555,7 +555,7 @@ name##_convert_one_value_regular_dict: \ })); \ break; \ } \ - default: { \ + case VAR_UNKNOWN: { \ EMSG2(_(e_intern2), #name "_convert_one_value()"); \ return FAIL; \ } \ From e213ba150665328bae2b532491de5e12f72bc9ca Mon Sep 17 00:00:00 2001 From: ZyX Date: Mon, 1 Feb 2016 21:22:07 +0300 Subject: [PATCH 11/82] eval: Add jsondecode() function --- runtime/doc/eval.txt | 12 + runtime/doc/vim_diff.txt | 6 + src/nvim/encode.c | 11 +- src/nvim/eval.c | 535 +++++++++++++++++++ src/nvim/lib/kvec.h | 1 + src/nvim/version.c | 2 +- test/functional/eval/json_functions_spec.lua | 232 ++++++++ 7 files changed, 793 insertions(+), 6 deletions(-) diff --git a/runtime/doc/eval.txt b/runtime/doc/eval.txt index 572cf4c03f..25005885c3 100644 --- a/runtime/doc/eval.txt +++ b/runtime/doc/eval.txt @@ -1960,6 +1960,7 @@ jobstart( {cmd}[, {opts}]) Number Spawns {cmd} as a job jobstop( {job}) Number Stops a job jobwait( {ids}[, {timeout}]) Number Wait for a set of jobs join( {list} [, {sep}]) String join {list} items into one String +jsondecode( {expr}) any Convert {expr} from JSON jsonencode( {expr}) String Convert {expr} to JSON keys( {dict}) List keys in {dict} len( {expr}) Number the length of {expr} @@ -4320,6 +4321,17 @@ join({list} [, {sep}]) *join()* converted into a string like with |string()|. The opposite function is |split()|. +jsondecode({expr}) *jsondecode()* + Convert {expr} from JSON object. Accepts |readfile()|-style + list as the input, as well as regular string. May output any + Vim value. In the following cases it will output + |msgpack-special-dict|: + 1. Dictionary contains duplicate key. + 2. Dictionary contains empty key. + 3. String contains NUL byte. Two special dictionaries: for + dictionary and for string will be emitted in case string + with NUL byte was a dictionary key. + jsonencode({expr}) *jsonencode()* Convert {expr} into a JSON string. Accepts |msgpack-special-dict| as the input. Will not convert diff --git a/runtime/doc/vim_diff.txt b/runtime/doc/vim_diff.txt index 17ee5975dd..ddb1fd3b8f 100644 --- a/runtime/doc/vim_diff.txt +++ b/runtime/doc/vim_diff.txt @@ -100,6 +100,12 @@ are always available and may be used simultaneously in separate plugins. The 4. Stringifyed infinite and NaN values now use |str2float()| and can be evaled back. +|jsondecode()| behaviour changed: +1. It may output |msgpack-special-dict|. +2. It accepts only valid JSON. |v:none| is never emitted. +|jsonencode()| behaviour slightly changed: now |msgpack-special-dict| values +are accepted. + Viminfo text files were replaced with binary (messagepack) ShaDa files. Additional differences: diff --git a/src/nvim/encode.c b/src/nvim/encode.c index 2968c262ff..57163253c3 100644 --- a/src/nvim/encode.c +++ b/src/nvim/encode.c @@ -200,8 +200,8 @@ static int conv_error(const char *const msg, const MPConvStack *const mpstack, /// zero. /// /// @return true in case of success, false in case of failure. -static inline bool vim_list_to_buf(const list_T *const list, - size_t *const ret_len, char **const ret_buf) +bool encode_vim_list_to_buf(const list_T *const list, size_t *const ret_len, + char **const ret_buf) FUNC_ATTR_NONNULL_ARG(2,3) FUNC_ATTR_WARN_UNUSED_RESULT { size_t len = 0; @@ -457,7 +457,8 @@ static int name##_convert_one_value(firstargtype firstargname, \ } \ size_t len; \ char *buf; \ - if (!vim_list_to_buf(val_di->di_tv.vval.v_list, &len, &buf)) { \ + if (!encode_vim_list_to_buf(val_di->di_tv.vval.v_list, &len, \ + &buf)) { \ goto name##_convert_one_value_regular_dict; \ } \ if (is_string) { \ @@ -529,8 +530,8 @@ static int name##_convert_one_value(firstargtype firstargname, \ } \ size_t len; \ char *buf; \ - if (!vim_list_to_buf(val_list->lv_last->li_tv.vval.v_list, \ - &len, &buf)) { \ + if (!encode_vim_list_to_buf(val_list->lv_last->li_tv.vval.v_list, \ + &len, &buf)) { \ goto name##_convert_one_value_regular_dict; \ } \ CONV_EXT_STRING(buf, len, type); \ diff --git a/src/nvim/eval.c b/src/nvim/eval.c index 819b3059e2..cfbbb6f93f 100644 --- a/src/nvim/eval.c +++ b/src/nvim/eval.c @@ -89,6 +89,7 @@ #include "nvim/os/input.h" #include "nvim/event/loop.h" #include "nvim/lib/queue.h" +#include "nvim/lib/kvec.h" #define DICT_MAXNEST 100 /* maximum nesting of lists and dicts */ @@ -415,6 +416,16 @@ typedef struct { int status; } JobEvent; +/// Helper structure for container_struct +typedef struct { + size_t stack_index; ///< Index of current container in stack. + typval_T container; ///< Container. Either VAR_LIST, VAR_DICT or VAR_LIST + ///< which is _VAL from special dictionary. +} ContainerStackItem; + +typedef kvec_t(typval_T) ValuesStack; +typedef kvec_t(ContainerStackItem) ContainerStack; + #ifdef INCLUDE_GENERATED_DECLARATIONS # include "eval.c.generated.h" #endif @@ -6766,6 +6777,7 @@ static struct fst { { "jobstop", 1, 1, f_jobstop }, { "jobwait", 1, 2, f_jobwait }, { "join", 1, 2, f_join }, + { "jsondecode", 1, 1, f_jsondecode }, { "jsonencode", 1, 1, f_jsonencode }, { "keys", 1, 1, f_keys }, { "last_buffer_nr", 0, 0, f_last_buffer_nr }, // obsolete @@ -11559,6 +11571,529 @@ static void f_join(typval_T *argvars, typval_T *rettv) rettv->vval.v_string = NULL; } +/// Helper function used for working with stack vectors used by JSON decoder +/// +/// @param[in] obj New object. +/// @param[out] stack Object stack. +/// @param[out] container_stack Container objects stack. +/// @param[in] p Position in string which is currently being parsed. +/// +/// @return OK in case of success, FAIL in case of error. +static inline int json_decoder_pop(typval_T obj, ValuesStack *const stack, + ContainerStack *const container_stack, + const char *const p) + FUNC_ATTR_NONNULL_ALL +{ + if (kv_size(*container_stack) == 0) { + kv_push(typval_T, *stack, obj); + return OK; + } + ContainerStackItem last_container = kv_last(*container_stack); + if (obj.v_type == last_container.container.v_type + // vval.v_list and vval.v_dict should have the same size and offset + && ((void *) obj.vval.v_list + == (void *) last_container.container.vval.v_list)) { + kv_pop(*container_stack); + last_container = kv_last(*container_stack); + } + if (last_container.container.v_type == VAR_LIST) { + listitem_T *obj_li = listitem_alloc(); + obj_li->li_tv = obj; + list_append(last_container.container.vval.v_list, obj_li); + } else if (last_container.stack_index == kv_size(*stack) - 2) { + typval_T key = kv_pop(*stack); + if (key.v_type != VAR_STRING) { + assert(false); + } else if (key.vval.v_string == NULL || *key.vval.v_string == NUL) { + // TODO: fall back to special dict in case of empty key + EMSG(_("E474: Empty key")); + clear_tv(&obj); + return FAIL; + } + dictitem_T *obj_di = dictitem_alloc(key.vval.v_string); + clear_tv(&key); + if (dict_add(last_container.container.vval.v_dict, obj_di) + == FAIL) { + // TODO: fall back to special dict in case of duplicate keys + EMSG(_("E474: Duplicate key")); + dictitem_free(obj_di); + clear_tv(&obj); + return FAIL; + } + obj_di->di_tv = obj; + } else { + // Object with key only + if (obj.v_type != VAR_STRING) { + EMSG2(_("E474: Expected string key: %s"), p); + clear_tv(&obj); + return FAIL; + } + kv_push(typval_T, *stack, obj); + } + return OK; +} + +/// Convert JSON string into VimL object +/// +/// @param[in] buf String to convert. UTF-8 encoding is assumed. +/// @param[in] len Length of the string. +/// @param[out] rettv Location where to save results. +/// +/// @return OK in case of success, FAIL otherwise. +static int json_decode_string(const char *const buf, const size_t len, + typval_T *rettv) + FUNC_ATTR_NONNULL_ALL FUNC_ATTR_WARN_UNUSED_RESULT +{ + vimconv_T conv; + convert_setup(&conv, (char_u *) "utf-8", p_enc); + conv.vc_fail = true; + int ret = OK; + ValuesStack stack; + kv_init(stack); + ContainerStack container_stack; + kv_init(container_stack); + rettv->v_type = VAR_UNKNOWN; + const char *const e = buf + len; + bool didcomma = false; + bool didcolon = false; +#define POP(obj) \ + do { \ + if (json_decoder_pop(obj, &stack, &container_stack, p) == FAIL) { \ + goto json_decode_string_fail; \ + } \ + } while (0) + const char *p = buf; + for (; p < e; p++) { + switch (*p) { + case '}': + case ']': { + if (kv_size(container_stack) == 0) { + EMSG2(_("E474: No container to close: %s"), p); + goto json_decode_string_fail; + } + ContainerStackItem last_container = kv_last(container_stack); + if (*p == '}' && last_container.container.v_type != VAR_DICT) { + EMSG2(_("E474: Closing list with figure brace: %s"), p); + goto json_decode_string_fail; + } else if (*p == ']' && last_container.container.v_type != VAR_LIST) { + EMSG2(_("E474: Closing dictionary with bracket: %s"), p); + goto json_decode_string_fail; + } else if (didcomma) { + EMSG2(_("E474: Trailing comma: %s"), p); + goto json_decode_string_fail; + } else if (didcolon) { + EMSG2(_("E474: Expected value after colon: %s"), p); + goto json_decode_string_fail; + } else if (last_container.stack_index != kv_size(stack) - 1) { + assert(last_container.stack_index < kv_size(stack) - 1); + EMSG2(_("E474: Expected value: %s"), p); + goto json_decode_string_fail; + } + if (kv_size(stack) == 1) { + p++; + kv_pop(container_stack); + goto json_decode_string_after_cycle; + } else { + typval_T obj = kv_pop(stack); + POP(obj); + break; + } + } + case ',': { + if (kv_size(container_stack) == 0) { + EMSG2(_("E474: Comma not inside container: %s"), p); + goto json_decode_string_fail; + } + ContainerStackItem last_container = kv_last(container_stack); + if (didcomma) { + EMSG2(_("E474: Duplicate comma: %s"), p); + goto json_decode_string_fail; + } else if (didcolon) { + EMSG2(_("E474: Comma after colon: %s"), p); + goto json_decode_string_fail; + } if (last_container.container.v_type == VAR_DICT + && last_container.stack_index != kv_size(stack) - 1) { + EMSG2(_("E474: Using comma in place of colon: %s"), p); + goto json_decode_string_fail; + } else if ((last_container.container.v_type == VAR_DICT + && (last_container.container.vval.v_dict->dv_hashtab.ht_used + == 0)) + || (last_container.container.v_type == VAR_LIST + && last_container.container.vval.v_list->lv_len == 0)) { + EMSG2(_("E474: Leading comma: %s"), p); + goto json_decode_string_fail; + } + didcomma = true; + continue; + } + case ':': { + if (kv_size(container_stack) == 0) { + EMSG2(_("E474: Colon not inside container: %s"), p); + goto json_decode_string_fail; + } + ContainerStackItem last_container = kv_last(container_stack); + if (last_container.container.v_type != VAR_DICT) { + EMSG2(_("E474: Using colon not in dictionary: %s"), p); + goto json_decode_string_fail; + } else if (last_container.stack_index != kv_size(stack) - 2) { + EMSG2(_("E474: Unexpected colon: %s"), p); + goto json_decode_string_fail; + } else if (didcomma) { + EMSG2(_("E474: Colon after comma: %s"), p); + goto json_decode_string_fail; + } else if (didcolon) { + EMSG2(_("E474: Duplicate colon: %s"), p); + goto json_decode_string_fail; + } + didcolon = true; + continue; + } + case ' ': + case TAB: + case NL: { + continue; + } + case 'n': { + if (strncmp(p + 1, "ull", 3) != 0) { + EMSG2(_("E474: Expected null: %s"), p); + goto json_decode_string_fail; + } + p += 3; + POP(vimvars[VV_NULL].vv_di.di_tv); + break; + } + case 't': { + if (strncmp(p + 1, "rue", 3) != 0) { + EMSG2(_("E474: Expected true: %s"), p); + goto json_decode_string_fail; + } + p += 3; + POP(vimvars[VV_TRUE].vv_di.di_tv); + break; + } + case 'f': { + if (strncmp(p + 1, "alse", 4) != 0) { + EMSG2(_("E474: Expected false: %s"), p); + goto json_decode_string_fail; + } + p += 4; + POP(vimvars[VV_FALSE].vv_di.di_tv); + break; + } + case '"': { + size_t len = 0; + const char *s; + for (s = ++p; p < e && *p != '"'; p++) { + if (*p == '\\') { + p++; + if (p == e) { + EMSG2(_("E474: Unfinished escape sequence: %s"), buf); + goto json_decode_string_fail; + } + switch (*p) { + case 'u': { + if (p + 4 >= e) { + EMSG2(_("E474: Unfinished unicode escape sequence: %s"), buf); + goto json_decode_string_fail; + } else if (!ascii_isxdigit(p[1]) + || !ascii_isxdigit(p[2]) + || !ascii_isxdigit(p[3]) + || !ascii_isxdigit(p[4])) { + EMSG2(_("E474: Expected four hex digits after \\u: %s"), + p - 1); + goto json_decode_string_fail; + } + // One UTF-8 character below U+10000 can take up to 3 bytes + len += 3; + p += 4; + break; + } + case '\\': + case '/': + case '"': + case 't': + case 'b': + case 'n': + case 'r': + case 'f': { + len++; + break; + } + default: { + EMSG2(_("E474: Unknown escape sequence: %s"), p - 1); + goto json_decode_string_fail; + } + } + } else { + len++; + } + } + if (*p != '"') { + EMSG2(_("E474: Expected string end: %s"), buf); + goto json_decode_string_fail; + } + char *str = xmalloc(len + 1); + uint16_t fst_in_pair = 0; + char *str_end = str; + for (const char *t = s; t < p; t++) { + if (t[0] != '\\' || t[1] != 'u') { + if (fst_in_pair != 0) { + str_end += utf_char2bytes((int) fst_in_pair, (char_u *) str_end); + fst_in_pair = 0; + } + } + if (*t == '\\') { + t++; + switch (*t) { + case 'u': { + char ubuf[] = { t[1], t[2], t[3], t[4], 0 }; + t += 4; + unsigned long ch; + vim_str2nr((char_u *) ubuf, NULL, NULL, 0, 0, 2, NULL, &ch); + if (0xD800UL <= ch && ch <= 0xDB7FUL) { + fst_in_pair = (uint16_t) ch; + } else if (0xDC00ULL <= ch && ch <= 0xDB7FUL) { + if (fst_in_pair != 0) { + int full_char = ( + (int) (ch - 0xDC00UL) + + (((int) (fst_in_pair - 0xD800)) << 10) + ); + str_end += utf_char2bytes(full_char, (char_u *) str_end); + } + } else { + str_end += utf_char2bytes((int) ch, (char_u *) str_end); + } + break; + } + case '\\': + case '/': + case '"': + case 't': + case 'b': + case 'n': + case 'r': + case 'f': { + static const char escapes[] = { + ['\\'] = '\\', + ['/'] = '/', + ['"'] = '"', + ['t'] = TAB, + ['b'] = BS, + ['n'] = NL, + ['r'] = CAR, + ['f'] = FF, + }; + *str_end++ = escapes[(int) *t]; + break; + } + default: { + assert(false); + } + } + } else { + *str_end++ = *t; + } + } + if (fst_in_pair != 0) { + str_end += utf_char2bytes((int) fst_in_pair, (char_u *) str_end); + } + if (conv.vc_type != CONV_NONE) { + size_t len = (str_end - str); + char *const new_str = (char *) string_convert(&conv, (char_u *) str, + &len); + if (new_str == NULL) { + EMSG2(_("E474: Failed to convert string \"%s\" from UTF-8"), str); + xfree(str); + goto json_decode_string_fail; + } + xfree(str); + str = new_str; + str_end = new_str + len; + } + *str_end = NUL; + // TODO: return special string in case of NUL bytes + POP(((typval_T) { + .v_type = VAR_STRING, + .vval = { .v_string = (char_u *) str, }, + })); + break; + } + case '-': + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': { + // a.bE[+-]exp + const char *const s = p; + const char *ints = NULL; + const char *fracs = NULL; + const char *exps = NULL; + if (*p == '-') { + p++; + } + ints = p; + while (p < e && ascii_isdigit(*p)) { + p++; + } + if (p < e && *p == '.') { + p++; + fracs = p; + while (p < e && ascii_isdigit(*p)) { + p++; + } + if (p < e && (*p == 'e' || *p == 'E')) { + p++; + if (p < e && (*p == '-' || *p == '+')) { + p++; + } + exps = p; + while (p < e && ascii_isdigit(*p)) { + p++; + } + } + } + if (p == ints) { + EMSG2(_("E474: Missing number after minus sign: %s"), s); + goto json_decode_string_fail; + } else if (p == fracs) { + EMSG2(_("E474: Missing number after decimal dot: %s"), s); + goto json_decode_string_fail; + } else if (p == exps) { + EMSG2(_("E474: Missing exponent: %s"), s); + goto json_decode_string_fail; + } + typval_T tv = { + .v_type = VAR_NUMBER, + .v_lock = VAR_UNLOCKED, + }; + if (fracs) { + // Convert floating-point number + (void) string2float((char_u *) s, &tv.vval.v_float); + tv.v_type = VAR_FLOAT; + } else { + // Convert integer + long nr; + vim_str2nr((char_u *) s, NULL, NULL, 0, 0, 0, &nr, NULL); + tv.vval.v_number = (varnumber_T) nr; + } + POP(tv); + p--; + break; + } + case '[': { + list_T *list = list_alloc(); + list->lv_refcount++; + typval_T tv = { + .v_type = VAR_LIST, + .v_lock = VAR_UNLOCKED, + .vval = { .v_list = list }, + }; + kv_push(ContainerStackItem, container_stack, ((ContainerStackItem) { + .stack_index = kv_size(stack), + .container = tv, + })); + kv_push(typval_T, stack, tv); + break; + } + case '{': { + dict_T *dict = dict_alloc(); + dict->dv_refcount++; + typval_T tv = { + .v_type = VAR_DICT, + .v_lock = VAR_UNLOCKED, + .vval = { .v_dict = dict }, + }; + kv_push(ContainerStackItem, container_stack, ((ContainerStackItem) { + .stack_index = kv_size(stack), + .container = tv, + })); + kv_push(typval_T, stack, tv); + break; + } + default: { + EMSG2(_("E474: Unidentified byte: %s"), p); + goto json_decode_string_fail; + } + } + didcomma = false; + didcolon = false; + if (kv_size(container_stack) == 0) { + p++; + break; + } + } +#undef POP +json_decode_string_after_cycle: + for (; p < e; p++) { + switch (*p) { + case NL: + case ' ': + case TAB: { + break; + } + default: { + EMSG2(_("E474: Trailing characters: %s"), p); + goto json_decode_string_fail; + } + } + } + if (kv_size(stack) > 1 || kv_size(container_stack)) { + EMSG2(_("E474: Unexpected end of input: %s"), buf); + goto json_decode_string_fail; + } + goto json_decode_string_ret; +json_decode_string_fail: + ret = FAIL; + while (kv_size(stack)) { + clear_tv(&kv_pop(stack)); + } +json_decode_string_ret: + if (ret != FAIL) { + assert(kv_size(stack) == 1); + *rettv = kv_pop(stack); + } + kv_destroy(stack); + kv_destroy(container_stack); + return ret; +} + +/// jsondecode() function +static void f_jsondecode(typval_T *argvars, typval_T *rettv) +{ + char numbuf[NUMBUFLEN]; + char *s = NULL; + char *tofree = NULL; + size_t len; + if (argvars[0].v_type == VAR_LIST) { + if (!encode_vim_list_to_buf(argvars[0].vval.v_list, &len, &s)) { + EMSG(_("E474: Failed to convert list to string")); + return; + } + tofree = s; + } else { + s = (char *) get_tv_string_buf_chk(&argvars[0], (char_u *) numbuf); + if (s) { + len = strlen(s); + } + } + if (s == NULL) { + return; + } + if (json_decode_string(s, len, rettv) == FAIL) { + EMSG2(_("E474: Failed to parse %s"), s); + rettv->v_type = VAR_NUMBER; + rettv->vval.v_number = 0; + } + assert(rettv->v_type != VAR_UNKNOWN); + xfree(tofree); +} + /// jsonencode() function static void f_jsonencode(typval_T *argvars, typval_T *rettv) { diff --git a/src/nvim/lib/kvec.h b/src/nvim/lib/kvec.h index 53ecf232c6..b41ef0cc9f 100644 --- a/src/nvim/lib/kvec.h +++ b/src/nvim/lib/kvec.h @@ -60,6 +60,7 @@ int main() { #define kv_pop(v) ((v).items[--(v).size]) #define kv_size(v) ((v).size) #define kv_max(v) ((v).capacity) +#define kv_last(v) kv_A(v, kv_size(v) - 1) #define kv_resize(type, v, s) ((v).capacity = (s), (v).items = (type*)xrealloc((v).items, sizeof(type) * (v).capacity)) diff --git a/src/nvim/version.c b/src/nvim/version.c index 9ccd53f76b..262adcba34 100644 --- a/src/nvim/version.c +++ b/src/nvim/version.c @@ -134,7 +134,7 @@ static int included_patches[] = { // 1231 // 1230 // 1229 - // 1228 + 1228, // 1227 // 1226 // 1225 diff --git a/test/functional/eval/json_functions_spec.lua b/test/functional/eval/json_functions_spec.lua index 5e1c6a3984..f979a6dd7c 100644 --- a/test/functional/eval/json_functions_spec.lua +++ b/test/functional/eval/json_functions_spec.lua @@ -6,6 +6,238 @@ local eval = helpers.eval local execute = helpers.execute local exc_exec = helpers.exc_exec +describe('jsondecode() function', function() + before_each(clear) + + it('accepts readfile()-style list', function() + eq({Test=1}, funcs.jsondecode({ + '{', + '\t"Test": 1', + '}', + })) + end) + + it('accepts strings with newlines', function() + eq({Test=1}, funcs.jsondecode([[ + { + "Test": 1 + } + ]])) + end) + + it('parses null, true, false', function() + eq(nil, funcs.jsondecode('null')) + eq(true, funcs.jsondecode('true')) + eq(false, funcs.jsondecode('false')) + end) + + it('fails to parse incomplete null, true, false', function() + eq('Vim(call):E474: Expected null: n', + exc_exec('call jsondecode("n")')) + eq('Vim(call):E474: Expected null: nu', + exc_exec('call jsondecode("nu")')) + eq('Vim(call):E474: Expected null: nul', + exc_exec('call jsondecode("nul")')) + eq('Vim(call):E474: Expected null: nul\n\t', + exc_exec('call jsondecode("nul\\n\\t")')) + + eq('Vim(call):E474: Expected true: t', + exc_exec('call jsondecode("t")')) + eq('Vim(call):E474: Expected true: tr', + exc_exec('call jsondecode("tr")')) + eq('Vim(call):E474: Expected true: tru', + exc_exec('call jsondecode("tru")')) + eq('Vim(call):E474: Expected true: tru\t\n', + exc_exec('call jsondecode("tru\\t\\n")')) + + eq('Vim(call):E474: Expected false: f', + exc_exec('call jsondecode("f")')) + eq('Vim(call):E474: Expected false: fa', + exc_exec('call jsondecode("fa")')) + eq('Vim(call):E474: Expected false: fal', + exc_exec('call jsondecode("fal")')) + eq('Vim(call):E474: Expected false: fal <', + exc_exec('call jsondecode(" fal <")')) + eq('Vim(call):E474: Expected false: fals', + exc_exec('call jsondecode("fals")')) + end) + + it('parses integer numbers', function() + eq(100000, funcs.jsondecode('100000')) + eq(-100000, funcs.jsondecode('-100000')) + eq(100000, funcs.jsondecode(' 100000 ')) + eq(-100000, funcs.jsondecode(' -100000 ')) + end) + + it('fails to parse +numbers', function() + eq('Vim(call):E474: Unidentified byte: +1000', + exc_exec('call jsondecode("+1000")')) + end) + + it('fails to parse negative numbers with space after -', function() + eq('Vim(call):E474: Missing number after minus sign: - 1000', + exc_exec('call jsondecode("- 1000")')) + end) + + it('fails to parse -', function() + eq('Vim(call):E474: Missing number after minus sign: -', + exc_exec('call jsondecode("-")')) + end) + + it('parses floating-point numbers', function() + eq('100000.0', eval('string(jsondecode("100000.0"))')) + eq(100000.5, funcs.jsondecode('100000.5')) + eq(-100000.5, funcs.jsondecode('-100000.5')) + eq(-100000.5e50, funcs.jsondecode('-100000.5e50')) + eq(100000.5e50, funcs.jsondecode('100000.5e50')) + eq(100000.5e50, funcs.jsondecode('100000.5e+50')) + eq(-100000.5e-50, funcs.jsondecode('-100000.5e-50')) + eq(100000.5e-50, funcs.jsondecode('100000.5e-50')) + end) + + it('fails to parse incomplete floating-point numbers', function() + eq('Vim(call):E474: Missing number after decimal dot: 0.', + exc_exec('call jsondecode("0.")')) + eq('Vim(call):E474: Missing exponent: 0.0e', + exc_exec('call jsondecode("0.0e")')) + eq('Vim(call):E474: Missing exponent: 0.0e+', + exc_exec('call jsondecode("0.0e+")')) + eq('Vim(call):E474: Missing exponent: 0.0e-', + exc_exec('call jsondecode("0.0e-")')) + end) + + it('fails to parse floating-point numbers with spaces inside', function() + eq('Vim(call):E474: Missing number after decimal dot: 0. ', + exc_exec('call jsondecode("0. ")')) + eq('Vim(call):E474: Missing number after decimal dot: 0. 0', + exc_exec('call jsondecode("0. 0")')) + eq('Vim(call):E474: Missing exponent: 0.0e 1', + exc_exec('call jsondecode("0.0e 1")')) + eq('Vim(call):E474: Missing exponent: 0.0e+ 1', + exc_exec('call jsondecode("0.0e+ 1")')) + eq('Vim(call):E474: Missing exponent: 0.0e- 1', + exc_exec('call jsondecode("0.0e- 1")')) + end) + + it('fails to parse "," and ":"', function() + eq('Vim(call):E474: Comma not inside container: , ', + exc_exec('call jsondecode(" , ")')) + eq('Vim(call):E474: Colon not inside container: : ', + exc_exec('call jsondecode(" : ")')) + end) + + it('parses empty containers', function() + eq({}, funcs.jsondecode('[]')) + eq('[]', eval('string(jsondecode("[]"))')) + end) + + it('fails to parse "[" and "{"', function() + eq('Vim(call):E474: Unexpected end of input: {', + exc_exec('call jsondecode("{")')) + eq('Vim(call):E474: Unexpected end of input: [', + exc_exec('call jsondecode("[")')) + end) + + it('fails to parse "}" and "]"', function() + eq('Vim(call):E474: No container to close: ]', + exc_exec('call jsondecode("]")')) + eq('Vim(call):E474: No container to close: }', + exc_exec('call jsondecode("}")')) + end) + + it('fails to parse containers which are closed by different brackets', + function() + eq('Vim(call):E474: Closing dictionary with bracket: ]', + exc_exec('call jsondecode("{]")')) + eq('Vim(call):E474: Closing list with figure brace: }', + exc_exec('call jsondecode("[}")')) + end) + + it('fails to parse containers with leading comma or colon', function() + eq('Vim(call):E474: Leading comma: ,}', + exc_exec('call jsondecode("{,}")')) + eq('Vim(call):E474: Leading comma: ,]', + exc_exec('call jsondecode("[,]")')) + eq('Vim(call):E474: Using colon not in dictionary: :]', + exc_exec('call jsondecode("[:]")')) + eq('Vim(call):E474: Unexpected colon: :}', + exc_exec('call jsondecode("{:}")')) + end) + + it('fails to parse containers with trailing comma', function() + eq('Vim(call):E474: Trailing comma: ]', + exc_exec('call jsondecode("[1,]")')) + eq('Vim(call):E474: Trailing comma: }', + exc_exec('call jsondecode("{\\"1\\": 2,}")')) + end) + + it('fails to parse dictionaries with missing value', function() + eq('Vim(call):E474: Expected value after colon: }', + exc_exec('call jsondecode("{\\"1\\":}")')) + eq('Vim(call):E474: Expected value: }', + exc_exec('call jsondecode("{\\"1\\"}")')) + end) + + it('fails to parse containers with two commas or colons', function() + eq('Vim(call):E474: Duplicate comma: , "2": 2}', + exc_exec('call jsondecode("{\\"1\\": 1,, \\"2\\": 2}")')) + eq('Vim(call):E474: Duplicate comma: , "2", 2]', + exc_exec('call jsondecode("[\\"1\\", 1,, \\"2\\", 2]")')) + eq('Vim(call):E474: Duplicate colon: : 2}', + exc_exec('call jsondecode("{\\"1\\": 1, \\"2\\":: 2}")')) + eq('Vim(call):E474: Comma after colon: , 2}', + exc_exec('call jsondecode("{\\"1\\": 1, \\"2\\":, 2}")')) + eq('Vim(call):E474: Unexpected colon: : "2": 2}', + exc_exec('call jsondecode("{\\"1\\": 1,: \\"2\\": 2}")')) + eq('Vim(call):E474: Unexpected colon: :, "2": 2}', + exc_exec('call jsondecode("{\\"1\\": 1:, \\"2\\": 2}")')) + end) + + it('fails to parse concat of two values', function() + eq('Vim(call):E474: Trailing characters: []', + exc_exec('call jsondecode("{}[]")')) + end) + + it('parses containers', function() + eq({1}, funcs.jsondecode('[1]')) + eq({nil, 1}, funcs.jsondecode('[null, 1]')) + eq({['1']=2}, funcs.jsondecode('{"1": 2}')) + eq({['1']=2, ['3']={{['4']={['5']={{}, 1}}}}}, + funcs.jsondecode('{"1": 2, "3": [{"4": {"5": [[], 1]}}]}')) + end) + + it('fails to parse incomplete strings', function() + eq('Vim(call):E474: Expected string end: \t"', + exc_exec('call jsondecode("\\t\\"")')) + eq('Vim(call):E474: Expected string end: \t"abc', + exc_exec('call jsondecode("\\t\\"abc")')) + eq('Vim(call):E474: Unfinished escape sequence: \t"abc\\', + exc_exec('call jsondecode("\\t\\"abc\\\\")')) + eq('Vim(call):E474: Unfinished unicode escape sequence: \t"abc\\u', + exc_exec('call jsondecode("\\t\\"abc\\\\u")')) + eq('Vim(call):E474: Unfinished unicode escape sequence: \t"abc\\u0', + exc_exec('call jsondecode("\\t\\"abc\\\\u0")')) + eq('Vim(call):E474: Unfinished unicode escape sequence: \t"abc\\u00', + exc_exec('call jsondecode("\\t\\"abc\\\\u00")')) + eq('Vim(call):E474: Unfinished unicode escape sequence: \t"abc\\u000', + exc_exec('call jsondecode("\\t\\"abc\\\\u000")')) + eq('Vim(call):E474: Expected string end: \t"abc\\u0000', + exc_exec('call jsondecode("\\t\\"abc\\\\u0000")')) + end) + + it('fails to parse unknown escape sequnces', function() + eq('Vim(call):E474: Unknown escape sequence: \\a"', + exc_exec('call jsondecode("\\t\\"\\\\a\\"")')) + end) + + it('parses strings properly', function() + eq('\n', funcs.jsondecode('"\\n"')) + eq('', funcs.jsondecode('""')) + eq('\\/"\t\b\n\r\f', funcs.jsondecode([["\\\/\"\t\b\n\r\f"]])) + eq('/a', funcs.jsondecode([["\/a"]])) + end) +end) + describe('jsonencode() function', function() before_each(clear) From 704accfbfae881f60c77bf9d6147a4431f0ed5da Mon Sep 17 00:00:00 2001 From: ZyX Date: Wed, 3 Feb 2016 18:46:21 +0300 Subject: [PATCH 12/82] cmake: Refactor code that creates directories and lists C files --- src/nvim/CMakeLists.txt | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/src/nvim/CMakeLists.txt b/src/nvim/CMakeLists.txt index 487b554d6d..c53a308618 100644 --- a/src/nvim/CMakeLists.txt +++ b/src/nvim/CMakeLists.txt @@ -41,22 +41,24 @@ include_directories(${GENERATED_DIR}) include_directories(${GENERATED_INCLUDES_DIR}) file(MAKE_DIRECTORY ${GENERATED_DIR}) -file(MAKE_DIRECTORY ${GENERATED_DIR}/os) -file(MAKE_DIRECTORY ${GENERATED_DIR}/api) -file(MAKE_DIRECTORY ${GENERATED_DIR}/api/private) -file(MAKE_DIRECTORY ${GENERATED_DIR}/msgpack_rpc) -file(MAKE_DIRECTORY ${GENERATED_DIR}/tui) -file(MAKE_DIRECTORY ${GENERATED_DIR}/event) file(MAKE_DIRECTORY ${GENERATED_INCLUDES_DIR}) -file(MAKE_DIRECTORY ${GENERATED_INCLUDES_DIR}/os) -file(MAKE_DIRECTORY ${GENERATED_INCLUDES_DIR}/api) -file(MAKE_DIRECTORY ${GENERATED_INCLUDES_DIR}/api/private) -file(MAKE_DIRECTORY ${GENERATED_INCLUDES_DIR}/msgpack_rpc) -file(MAKE_DIRECTORY ${GENERATED_INCLUDES_DIR}/tui) -file(MAKE_DIRECTORY ${GENERATED_INCLUDES_DIR}/event) -file(GLOB NEOVIM_SOURCES *.c os/*.c api/*.c api/private/*.c msgpack_rpc/*.c - tui/*.c event/*.c) +file(GLOB NEOVIM_SOURCES *.c) + +foreach(subdir + os + api + api/private + msgpack_rpc + tui + event + ) + file(MAKE_DIRECTORY ${GENERATED_DIR}/${subdir}) + file(MAKE_DIRECTORY ${GENERATED_INCLUDES_DIR}/${subdir}) + file(GLOB sources ${subdir}/*.c) + list(APPEND NEOVIM_SOURCES ${sources}) +endforeach() + file(GLOB_RECURSE NEOVIM_HEADERS *.h) file(GLOB UNIT_TEST_FIXTURES ${PROJECT_SOURCE_DIR}/test/unit/fixtures/*.c) From 41b44d114c030e01a7e15084d0510555ec363605 Mon Sep 17 00:00:00 2001 From: ZyX Date: Wed, 3 Feb 2016 18:50:19 +0300 Subject: [PATCH 13/82] eval: Move encode.c to eval/encode.c --- src/nvim/CMakeLists.txt | 1 + src/nvim/eval.c | 2 +- src/nvim/{ => eval}/encode.c | 6 ++---- src/nvim/{ => eval}/encode.h | 8 ++++---- src/nvim/shada.c | 2 +- 5 files changed, 9 insertions(+), 10 deletions(-) rename src/nvim/{ => eval}/encode.c (99%) rename src/nvim/{ => eval}/encode.h (93%) diff --git a/src/nvim/CMakeLists.txt b/src/nvim/CMakeLists.txt index c53a308618..7be06a99cb 100644 --- a/src/nvim/CMakeLists.txt +++ b/src/nvim/CMakeLists.txt @@ -52,6 +52,7 @@ foreach(subdir msgpack_rpc tui event + eval ) file(MAKE_DIRECTORY ${GENERATED_DIR}/${subdir}) file(MAKE_DIRECTORY ${GENERATED_INCLUDES_DIR}/${subdir}) diff --git a/src/nvim/eval.c b/src/nvim/eval.c index cfbbb6f93f..f13be7954b 100644 --- a/src/nvim/eval.c +++ b/src/nvim/eval.c @@ -73,7 +73,7 @@ #include "nvim/undo.h" #include "nvim/version.h" #include "nvim/window.h" -#include "nvim/encode.h" +#include "nvim/eval/encode.h" #include "nvim/os/os.h" #include "nvim/event/libuv_process.h" #include "nvim/event/pty_process.h" diff --git a/src/nvim/encode.c b/src/nvim/eval/encode.c similarity index 99% rename from src/nvim/encode.c rename to src/nvim/eval/encode.c index 57163253c3..e23e68dc62 100644 --- a/src/nvim/encode.c +++ b/src/nvim/eval/encode.c @@ -4,13 +4,11 @@ /// /// Split out from eval.c. -// TODO(ZyX-I): Move this to src/nvim/viml or src/nvim/eval - #include #include #include -#include "nvim/encode.h" +#include "nvim/eval/encode.h" #include "nvim/buffer_defs.h" // vimconv_T #include "nvim/eval.h" #include "nvim/eval_defs.h" @@ -63,7 +61,7 @@ const char *const encode_special_var_names[] = { }; #ifdef INCLUDE_GENERATED_DECLARATIONS -# include "encode.c.generated.h" +# include "eval/encode.c.generated.h" #endif /// Msgpack callback for writing to readfile()-style list diff --git a/src/nvim/encode.h b/src/nvim/eval/encode.h similarity index 93% rename from src/nvim/encode.h rename to src/nvim/eval/encode.h index 5b81ed84dc..0e60c96155 100644 --- a/src/nvim/encode.h +++ b/src/nvim/eval/encode.h @@ -1,5 +1,5 @@ -#ifndef NVIM_ENCODE_H -#define NVIM_ENCODE_H +#ifndef NVIM_EVAL_ENCODE_H +#define NVIM_EVAL_ENCODE_H #include @@ -55,6 +55,6 @@ static inline ListReaderState encode_init_lrstate(const list_T *const list) extern const char *const encode_special_var_names[]; #ifdef INCLUDE_GENERATED_DECLARATIONS -# include "encode.h.generated.h" +# include "eval/encode.h.generated.h" #endif -#endif // NVIM_ENCODE_H +#endif // NVIM_EVAL_ENCODE_H diff --git a/src/nvim/shada.c b/src/nvim/shada.c index 22767dbe35..f402b75ad6 100644 --- a/src/nvim/shada.c +++ b/src/nvim/shada.c @@ -39,7 +39,7 @@ #include "nvim/fileio.h" #include "nvim/strings.h" #include "nvim/quickfix.h" -#include "nvim/encode.h" +#include "nvim/eval/encode.h" #include "nvim/lib/khash.h" #include "nvim/lib/kvec.h" From 85244e68e3c2bf2bb8a175665fe759ef426c8e2a Mon Sep 17 00:00:00 2001 From: ZyX Date: Wed, 3 Feb 2016 19:12:18 +0300 Subject: [PATCH 14/82] vim: Move *MSG* macros to message.h Note: OUT* macros were removed because they are no longer used anywhere. --- src/nvim/message.h | 40 ++++++++++++++++++++++++++++++++++++++++ src/nvim/vim.h | 16 +--------------- 2 files changed, 41 insertions(+), 15 deletions(-) diff --git a/src/nvim/message.h b/src/nvim/message.h index 019c7bfb73..9249596bec 100644 --- a/src/nvim/message.h +++ b/src/nvim/message.h @@ -4,6 +4,7 @@ #include #include #include "nvim/eval_defs.h" // for typval_T +#include "nvim/ex_cmds_defs.h" // for exarg_T /* * Types of dialogs passed to do_dialog(). @@ -24,6 +25,45 @@ #define VIM_ALL 5 #define VIM_DISCARDALL 6 +/// Show plain message +#define MSG(s) msg((char_u *)(s)) + +/// Show message highlighted according to the attr +#define MSG_ATTR(s, attr) msg_attr((char_u *)(s), (attr)) + +/// Display error message +/// +/// Sets error flag in process, can be transformed into an exception. +#define EMSG(s) emsg((char_u *)(s)) + +/// Like #EMSG, but for messages with one "%s" inside +#define EMSG2(s, p) emsg2((char_u *)(s), (char_u *)(p)) + +/// Like #EMSG, but for messages with two "%s" inside +#define EMSG3(s, p, q) emsg3((char_u *)(s), (char_u *)(p), \ + (char_u *)(q)) + +/// Like #EMSG, but for messages with one "%" PRId64 inside +#define EMSGN(s, n) emsgn((char_u *)(s), (int64_t)(n)) + +/// Like #EMSG, but for messages with one "%" PRIu64 inside +#define EMSGU(s, n) emsgu((char_u *)(s), (uint64_t)(n)) + +/// Display message at the recorded position +#define MSG_PUTS(s) msg_puts((char_u *)(s)) + +/// Display message at the recorded position, highlighted +#define MSG_PUTS_ATTR(s, a) msg_puts_attr((char_u *)(s), (a)) + +/// Like #MSG_PUTS, but highlight like title +#define MSG_PUTS_TITLE(s) msg_puts_title((char_u *)(s)) + +/// Like #MSG_PUTS, but if middle part of too long messages it will be replaced +#define MSG_PUTS_LONG(s) msg_puts_long_attr((char_u *)(s), 0) + +/// Like #MSG_PUTS_ATTR, but if middle part of long messages will be replaced +#define MSG_PUTS_LONG_ATTR(s, a) msg_puts_long_attr((char_u *)(s), (a)) + #ifdef INCLUDE_GENERATED_DECLARATIONS # include "message.h.generated.h" #endif diff --git a/src/nvim/vim.h b/src/nvim/vim.h index 545b903d2f..a94b2e12f2 100644 --- a/src/nvim/vim.h +++ b/src/nvim/vim.h @@ -271,21 +271,7 @@ enum { # define vim_strpbrk(s, cs) (char_u *)strpbrk((char *)(s), (char *)(cs)) -#define MSG(s) msg((char_u *)(s)) -#define MSG_ATTR(s, attr) msg_attr((char_u *)(s), (attr)) -#define EMSG(s) emsg((char_u *)(s)) -#define EMSG2(s, p) emsg2((char_u *)(s), (char_u *)(p)) -#define EMSG3(s, p, q) emsg3((char_u *)(s), (char_u *)(p), \ - (char_u *)(q)) -#define EMSGN(s, n) emsgn((char_u *)(s), (int64_t)(n)) -#define EMSGU(s, n) emsgu((char_u *)(s), (uint64_t)(n)) -#define OUT_STR(s) out_str((char_u *)(s)) -#define OUT_STR_NF(s) out_str_nf((char_u *)(s)) -#define MSG_PUTS(s) msg_puts((char_u *)(s)) -#define MSG_PUTS_ATTR(s, a) msg_puts_attr((char_u *)(s), (a)) -#define MSG_PUTS_TITLE(s) msg_puts_title((char_u *)(s)) -#define MSG_PUTS_LONG(s) msg_puts_long_attr((char_u *)(s), 0) -#define MSG_PUTS_LONG_ATTR(s, a) msg_puts_long_attr((char_u *)(s), (a)) +#include "nvim/message.h" /* Prefer using emsg3(), because perror() may send the output to the wrong * destination and mess up the screen. */ From 3c3921955040967b8f5489316342d95d80c235bb Mon Sep 17 00:00:00 2001 From: ZyX Date: Wed, 3 Feb 2016 19:35:49 +0300 Subject: [PATCH 15/82] eval: Get rid of VV_LEN constant --- src/nvim/eval.c | 22 +++++++++++----------- src/nvim/eval.h | 5 ++--- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/src/nvim/eval.c b/src/nvim/eval.c index f13be7954b..7edc1402f5 100644 --- a/src/nvim/eval.c +++ b/src/nvim/eval.c @@ -289,11 +289,11 @@ typedef enum { // The reason to use this table anyway is for very quick access to the // variables with the VV_ defines. static struct vimvar { - char *vv_name; /* name of variable, without v: */ - dictitem_T vv_di; /* value and name for key */ - char vv_filler[16]; /* space for LONGEST name below!!! */ - char vv_flags; /* VV_COMPAT, VV_RO, VV_RO_SBX */ -} vimvars[VV_LEN] = + char *vv_name; ///< Name of the variable, without v:. + dictitem_T vv_di; ///< Value of the variable, with name. + char vv_filler[16]; ///< Space for longest name from below. + char vv_flags; ///< Flags: #VV_COMPAT, #VV_RO, #VV_RO_SBX. +} vimvars[] = { /* * The order here must match the VV_ defines in eval.h! @@ -465,7 +465,6 @@ const list_T *eval_msgpack_type_lists[] = { void eval_init(void) { jobs = pmap_new(uint64_t)(); - int i; struct vimvar *p; init_var_dict(&globvardict, &globvars_var, VAR_DEF_SCOPE); @@ -474,7 +473,7 @@ void eval_init(void) hash_init(&compat_hashtab); hash_init(&func_hashtab); - for (i = 0; i < VV_LEN; ++i) { + for (size_t i = 0; i < ARRAY_SIZE(vimvars); ++i) { p = &vimvars[i]; STRCPY(p->vv_di.di_key, p->vv_name); if (p->vv_flags & VV_RO) @@ -534,7 +533,7 @@ void eval_clear(void) { struct vimvar *p; - for (int i = 0; i < VV_LEN; ++i) { + for (size_t i = 0; i < ARRAY_SIZE(vimvars); ++i) { p = &vimvars[i]; if (p->vv_di.di_tv.v_type == VAR_STRING) { xfree(p->vv_str); @@ -3188,7 +3187,7 @@ char_u *get_user_var_name(expand_T *xp, int idx) static size_t bdone; static size_t wdone; static size_t tdone; - static int vidx; + static size_t vidx; static hashitem_T *hi; hashtab_T *ht; @@ -3250,9 +3249,10 @@ char_u *get_user_var_name(expand_T *xp, int idx) return cat_prefix_varname('t', hi->hi_key); } - /* v: variables */ - if (vidx < VV_LEN) + // v: variables + if (vidx < ARRAY_SIZE(vimvars)) { return cat_prefix_varname('v', (char_u *)vimvars[vidx++].vv_name); + } xfree(varnamebuf); varnamebuf = NULL; diff --git a/src/nvim/eval.h b/src/nvim/eval.h index 89aa263434..40111abf8d 100644 --- a/src/nvim/eval.h +++ b/src/nvim/eval.h @@ -54,7 +54,7 @@ EXTERN ufunc_T dumuf; #define HI2UF(hi) HIKEY2UF((hi)->hi_key) /* Defines for Vim variables. These must match vimvars[] in eval.c! */ -enum { +typedef enum { VV_COUNT, VV_COUNT1, VV_PREVCOUNT, @@ -125,8 +125,7 @@ enum { VV_TRUE, VV_NULL, VV_NONE, - VV_LEN, ///< Number of v: variables -}; +} VimVarIndex; /// All recognized msgpack types typedef enum { From 700b32a2b3af859299cbe92914f1a4cd800de724 Mon Sep 17 00:00:00 2001 From: ZyX Date: Wed, 3 Feb 2016 19:43:48 +0300 Subject: [PATCH 16/82] eval: Move some decoding functions to eval/decode.c --- src/nvim/eval.c | 839 ++--------------------------------------- src/nvim/eval/decode.c | 753 ++++++++++++++++++++++++++++++++++++ src/nvim/eval/decode.h | 13 + src/nvim/eval/encode.c | 2 +- src/nvim/shada.c | 2 +- 5 files changed, 798 insertions(+), 811 deletions(-) create mode 100644 src/nvim/eval/decode.c create mode 100644 src/nvim/eval/decode.h diff --git a/src/nvim/eval.c b/src/nvim/eval.c index 7edc1402f5..38dc7c1034 100644 --- a/src/nvim/eval.c +++ b/src/nvim/eval.c @@ -74,6 +74,7 @@ #include "nvim/version.h" #include "nvim/window.h" #include "nvim/eval/encode.h" +#include "nvim/eval/decode.h" #include "nvim/os/os.h" #include "nvim/event/libuv_process.h" #include "nvim/event/pty_process.h" @@ -89,7 +90,6 @@ #include "nvim/os/input.h" #include "nvim/event/loop.h" #include "nvim/lib/queue.h" -#include "nvim/lib/kvec.h" #define DICT_MAXNEST 100 /* maximum nesting of lists and dicts */ @@ -416,16 +416,6 @@ typedef struct { int status; } JobEvent; -/// Helper structure for container_struct -typedef struct { - size_t stack_index; ///< Index of current container in stack. - typval_T container; ///< Container. Either VAR_LIST, VAR_DICT or VAR_LIST - ///< which is _VAL from special dictionary. -} ContainerStackItem; - -typedef kvec_t(typval_T) ValuesStack; -typedef kvec_t(ContainerStackItem) ContainerStack; - #ifdef INCLUDE_GENERATED_DECLARATIONS # include "eval.c.generated.h" #endif @@ -4142,7 +4132,7 @@ static int eval7( if (get_float) { float_T f; - *arg += string2float(*arg, &f); + *arg += string2float((char *) *arg, &f); if (evaluate) { rettv->v_type = VAR_FLOAT; rettv->vval.v_float = f; @@ -6550,24 +6540,22 @@ failret: return OK; } -/* - * Convert the string "text" to a floating point number. - * This uses strtod(). setlocale(LC_NUMERIC, "C") has been used to make sure - * this always uses a decimal point. - * Returns the length of the text that was consumed. - */ -static int -string2float ( - char_u *text, - float_T *value /* result stored here */ -) +/// Convert the string to a floating point number +/// +/// This uses strtod(). setlocale(LC_NUMERIC, "C") has been used earlier to +/// make sure this always uses a decimal point. +/// +/// @param[in] text String to convert. +/// @param[out] ret_value Location where conversion result is saved. +/// +/// @return Length of the text that was consumed. +size_t string2float(const char *const text, float_T *const ret_value) + FUNC_ATTR_NONNULL_ALL { - char *s = (char *)text; - float_T f; + char *s = NULL; - f = strtod(s, &s); - *value = f; - return (int)((char_u *)s - text); + *ret_value = strtod(text, &s); + return (size_t) (s - text); } /// Get the value of an environment variable. @@ -11571,498 +11559,6 @@ static void f_join(typval_T *argvars, typval_T *rettv) rettv->vval.v_string = NULL; } -/// Helper function used for working with stack vectors used by JSON decoder -/// -/// @param[in] obj New object. -/// @param[out] stack Object stack. -/// @param[out] container_stack Container objects stack. -/// @param[in] p Position in string which is currently being parsed. -/// -/// @return OK in case of success, FAIL in case of error. -static inline int json_decoder_pop(typval_T obj, ValuesStack *const stack, - ContainerStack *const container_stack, - const char *const p) - FUNC_ATTR_NONNULL_ALL -{ - if (kv_size(*container_stack) == 0) { - kv_push(typval_T, *stack, obj); - return OK; - } - ContainerStackItem last_container = kv_last(*container_stack); - if (obj.v_type == last_container.container.v_type - // vval.v_list and vval.v_dict should have the same size and offset - && ((void *) obj.vval.v_list - == (void *) last_container.container.vval.v_list)) { - kv_pop(*container_stack); - last_container = kv_last(*container_stack); - } - if (last_container.container.v_type == VAR_LIST) { - listitem_T *obj_li = listitem_alloc(); - obj_li->li_tv = obj; - list_append(last_container.container.vval.v_list, obj_li); - } else if (last_container.stack_index == kv_size(*stack) - 2) { - typval_T key = kv_pop(*stack); - if (key.v_type != VAR_STRING) { - assert(false); - } else if (key.vval.v_string == NULL || *key.vval.v_string == NUL) { - // TODO: fall back to special dict in case of empty key - EMSG(_("E474: Empty key")); - clear_tv(&obj); - return FAIL; - } - dictitem_T *obj_di = dictitem_alloc(key.vval.v_string); - clear_tv(&key); - if (dict_add(last_container.container.vval.v_dict, obj_di) - == FAIL) { - // TODO: fall back to special dict in case of duplicate keys - EMSG(_("E474: Duplicate key")); - dictitem_free(obj_di); - clear_tv(&obj); - return FAIL; - } - obj_di->di_tv = obj; - } else { - // Object with key only - if (obj.v_type != VAR_STRING) { - EMSG2(_("E474: Expected string key: %s"), p); - clear_tv(&obj); - return FAIL; - } - kv_push(typval_T, *stack, obj); - } - return OK; -} - -/// Convert JSON string into VimL object -/// -/// @param[in] buf String to convert. UTF-8 encoding is assumed. -/// @param[in] len Length of the string. -/// @param[out] rettv Location where to save results. -/// -/// @return OK in case of success, FAIL otherwise. -static int json_decode_string(const char *const buf, const size_t len, - typval_T *rettv) - FUNC_ATTR_NONNULL_ALL FUNC_ATTR_WARN_UNUSED_RESULT -{ - vimconv_T conv; - convert_setup(&conv, (char_u *) "utf-8", p_enc); - conv.vc_fail = true; - int ret = OK; - ValuesStack stack; - kv_init(stack); - ContainerStack container_stack; - kv_init(container_stack); - rettv->v_type = VAR_UNKNOWN; - const char *const e = buf + len; - bool didcomma = false; - bool didcolon = false; -#define POP(obj) \ - do { \ - if (json_decoder_pop(obj, &stack, &container_stack, p) == FAIL) { \ - goto json_decode_string_fail; \ - } \ - } while (0) - const char *p = buf; - for (; p < e; p++) { - switch (*p) { - case '}': - case ']': { - if (kv_size(container_stack) == 0) { - EMSG2(_("E474: No container to close: %s"), p); - goto json_decode_string_fail; - } - ContainerStackItem last_container = kv_last(container_stack); - if (*p == '}' && last_container.container.v_type != VAR_DICT) { - EMSG2(_("E474: Closing list with figure brace: %s"), p); - goto json_decode_string_fail; - } else if (*p == ']' && last_container.container.v_type != VAR_LIST) { - EMSG2(_("E474: Closing dictionary with bracket: %s"), p); - goto json_decode_string_fail; - } else if (didcomma) { - EMSG2(_("E474: Trailing comma: %s"), p); - goto json_decode_string_fail; - } else if (didcolon) { - EMSG2(_("E474: Expected value after colon: %s"), p); - goto json_decode_string_fail; - } else if (last_container.stack_index != kv_size(stack) - 1) { - assert(last_container.stack_index < kv_size(stack) - 1); - EMSG2(_("E474: Expected value: %s"), p); - goto json_decode_string_fail; - } - if (kv_size(stack) == 1) { - p++; - kv_pop(container_stack); - goto json_decode_string_after_cycle; - } else { - typval_T obj = kv_pop(stack); - POP(obj); - break; - } - } - case ',': { - if (kv_size(container_stack) == 0) { - EMSG2(_("E474: Comma not inside container: %s"), p); - goto json_decode_string_fail; - } - ContainerStackItem last_container = kv_last(container_stack); - if (didcomma) { - EMSG2(_("E474: Duplicate comma: %s"), p); - goto json_decode_string_fail; - } else if (didcolon) { - EMSG2(_("E474: Comma after colon: %s"), p); - goto json_decode_string_fail; - } if (last_container.container.v_type == VAR_DICT - && last_container.stack_index != kv_size(stack) - 1) { - EMSG2(_("E474: Using comma in place of colon: %s"), p); - goto json_decode_string_fail; - } else if ((last_container.container.v_type == VAR_DICT - && (last_container.container.vval.v_dict->dv_hashtab.ht_used - == 0)) - || (last_container.container.v_type == VAR_LIST - && last_container.container.vval.v_list->lv_len == 0)) { - EMSG2(_("E474: Leading comma: %s"), p); - goto json_decode_string_fail; - } - didcomma = true; - continue; - } - case ':': { - if (kv_size(container_stack) == 0) { - EMSG2(_("E474: Colon not inside container: %s"), p); - goto json_decode_string_fail; - } - ContainerStackItem last_container = kv_last(container_stack); - if (last_container.container.v_type != VAR_DICT) { - EMSG2(_("E474: Using colon not in dictionary: %s"), p); - goto json_decode_string_fail; - } else if (last_container.stack_index != kv_size(stack) - 2) { - EMSG2(_("E474: Unexpected colon: %s"), p); - goto json_decode_string_fail; - } else if (didcomma) { - EMSG2(_("E474: Colon after comma: %s"), p); - goto json_decode_string_fail; - } else if (didcolon) { - EMSG2(_("E474: Duplicate colon: %s"), p); - goto json_decode_string_fail; - } - didcolon = true; - continue; - } - case ' ': - case TAB: - case NL: { - continue; - } - case 'n': { - if (strncmp(p + 1, "ull", 3) != 0) { - EMSG2(_("E474: Expected null: %s"), p); - goto json_decode_string_fail; - } - p += 3; - POP(vimvars[VV_NULL].vv_di.di_tv); - break; - } - case 't': { - if (strncmp(p + 1, "rue", 3) != 0) { - EMSG2(_("E474: Expected true: %s"), p); - goto json_decode_string_fail; - } - p += 3; - POP(vimvars[VV_TRUE].vv_di.di_tv); - break; - } - case 'f': { - if (strncmp(p + 1, "alse", 4) != 0) { - EMSG2(_("E474: Expected false: %s"), p); - goto json_decode_string_fail; - } - p += 4; - POP(vimvars[VV_FALSE].vv_di.di_tv); - break; - } - case '"': { - size_t len = 0; - const char *s; - for (s = ++p; p < e && *p != '"'; p++) { - if (*p == '\\') { - p++; - if (p == e) { - EMSG2(_("E474: Unfinished escape sequence: %s"), buf); - goto json_decode_string_fail; - } - switch (*p) { - case 'u': { - if (p + 4 >= e) { - EMSG2(_("E474: Unfinished unicode escape sequence: %s"), buf); - goto json_decode_string_fail; - } else if (!ascii_isxdigit(p[1]) - || !ascii_isxdigit(p[2]) - || !ascii_isxdigit(p[3]) - || !ascii_isxdigit(p[4])) { - EMSG2(_("E474: Expected four hex digits after \\u: %s"), - p - 1); - goto json_decode_string_fail; - } - // One UTF-8 character below U+10000 can take up to 3 bytes - len += 3; - p += 4; - break; - } - case '\\': - case '/': - case '"': - case 't': - case 'b': - case 'n': - case 'r': - case 'f': { - len++; - break; - } - default: { - EMSG2(_("E474: Unknown escape sequence: %s"), p - 1); - goto json_decode_string_fail; - } - } - } else { - len++; - } - } - if (*p != '"') { - EMSG2(_("E474: Expected string end: %s"), buf); - goto json_decode_string_fail; - } - char *str = xmalloc(len + 1); - uint16_t fst_in_pair = 0; - char *str_end = str; - for (const char *t = s; t < p; t++) { - if (t[0] != '\\' || t[1] != 'u') { - if (fst_in_pair != 0) { - str_end += utf_char2bytes((int) fst_in_pair, (char_u *) str_end); - fst_in_pair = 0; - } - } - if (*t == '\\') { - t++; - switch (*t) { - case 'u': { - char ubuf[] = { t[1], t[2], t[3], t[4], 0 }; - t += 4; - unsigned long ch; - vim_str2nr((char_u *) ubuf, NULL, NULL, 0, 0, 2, NULL, &ch); - if (0xD800UL <= ch && ch <= 0xDB7FUL) { - fst_in_pair = (uint16_t) ch; - } else if (0xDC00ULL <= ch && ch <= 0xDB7FUL) { - if (fst_in_pair != 0) { - int full_char = ( - (int) (ch - 0xDC00UL) - + (((int) (fst_in_pair - 0xD800)) << 10) - ); - str_end += utf_char2bytes(full_char, (char_u *) str_end); - } - } else { - str_end += utf_char2bytes((int) ch, (char_u *) str_end); - } - break; - } - case '\\': - case '/': - case '"': - case 't': - case 'b': - case 'n': - case 'r': - case 'f': { - static const char escapes[] = { - ['\\'] = '\\', - ['/'] = '/', - ['"'] = '"', - ['t'] = TAB, - ['b'] = BS, - ['n'] = NL, - ['r'] = CAR, - ['f'] = FF, - }; - *str_end++ = escapes[(int) *t]; - break; - } - default: { - assert(false); - } - } - } else { - *str_end++ = *t; - } - } - if (fst_in_pair != 0) { - str_end += utf_char2bytes((int) fst_in_pair, (char_u *) str_end); - } - if (conv.vc_type != CONV_NONE) { - size_t len = (str_end - str); - char *const new_str = (char *) string_convert(&conv, (char_u *) str, - &len); - if (new_str == NULL) { - EMSG2(_("E474: Failed to convert string \"%s\" from UTF-8"), str); - xfree(str); - goto json_decode_string_fail; - } - xfree(str); - str = new_str; - str_end = new_str + len; - } - *str_end = NUL; - // TODO: return special string in case of NUL bytes - POP(((typval_T) { - .v_type = VAR_STRING, - .vval = { .v_string = (char_u *) str, }, - })); - break; - } - case '-': - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': - case '8': - case '9': { - // a.bE[+-]exp - const char *const s = p; - const char *ints = NULL; - const char *fracs = NULL; - const char *exps = NULL; - if (*p == '-') { - p++; - } - ints = p; - while (p < e && ascii_isdigit(*p)) { - p++; - } - if (p < e && *p == '.') { - p++; - fracs = p; - while (p < e && ascii_isdigit(*p)) { - p++; - } - if (p < e && (*p == 'e' || *p == 'E')) { - p++; - if (p < e && (*p == '-' || *p == '+')) { - p++; - } - exps = p; - while (p < e && ascii_isdigit(*p)) { - p++; - } - } - } - if (p == ints) { - EMSG2(_("E474: Missing number after minus sign: %s"), s); - goto json_decode_string_fail; - } else if (p == fracs) { - EMSG2(_("E474: Missing number after decimal dot: %s"), s); - goto json_decode_string_fail; - } else if (p == exps) { - EMSG2(_("E474: Missing exponent: %s"), s); - goto json_decode_string_fail; - } - typval_T tv = { - .v_type = VAR_NUMBER, - .v_lock = VAR_UNLOCKED, - }; - if (fracs) { - // Convert floating-point number - (void) string2float((char_u *) s, &tv.vval.v_float); - tv.v_type = VAR_FLOAT; - } else { - // Convert integer - long nr; - vim_str2nr((char_u *) s, NULL, NULL, 0, 0, 0, &nr, NULL); - tv.vval.v_number = (varnumber_T) nr; - } - POP(tv); - p--; - break; - } - case '[': { - list_T *list = list_alloc(); - list->lv_refcount++; - typval_T tv = { - .v_type = VAR_LIST, - .v_lock = VAR_UNLOCKED, - .vval = { .v_list = list }, - }; - kv_push(ContainerStackItem, container_stack, ((ContainerStackItem) { - .stack_index = kv_size(stack), - .container = tv, - })); - kv_push(typval_T, stack, tv); - break; - } - case '{': { - dict_T *dict = dict_alloc(); - dict->dv_refcount++; - typval_T tv = { - .v_type = VAR_DICT, - .v_lock = VAR_UNLOCKED, - .vval = { .v_dict = dict }, - }; - kv_push(ContainerStackItem, container_stack, ((ContainerStackItem) { - .stack_index = kv_size(stack), - .container = tv, - })); - kv_push(typval_T, stack, tv); - break; - } - default: { - EMSG2(_("E474: Unidentified byte: %s"), p); - goto json_decode_string_fail; - } - } - didcomma = false; - didcolon = false; - if (kv_size(container_stack) == 0) { - p++; - break; - } - } -#undef POP -json_decode_string_after_cycle: - for (; p < e; p++) { - switch (*p) { - case NL: - case ' ': - case TAB: { - break; - } - default: { - EMSG2(_("E474: Trailing characters: %s"), p); - goto json_decode_string_fail; - } - } - } - if (kv_size(stack) > 1 || kv_size(container_stack)) { - EMSG2(_("E474: Unexpected end of input: %s"), buf); - goto json_decode_string_fail; - } - goto json_decode_string_ret; -json_decode_string_fail: - ret = FAIL; - while (kv_size(stack)) { - clear_tv(&kv_pop(stack)); - } -json_decode_string_ret: - if (ret != FAIL) { - assert(kv_size(stack) == 1); - *rettv = kv_pop(stack); - } - kv_destroy(stack); - kv_destroy(container_stack); - return ret; -} - /// jsondecode() function static void f_jsondecode(typval_T *argvars, typval_T *rettv) { @@ -12865,58 +12361,6 @@ static void f_mode(typval_T *argvars, typval_T *rettv) rettv->v_type = VAR_STRING; } -/// Msgpack callback for writing to readfile()-style list -static int msgpack_list_write(void *data, const char *buf, size_t len) -{ - if (len == 0) { - return 0; - } - list_T *const list = (list_T *) data; - const char *const end = buf + len; - const char *line_end = buf; - if (list->lv_last == NULL) { - list_append_string(list, NULL, 0); - } - listitem_T *li = list->lv_last; - do { - const char *line_start = line_end; - line_end = xmemscan(line_start, NL, end - line_start); - if (line_end == line_start) { - list_append_allocated_string(list, NULL); - } else { - const size_t line_length = line_end - line_start; - char *str; - if (li == NULL) { - str = xmemdupz(line_start, line_length); - } else { - const size_t li_len = (li->li_tv.vval.v_string == NULL - ? 0 - : STRLEN(li->li_tv.vval.v_string)); - li->li_tv.vval.v_string = xrealloc(li->li_tv.vval.v_string, - li_len + line_length + 1); - str = (char *) li->li_tv.vval.v_string + li_len; - memmove(str, line_start, line_length); - str[line_length] = 0; - } - for (size_t i = 0; i < line_length; i++) { - if (str[i] == NUL) { - str[i] = NL; - } - } - if (li == NULL) { - list_append_allocated_string(list, str); - } else { - li = NULL; - } - if (line_end == end - 1) { - list_append_allocated_string(list, NULL); - } - } - line_end++; - } while (line_end < end); - return 0; -} - /// "msgpackdump()" function static void f_msgpackdump(typval_T *argvars, typval_T *rettv) FUNC_ATTR_NONNULL_ALL @@ -12930,7 +12374,7 @@ static void f_msgpackdump(typval_T *argvars, typval_T *rettv) if (list == NULL) { return; } - msgpack_packer *lpacker = msgpack_packer_new(ret_list, &msgpack_list_write); + msgpack_packer *lpacker = msgpack_packer_new(ret_list, &encode_list_write); const char *const msg = _("msgpackdump() argument, index %i"); // Assume that translation will not take more then 4 times more space char msgbuf[sizeof("msgpackdump() argument, index ") * 4 + NUMBUFLEN]; @@ -12945,241 +12389,6 @@ static void f_msgpackdump(typval_T *argvars, typval_T *rettv) msgpack_packer_free(lpacker); } -/// Convert msgpack object to a VimL one -int msgpack_to_vim(const msgpack_object mobj, typval_T *const rettv) - FUNC_ATTR_NONNULL_ALL FUNC_ATTR_WARN_UNUSED_RESULT -{ -#define INIT_SPECIAL_DICT(tv, type, val) \ - do { \ - dict_T *const dict = dict_alloc(); \ - dictitem_T *const type_di = dictitem_alloc((char_u *) "_TYPE"); \ - type_di->di_tv.v_type = VAR_LIST; \ - type_di->di_tv.v_lock = 0; \ - type_di->di_tv.vval.v_list = (list_T *) eval_msgpack_type_lists[type]; \ - type_di->di_tv.vval.v_list->lv_refcount++; \ - dict_add(dict, type_di); \ - dictitem_T *const val_di = dictitem_alloc((char_u *) "_VAL"); \ - val_di->di_tv = val; \ - dict_add(dict, val_di); \ - tv->v_type = VAR_DICT; \ - dict->dv_refcount++; \ - tv->vval.v_dict = dict; \ - } while (0) - switch (mobj.type) { - case MSGPACK_OBJECT_NIL: { - INIT_SPECIAL_DICT(rettv, kMPNil, ((typval_T) { - .v_type = VAR_NUMBER, - .v_lock = 0, - .vval = { .v_number = 0 }, - })); - break; - } - case MSGPACK_OBJECT_BOOLEAN: { - INIT_SPECIAL_DICT(rettv, kMPBoolean, - ((typval_T) { - .v_type = VAR_NUMBER, - .v_lock = 0, - .vval = { - .v_number = (varnumber_T) mobj.via.boolean, - }, - })); - break; - } - case MSGPACK_OBJECT_POSITIVE_INTEGER: { - if (mobj.via.u64 <= VARNUMBER_MAX) { - *rettv = (typval_T) { - .v_type = VAR_NUMBER, - .v_lock = 0, - .vval = { .v_number = (varnumber_T) mobj.via.u64 }, - }; - } else { - list_T *const list = list_alloc(); - list->lv_refcount++; - INIT_SPECIAL_DICT(rettv, kMPInteger, - ((typval_T) { - .v_type = VAR_LIST, - .v_lock = 0, - .vval = { .v_list = list }, - })); - uint64_t n = mobj.via.u64; - list_append_number(list, 1); - list_append_number(list, (varnumber_T) ((n >> 62) & 0x3)); - list_append_number(list, (varnumber_T) ((n >> 31) & 0x7FFFFFFF)); - list_append_number(list, (varnumber_T) (n & 0x7FFFFFFF)); - } - break; - } - case MSGPACK_OBJECT_NEGATIVE_INTEGER: { - if (mobj.via.i64 >= VARNUMBER_MIN) { - *rettv = (typval_T) { - .v_type = VAR_NUMBER, - .v_lock = 0, - .vval = { .v_number = (varnumber_T) mobj.via.i64 }, - }; - } else { - list_T *const list = list_alloc(); - list->lv_refcount++; - INIT_SPECIAL_DICT(rettv, kMPInteger, - ((typval_T) { - .v_type = VAR_LIST, - .v_lock = 0, - .vval = { .v_list = list }, - })); - uint64_t n = -((uint64_t) mobj.via.i64); - list_append_number(list, -1); - list_append_number(list, (varnumber_T) ((n >> 62) & 0x3)); - list_append_number(list, (varnumber_T) ((n >> 31) & 0x7FFFFFFF)); - list_append_number(list, (varnumber_T) (n & 0x7FFFFFFF)); - } - break; - } - case MSGPACK_OBJECT_FLOAT: { - *rettv = (typval_T) { - .v_type = VAR_FLOAT, - .v_lock = 0, - .vval = { .v_float = mobj.via.f64 }, - }; - break; - } - case MSGPACK_OBJECT_STR: { - list_T *const list = list_alloc(); - list->lv_refcount++; - INIT_SPECIAL_DICT(rettv, kMPString, - ((typval_T) { - .v_type = VAR_LIST, - .v_lock = 0, - .vval = { .v_list = list }, - })); - if (msgpack_list_write((void *) list, mobj.via.str.ptr, mobj.via.str.size) - == -1) { - return FAIL; - } - break; - } - case MSGPACK_OBJECT_BIN: { - if (memchr(mobj.via.bin.ptr, NUL, mobj.via.bin.size) == NULL) { - *rettv = (typval_T) { - .v_type = VAR_STRING, - .v_lock = 0, - .vval = { .v_string = xmemdupz(mobj.via.bin.ptr, mobj.via.bin.size) }, - }; - break; - } - list_T *const list = list_alloc(); - list->lv_refcount++; - INIT_SPECIAL_DICT(rettv, kMPBinary, - ((typval_T) { - .v_type = VAR_LIST, - .v_lock = 0, - .vval = { .v_list = list }, - })); - if (msgpack_list_write((void *) list, mobj.via.bin.ptr, mobj.via.bin.size) - == -1) { - return FAIL; - } - break; - } - case MSGPACK_OBJECT_ARRAY: { - list_T *const list = list_alloc(); - list->lv_refcount++; - *rettv = (typval_T) { - .v_type = VAR_LIST, - .v_lock = 0, - .vval = { .v_list = list }, - }; - for (size_t i = 0; i < mobj.via.array.size; i++) { - listitem_T *const li = listitem_alloc(); - li->li_tv.v_type = VAR_UNKNOWN; - list_append(list, li); - if (msgpack_to_vim(mobj.via.array.ptr[i], &li->li_tv) == FAIL) { - return FAIL; - } - } - break; - } - case MSGPACK_OBJECT_MAP: { - for (size_t i = 0; i < mobj.via.map.size; i++) { - if (mobj.via.map.ptr[i].key.type != MSGPACK_OBJECT_STR - || mobj.via.map.ptr[i].key.via.str.size == 0 - || memchr(mobj.via.map.ptr[i].key.via.str.ptr, NUL, - mobj.via.map.ptr[i].key.via.str.size) != NULL) { - goto msgpack_to_vim_generic_map; - } - } - dict_T *const dict = dict_alloc(); - dict->dv_refcount++; - *rettv = (typval_T) { - .v_type = VAR_DICT, - .v_lock = 0, - .vval = { .v_dict = dict }, - }; - for (size_t i = 0; i < mobj.via.map.size; i++) { - dictitem_T *const di = xmallocz(offsetof(dictitem_T, di_key) - + mobj.via.map.ptr[i].key.via.str.size); - memcpy(&di->di_key[0], mobj.via.map.ptr[i].key.via.str.ptr, - mobj.via.map.ptr[i].key.via.str.size); - di->di_tv.v_type = VAR_UNKNOWN; - if (dict_add(dict, di) == FAIL) { - // Duplicate key: fallback to generic map - clear_tv(rettv); - xfree(di); - goto msgpack_to_vim_generic_map; - } - if (msgpack_to_vim(mobj.via.map.ptr[i].val, &di->di_tv) == FAIL) { - return FAIL; - } - } - break; -msgpack_to_vim_generic_map: {} - list_T *const list = list_alloc(); - list->lv_refcount++; - INIT_SPECIAL_DICT(rettv, kMPMap, - ((typval_T) { - .v_type = VAR_LIST, - .v_lock = 0, - .vval = { .v_list = list }, - })); - for (size_t i = 0; i < mobj.via.map.size; i++) { - list_T *const kv_pair = list_alloc(); - list_append_list(list, kv_pair); - listitem_T *const key_li = listitem_alloc(); - key_li->li_tv.v_type = VAR_UNKNOWN; - list_append(kv_pair, key_li); - listitem_T *const val_li = listitem_alloc(); - val_li->li_tv.v_type = VAR_UNKNOWN; - list_append(kv_pair, val_li); - if (msgpack_to_vim(mobj.via.map.ptr[i].key, &key_li->li_tv) == FAIL) { - return FAIL; - } - if (msgpack_to_vim(mobj.via.map.ptr[i].val, &val_li->li_tv) == FAIL) { - return FAIL; - } - } - break; - } - case MSGPACK_OBJECT_EXT: { - list_T *const list = list_alloc(); - list->lv_refcount++; - list_append_number(list, mobj.via.ext.type); - list_T *const ext_val_list = list_alloc(); - list_append_list(list, ext_val_list); - INIT_SPECIAL_DICT(rettv, kMPExt, - ((typval_T) { - .v_type = VAR_LIST, - .v_lock = 0, - .vval = { .v_list = list }, - })); - if (msgpack_list_write((void *) ext_val_list, mobj.via.ext.ptr, - mobj.via.ext.size) == -1) { - return FAIL; - } - break; - } - } -#undef INIT_SPECIAL_DICT - return OK; -} - /// "msgpackparse" function static void f_msgpackparse(typval_T *argvars, typval_T *rettv) FUNC_ATTR_NONNULL_ALL @@ -15927,7 +15136,7 @@ static void f_str2float(typval_T *argvars, typval_T *rettv) if (*p == '+') p = skipwhite(p + 1); - (void)string2float(p, &rettv->vval.v_float); + (void) string2float((char *) p, &rettv->vval.v_float); rettv->v_type = VAR_FLOAT; } @@ -17851,6 +17060,18 @@ long get_vim_var_nr(int idx) FUNC_ATTR_PURE return vimvars[idx].vv_nr; } +/// Get typval_T representing v: variable +/// +/// @warning if v: variable has reference counter it is not increased. +/// +/// @param[in] idx Variable index, @see VimVarIndex. +typval_T get_vim_var_tv(VimVarIndex idx) FUNC_ATTR_PURE +{ + typval_T ret = vimvars[idx].vv_di.di_tv; + ret.v_lock = VAR_UNLOCKED; + return ret; +} + /* * Get string v: variable value. Uses a static buffer, can only be used once. */ diff --git a/src/nvim/eval/decode.c b/src/nvim/eval/decode.c new file mode 100644 index 0000000000..e2b2574d83 --- /dev/null +++ b/src/nvim/eval/decode.c @@ -0,0 +1,753 @@ +#include + +#include + +#include "nvim/eval_defs.h" +#include "nvim/eval.h" +#include "nvim/eval/encode.h" +#include "nvim/ascii.h" +#include "nvim/message.h" +#include "nvim/charset.h" // vim_str2nr +#include "nvim/lib/kvec.h" +#include "nvim/vim.h" // OK, FAIL + +/// Helper structure for container_struct +typedef struct { + size_t stack_index; ///< Index of current container in stack. + typval_T container; ///< Container. Either VAR_LIST, VAR_DICT or VAR_LIST + ///< which is _VAL from special dictionary. +} ContainerStackItem; + +typedef kvec_t(typval_T) ValuesStack; +typedef kvec_t(ContainerStackItem) ContainerStack; + +#ifdef INCLUDE_GENERATED_DECLARATIONS +# include "eval/decode.c.generated.h" +#endif + +/// Helper function used for working with stack vectors used by JSON decoder +/// +/// @param[in] obj New object. +/// @param[out] stack Object stack. +/// @param[out] container_stack Container objects stack. +/// @param[in] p Position in string which is currently being parsed. +/// +/// @return OK in case of success, FAIL in case of error. +static inline int json_decoder_pop(typval_T obj, ValuesStack *const stack, + ContainerStack *const container_stack, + const char *const p) + FUNC_ATTR_NONNULL_ALL +{ + if (kv_size(*container_stack) == 0) { + kv_push(typval_T, *stack, obj); + return OK; + } + ContainerStackItem last_container = kv_last(*container_stack); + if (obj.v_type == last_container.container.v_type + // vval.v_list and vval.v_dict should have the same size and offset + && ((void *) obj.vval.v_list + == (void *) last_container.container.vval.v_list)) { + kv_pop(*container_stack); + last_container = kv_last(*container_stack); + } + if (last_container.container.v_type == VAR_LIST) { + listitem_T *obj_li = listitem_alloc(); + obj_li->li_tv = obj; + list_append(last_container.container.vval.v_list, obj_li); + } else if (last_container.stack_index == kv_size(*stack) - 2) { + typval_T key = kv_pop(*stack); + if (key.v_type != VAR_STRING) { + assert(false); + } else if (key.vval.v_string == NULL || *key.vval.v_string == NUL) { + // TODO: fall back to special dict in case of empty key + EMSG(_("E474: Empty key")); + clear_tv(&obj); + return FAIL; + } + dictitem_T *obj_di = dictitem_alloc(key.vval.v_string); + clear_tv(&key); + if (dict_add(last_container.container.vval.v_dict, obj_di) + == FAIL) { + // TODO: fall back to special dict in case of duplicate keys + EMSG(_("E474: Duplicate key")); + dictitem_free(obj_di); + clear_tv(&obj); + return FAIL; + } + obj_di->di_tv = obj; + } else { + // Object with key only + if (obj.v_type != VAR_STRING) { + EMSG2(_("E474: Expected string key: %s"), p); + clear_tv(&obj); + return FAIL; + } + kv_push(typval_T, *stack, obj); + } + return OK; +} + +/// Convert JSON string into VimL object +/// +/// @param[in] buf String to convert. UTF-8 encoding is assumed. +/// @param[in] len Length of the string. +/// @param[out] rettv Location where to save results. +/// +/// @return OK in case of success, FAIL otherwise. +int json_decode_string(const char *const buf, const size_t len, + typval_T *const rettv) + FUNC_ATTR_NONNULL_ALL FUNC_ATTR_WARN_UNUSED_RESULT +{ + vimconv_T conv; + convert_setup(&conv, (char_u *) "utf-8", p_enc); + conv.vc_fail = true; + int ret = OK; + ValuesStack stack; + kv_init(stack); + ContainerStack container_stack; + kv_init(container_stack); + rettv->v_type = VAR_UNKNOWN; + const char *const e = buf + len; + bool didcomma = false; + bool didcolon = false; +#define POP(obj) \ + do { \ + if (json_decoder_pop(obj, &stack, &container_stack, p) == FAIL) { \ + goto json_decode_string_fail; \ + } \ + } while (0) + const char *p = buf; + for (; p < e; p++) { + switch (*p) { + case '}': + case ']': { + if (kv_size(container_stack) == 0) { + EMSG2(_("E474: No container to close: %s"), p); + goto json_decode_string_fail; + } + ContainerStackItem last_container = kv_last(container_stack); + if (*p == '}' && last_container.container.v_type != VAR_DICT) { + EMSG2(_("E474: Closing list with figure brace: %s"), p); + goto json_decode_string_fail; + } else if (*p == ']' && last_container.container.v_type != VAR_LIST) { + EMSG2(_("E474: Closing dictionary with bracket: %s"), p); + goto json_decode_string_fail; + } else if (didcomma) { + EMSG2(_("E474: Trailing comma: %s"), p); + goto json_decode_string_fail; + } else if (didcolon) { + EMSG2(_("E474: Expected value after colon: %s"), p); + goto json_decode_string_fail; + } else if (last_container.stack_index != kv_size(stack) - 1) { + assert(last_container.stack_index < kv_size(stack) - 1); + EMSG2(_("E474: Expected value: %s"), p); + goto json_decode_string_fail; + } + if (kv_size(stack) == 1) { + p++; + kv_pop(container_stack); + goto json_decode_string_after_cycle; + } else { + typval_T obj = kv_pop(stack); + POP(obj); + break; + } + } + case ',': { + if (kv_size(container_stack) == 0) { + EMSG2(_("E474: Comma not inside container: %s"), p); + goto json_decode_string_fail; + } + ContainerStackItem last_container = kv_last(container_stack); + if (didcomma) { + EMSG2(_("E474: Duplicate comma: %s"), p); + goto json_decode_string_fail; + } else if (didcolon) { + EMSG2(_("E474: Comma after colon: %s"), p); + goto json_decode_string_fail; + } if (last_container.container.v_type == VAR_DICT + && last_container.stack_index != kv_size(stack) - 1) { + EMSG2(_("E474: Using comma in place of colon: %s"), p); + goto json_decode_string_fail; + } else if ((last_container.container.v_type == VAR_DICT + && (last_container.container.vval.v_dict->dv_hashtab.ht_used + == 0)) + || (last_container.container.v_type == VAR_LIST + && last_container.container.vval.v_list->lv_len == 0)) { + EMSG2(_("E474: Leading comma: %s"), p); + goto json_decode_string_fail; + } + didcomma = true; + continue; + } + case ':': { + if (kv_size(container_stack) == 0) { + EMSG2(_("E474: Colon not inside container: %s"), p); + goto json_decode_string_fail; + } + ContainerStackItem last_container = kv_last(container_stack); + if (last_container.container.v_type != VAR_DICT) { + EMSG2(_("E474: Using colon not in dictionary: %s"), p); + goto json_decode_string_fail; + } else if (last_container.stack_index != kv_size(stack) - 2) { + EMSG2(_("E474: Unexpected colon: %s"), p); + goto json_decode_string_fail; + } else if (didcomma) { + EMSG2(_("E474: Colon after comma: %s"), p); + goto json_decode_string_fail; + } else if (didcolon) { + EMSG2(_("E474: Duplicate colon: %s"), p); + goto json_decode_string_fail; + } + didcolon = true; + continue; + } + case ' ': + case TAB: + case NL: { + continue; + } + case 'n': { + if (strncmp(p + 1, "ull", 3) != 0) { + EMSG2(_("E474: Expected null: %s"), p); + goto json_decode_string_fail; + } + p += 3; + POP(get_vim_var_tv(VV_NULL)); + break; + } + case 't': { + if (strncmp(p + 1, "rue", 3) != 0) { + EMSG2(_("E474: Expected true: %s"), p); + goto json_decode_string_fail; + } + p += 3; + POP(get_vim_var_tv(VV_TRUE)); + break; + } + case 'f': { + if (strncmp(p + 1, "alse", 4) != 0) { + EMSG2(_("E474: Expected false: %s"), p); + goto json_decode_string_fail; + } + p += 4; + POP(get_vim_var_tv(VV_FALSE)); + break; + } + case '"': { + size_t len = 0; + const char *s; + for (s = ++p; p < e && *p != '"'; p++) { + if (*p == '\\') { + p++; + if (p == e) { + EMSG2(_("E474: Unfinished escape sequence: %s"), buf); + goto json_decode_string_fail; + } + switch (*p) { + case 'u': { + if (p + 4 >= e) { + EMSG2(_("E474: Unfinished unicode escape sequence: %s"), buf); + goto json_decode_string_fail; + } else if (!ascii_isxdigit(p[1]) + || !ascii_isxdigit(p[2]) + || !ascii_isxdigit(p[3]) + || !ascii_isxdigit(p[4])) { + EMSG2(_("E474: Expected four hex digits after \\u: %s"), + p - 1); + goto json_decode_string_fail; + } + // One UTF-8 character below U+10000 can take up to 3 bytes + len += 3; + p += 4; + break; + } + case '\\': + case '/': + case '"': + case 't': + case 'b': + case 'n': + case 'r': + case 'f': { + len++; + break; + } + default: { + EMSG2(_("E474: Unknown escape sequence: %s"), p - 1); + goto json_decode_string_fail; + } + } + } else { + len++; + } + } + if (*p != '"') { + EMSG2(_("E474: Expected string end: %s"), buf); + goto json_decode_string_fail; + } + char *str = xmalloc(len + 1); + uint16_t fst_in_pair = 0; + char *str_end = str; + for (const char *t = s; t < p; t++) { + if (t[0] != '\\' || t[1] != 'u') { + if (fst_in_pair != 0) { + str_end += utf_char2bytes((int) fst_in_pair, (char_u *) str_end); + fst_in_pair = 0; + } + } + if (*t == '\\') { + t++; + switch (*t) { + case 'u': { + char ubuf[] = { t[1], t[2], t[3], t[4], 0 }; + t += 4; + unsigned long ch; + vim_str2nr((char_u *) ubuf, NULL, NULL, 0, 0, 2, NULL, &ch); + if (0xD800UL <= ch && ch <= 0xDB7FUL) { + fst_in_pair = (uint16_t) ch; + } else if (0xDC00ULL <= ch && ch <= 0xDB7FUL) { + if (fst_in_pair != 0) { + int full_char = ( + (int) (ch - 0xDC00UL) + + (((int) (fst_in_pair - 0xD800)) << 10) + ); + str_end += utf_char2bytes(full_char, (char_u *) str_end); + } + } else { + str_end += utf_char2bytes((int) ch, (char_u *) str_end); + } + break; + } + case '\\': + case '/': + case '"': + case 't': + case 'b': + case 'n': + case 'r': + case 'f': { + static const char escapes[] = { + ['\\'] = '\\', + ['/'] = '/', + ['"'] = '"', + ['t'] = TAB, + ['b'] = BS, + ['n'] = NL, + ['r'] = CAR, + ['f'] = FF, + }; + *str_end++ = escapes[(int) *t]; + break; + } + default: { + assert(false); + } + } + } else { + *str_end++ = *t; + } + } + if (fst_in_pair != 0) { + str_end += utf_char2bytes((int) fst_in_pair, (char_u *) str_end); + } + if (conv.vc_type != CONV_NONE) { + size_t len = (size_t) (str_end - str); + char *const new_str = (char *) string_convert(&conv, (char_u *) str, + &len); + if (new_str == NULL) { + EMSG2(_("E474: Failed to convert string \"%s\" from UTF-8"), str); + xfree(str); + goto json_decode_string_fail; + } + xfree(str); + str = new_str; + str_end = new_str + len; + } + *str_end = NUL; + // TODO: return special string in case of NUL bytes + POP(((typval_T) { + .v_type = VAR_STRING, + .vval = { .v_string = (char_u *) str, }, + })); + break; + } + case '-': + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': { + // a.bE[+-]exp + const char *const s = p; + const char *ints = NULL; + const char *fracs = NULL; + const char *exps = NULL; + if (*p == '-') { + p++; + } + ints = p; + while (p < e && ascii_isdigit(*p)) { + p++; + } + if (p < e && *p == '.') { + p++; + fracs = p; + while (p < e && ascii_isdigit(*p)) { + p++; + } + if (p < e && (*p == 'e' || *p == 'E')) { + p++; + if (p < e && (*p == '-' || *p == '+')) { + p++; + } + exps = p; + while (p < e && ascii_isdigit(*p)) { + p++; + } + } + } + if (p == ints) { + EMSG2(_("E474: Missing number after minus sign: %s"), s); + goto json_decode_string_fail; + } else if (p == fracs) { + EMSG2(_("E474: Missing number after decimal dot: %s"), s); + goto json_decode_string_fail; + } else if (p == exps) { + EMSG2(_("E474: Missing exponent: %s"), s); + goto json_decode_string_fail; + } + typval_T tv = { + .v_type = VAR_NUMBER, + .v_lock = VAR_UNLOCKED, + }; + if (fracs) { + // Convert floating-point number + (void) string2float(s, &tv.vval.v_float); + tv.v_type = VAR_FLOAT; + } else { + // Convert integer + long nr; + vim_str2nr((char_u *) s, NULL, NULL, 0, 0, 0, &nr, NULL); + tv.vval.v_number = (varnumber_T) nr; + } + POP(tv); + p--; + break; + } + case '[': { + list_T *list = list_alloc(); + list->lv_refcount++; + typval_T tv = { + .v_type = VAR_LIST, + .v_lock = VAR_UNLOCKED, + .vval = { .v_list = list }, + }; + kv_push(ContainerStackItem, container_stack, ((ContainerStackItem) { + .stack_index = kv_size(stack), + .container = tv, + })); + kv_push(typval_T, stack, tv); + break; + } + case '{': { + dict_T *dict = dict_alloc(); + dict->dv_refcount++; + typval_T tv = { + .v_type = VAR_DICT, + .v_lock = VAR_UNLOCKED, + .vval = { .v_dict = dict }, + }; + kv_push(ContainerStackItem, container_stack, ((ContainerStackItem) { + .stack_index = kv_size(stack), + .container = tv, + })); + kv_push(typval_T, stack, tv); + break; + } + default: { + EMSG2(_("E474: Unidentified byte: %s"), p); + goto json_decode_string_fail; + } + } + didcomma = false; + didcolon = false; + if (kv_size(container_stack) == 0) { + p++; + break; + } + } +#undef POP +json_decode_string_after_cycle: + for (; p < e; p++) { + switch (*p) { + case NL: + case ' ': + case TAB: { + break; + } + default: { + EMSG2(_("E474: Trailing characters: %s"), p); + goto json_decode_string_fail; + } + } + } + if (kv_size(stack) > 1 || kv_size(container_stack)) { + EMSG2(_("E474: Unexpected end of input: %s"), buf); + goto json_decode_string_fail; + } + goto json_decode_string_ret; +json_decode_string_fail: + ret = FAIL; + while (kv_size(stack)) { + clear_tv(&kv_pop(stack)); + } +json_decode_string_ret: + if (ret != FAIL) { + assert(kv_size(stack) == 1); + *rettv = kv_pop(stack); + } + kv_destroy(stack); + kv_destroy(container_stack); + return ret; +} + +/// Convert msgpack object to a VimL one +int msgpack_to_vim(const msgpack_object mobj, typval_T *const rettv) + FUNC_ATTR_NONNULL_ALL FUNC_ATTR_WARN_UNUSED_RESULT +{ +#define INIT_SPECIAL_DICT(tv, type, val) \ + do { \ + dict_T *const dict = dict_alloc(); \ + dictitem_T *const type_di = dictitem_alloc((char_u *) "_TYPE"); \ + type_di->di_tv.v_type = VAR_LIST; \ + type_di->di_tv.v_lock = 0; \ + type_di->di_tv.vval.v_list = (list_T *) eval_msgpack_type_lists[type]; \ + type_di->di_tv.vval.v_list->lv_refcount++; \ + dict_add(dict, type_di); \ + dictitem_T *const val_di = dictitem_alloc((char_u *) "_VAL"); \ + val_di->di_tv = val; \ + dict_add(dict, val_di); \ + tv->v_type = VAR_DICT; \ + dict->dv_refcount++; \ + tv->vval.v_dict = dict; \ + } while (0) + switch (mobj.type) { + case MSGPACK_OBJECT_NIL: { + INIT_SPECIAL_DICT(rettv, kMPNil, ((typval_T) { + .v_type = VAR_NUMBER, + .v_lock = 0, + .vval = { .v_number = 0 }, + })); + break; + } + case MSGPACK_OBJECT_BOOLEAN: { + INIT_SPECIAL_DICT(rettv, kMPBoolean, + ((typval_T) { + .v_type = VAR_NUMBER, + .v_lock = 0, + .vval = { + .v_number = (varnumber_T) mobj.via.boolean, + }, + })); + break; + } + case MSGPACK_OBJECT_POSITIVE_INTEGER: { + if (mobj.via.u64 <= VARNUMBER_MAX) { + *rettv = (typval_T) { + .v_type = VAR_NUMBER, + .v_lock = 0, + .vval = { .v_number = (varnumber_T) mobj.via.u64 }, + }; + } else { + list_T *const list = list_alloc(); + list->lv_refcount++; + INIT_SPECIAL_DICT(rettv, kMPInteger, + ((typval_T) { + .v_type = VAR_LIST, + .v_lock = 0, + .vval = { .v_list = list }, + })); + uint64_t n = mobj.via.u64; + list_append_number(list, 1); + list_append_number(list, (varnumber_T) ((n >> 62) & 0x3)); + list_append_number(list, (varnumber_T) ((n >> 31) & 0x7FFFFFFF)); + list_append_number(list, (varnumber_T) (n & 0x7FFFFFFF)); + } + break; + } + case MSGPACK_OBJECT_NEGATIVE_INTEGER: { + if (mobj.via.i64 >= VARNUMBER_MIN) { + *rettv = (typval_T) { + .v_type = VAR_NUMBER, + .v_lock = 0, + .vval = { .v_number = (varnumber_T) mobj.via.i64 }, + }; + } else { + list_T *const list = list_alloc(); + list->lv_refcount++; + INIT_SPECIAL_DICT(rettv, kMPInteger, + ((typval_T) { + .v_type = VAR_LIST, + .v_lock = 0, + .vval = { .v_list = list }, + })); + uint64_t n = -((uint64_t) mobj.via.i64); + list_append_number(list, -1); + list_append_number(list, (varnumber_T) ((n >> 62) & 0x3)); + list_append_number(list, (varnumber_T) ((n >> 31) & 0x7FFFFFFF)); + list_append_number(list, (varnumber_T) (n & 0x7FFFFFFF)); + } + break; + } + case MSGPACK_OBJECT_FLOAT: { + *rettv = (typval_T) { + .v_type = VAR_FLOAT, + .v_lock = 0, + .vval = { .v_float = mobj.via.f64 }, + }; + break; + } + case MSGPACK_OBJECT_STR: { + list_T *const list = list_alloc(); + list->lv_refcount++; + INIT_SPECIAL_DICT(rettv, kMPString, + ((typval_T) { + .v_type = VAR_LIST, + .v_lock = 0, + .vval = { .v_list = list }, + })); + if (encode_list_write((void *) list, mobj.via.str.ptr, mobj.via.str.size) + == -1) { + return FAIL; + } + break; + } + case MSGPACK_OBJECT_BIN: { + if (memchr(mobj.via.bin.ptr, NUL, mobj.via.bin.size) == NULL) { + *rettv = (typval_T) { + .v_type = VAR_STRING, + .v_lock = 0, + .vval = { .v_string = xmemdupz(mobj.via.bin.ptr, mobj.via.bin.size) }, + }; + break; + } + list_T *const list = list_alloc(); + list->lv_refcount++; + INIT_SPECIAL_DICT(rettv, kMPBinary, + ((typval_T) { + .v_type = VAR_LIST, + .v_lock = 0, + .vval = { .v_list = list }, + })); + if (encode_list_write((void *) list, mobj.via.bin.ptr, mobj.via.bin.size) + == -1) { + return FAIL; + } + break; + } + case MSGPACK_OBJECT_ARRAY: { + list_T *const list = list_alloc(); + list->lv_refcount++; + *rettv = (typval_T) { + .v_type = VAR_LIST, + .v_lock = 0, + .vval = { .v_list = list }, + }; + for (size_t i = 0; i < mobj.via.array.size; i++) { + listitem_T *const li = listitem_alloc(); + li->li_tv.v_type = VAR_UNKNOWN; + list_append(list, li); + if (msgpack_to_vim(mobj.via.array.ptr[i], &li->li_tv) == FAIL) { + return FAIL; + } + } + break; + } + case MSGPACK_OBJECT_MAP: { + for (size_t i = 0; i < mobj.via.map.size; i++) { + if (mobj.via.map.ptr[i].key.type != MSGPACK_OBJECT_STR + || mobj.via.map.ptr[i].key.via.str.size == 0 + || memchr(mobj.via.map.ptr[i].key.via.str.ptr, NUL, + mobj.via.map.ptr[i].key.via.str.size) != NULL) { + goto msgpack_to_vim_generic_map; + } + } + dict_T *const dict = dict_alloc(); + dict->dv_refcount++; + *rettv = (typval_T) { + .v_type = VAR_DICT, + .v_lock = 0, + .vval = { .v_dict = dict }, + }; + for (size_t i = 0; i < mobj.via.map.size; i++) { + dictitem_T *const di = xmallocz(offsetof(dictitem_T, di_key) + + mobj.via.map.ptr[i].key.via.str.size); + memcpy(&di->di_key[0], mobj.via.map.ptr[i].key.via.str.ptr, + mobj.via.map.ptr[i].key.via.str.size); + di->di_tv.v_type = VAR_UNKNOWN; + if (dict_add(dict, di) == FAIL) { + // Duplicate key: fallback to generic map + clear_tv(rettv); + xfree(di); + goto msgpack_to_vim_generic_map; + } + if (msgpack_to_vim(mobj.via.map.ptr[i].val, &di->di_tv) == FAIL) { + return FAIL; + } + } + break; +msgpack_to_vim_generic_map: {} + list_T *const list = list_alloc(); + list->lv_refcount++; + INIT_SPECIAL_DICT(rettv, kMPMap, + ((typval_T) { + .v_type = VAR_LIST, + .v_lock = 0, + .vval = { .v_list = list }, + })); + for (size_t i = 0; i < mobj.via.map.size; i++) { + list_T *const kv_pair = list_alloc(); + list_append_list(list, kv_pair); + listitem_T *const key_li = listitem_alloc(); + key_li->li_tv.v_type = VAR_UNKNOWN; + list_append(kv_pair, key_li); + listitem_T *const val_li = listitem_alloc(); + val_li->li_tv.v_type = VAR_UNKNOWN; + list_append(kv_pair, val_li); + if (msgpack_to_vim(mobj.via.map.ptr[i].key, &key_li->li_tv) == FAIL) { + return FAIL; + } + if (msgpack_to_vim(mobj.via.map.ptr[i].val, &val_li->li_tv) == FAIL) { + return FAIL; + } + } + break; + } + case MSGPACK_OBJECT_EXT: { + list_T *const list = list_alloc(); + list->lv_refcount++; + list_append_number(list, mobj.via.ext.type); + list_T *const ext_val_list = list_alloc(); + list_append_list(list, ext_val_list); + INIT_SPECIAL_DICT(rettv, kMPExt, + ((typval_T) { + .v_type = VAR_LIST, + .v_lock = 0, + .vval = { .v_list = list }, + })); + if (encode_list_write((void *) ext_val_list, mobj.via.ext.ptr, + mobj.via.ext.size) == -1) { + return FAIL; + } + break; + } + } +#undef INIT_SPECIAL_DICT + return OK; +} diff --git a/src/nvim/eval/decode.h b/src/nvim/eval/decode.h new file mode 100644 index 0000000000..5c25a64f7a --- /dev/null +++ b/src/nvim/eval/decode.h @@ -0,0 +1,13 @@ +#ifndef NVIM_EVAL_DECODE_H +#define NVIM_EVAL_DECODE_H + +#include + +#include + +#include "nvim/eval_defs.h" + +#ifdef INCLUDE_GENERATED_DECLARATIONS +# include "eval/decode.h.generated.h" +#endif +#endif // NVIM_EVAL_DECODE_H diff --git a/src/nvim/eval/encode.c b/src/nvim/eval/encode.c index e23e68dc62..359c9b3de7 100644 --- a/src/nvim/eval/encode.c +++ b/src/nvim/eval/encode.c @@ -65,7 +65,7 @@ const char *const encode_special_var_names[] = { #endif /// Msgpack callback for writing to readfile()-style list -int msgpack_list_write(void *data, const char *buf, size_t len) +int encode_list_write(void *data, const char *buf, size_t len) { if (len == 0) { return 0; diff --git a/src/nvim/shada.c b/src/nvim/shada.c index f402b75ad6..a18fd725d2 100644 --- a/src/nvim/shada.c +++ b/src/nvim/shada.c @@ -31,7 +31,6 @@ #include "nvim/misc2.h" #include "nvim/ex_getln.h" #include "nvim/search.h" -#include "nvim/eval.h" #include "nvim/regexp.h" #include "nvim/eval_defs.h" #include "nvim/version.h" @@ -40,6 +39,7 @@ #include "nvim/strings.h" #include "nvim/quickfix.h" #include "nvim/eval/encode.h" +#include "nvim/eval/decode.h" #include "nvim/lib/khash.h" #include "nvim/lib/kvec.h" From ed6756563ca652581f2be14b9f90e55a3c83461b Mon Sep 17 00:00:00 2001 From: ZyX Date: Wed, 3 Feb 2016 19:56:37 +0300 Subject: [PATCH 17/82] eval/decode: Replace INIT_SPECIAL_DICT macros with inline function --- src/nvim/eval/decode.c | 136 +++++++++++++++++++++-------------------- 1 file changed, 70 insertions(+), 66 deletions(-) diff --git a/src/nvim/eval/decode.c b/src/nvim/eval/decode.c index e2b2574d83..2d295eabab 100644 --- a/src/nvim/eval/decode.c +++ b/src/nvim/eval/decode.c @@ -25,6 +25,34 @@ typedef kvec_t(ContainerStackItem) ContainerStack; # include "eval/decode.c.generated.h" #endif +/// Create special dictionary +/// +/// @param[out] rettv Location where created dictionary will be saved. +/// @param[in] type Type of the dictionary. +/// @param[in] val Value associated with the _VAL key. +static inline void create_special_dict(typval_T *const rettv, + const MessagePackType type, + typval_T val) + FUNC_ATTR_NONNULL_ALL +{ + dict_T *const dict = dict_alloc(); + dictitem_T *const type_di = dictitem_alloc((char_u *) "_TYPE"); + type_di->di_tv.v_type = VAR_LIST; + type_di->di_tv.v_lock = 0; + type_di->di_tv.vval.v_list = (list_T *) eval_msgpack_type_lists[type]; + type_di->di_tv.vval.v_list->lv_refcount++; + dict_add(dict, type_di); + dictitem_T *const val_di = dictitem_alloc((char_u *) "_VAL"); + val_di->di_tv = val; + dict_add(dict, val_di); + dict->dv_refcount++; + *rettv = (typval_T) { + .v_type = VAR_DICT, + .v_lock = VAR_UNLOCKED, + .vval = { .v_dict = dict }, + }; +} + /// Helper function used for working with stack vectors used by JSON decoder /// /// @param[in] obj New object. @@ -521,40 +549,23 @@ json_decode_string_ret: int msgpack_to_vim(const msgpack_object mobj, typval_T *const rettv) FUNC_ATTR_NONNULL_ALL FUNC_ATTR_WARN_UNUSED_RESULT { -#define INIT_SPECIAL_DICT(tv, type, val) \ - do { \ - dict_T *const dict = dict_alloc(); \ - dictitem_T *const type_di = dictitem_alloc((char_u *) "_TYPE"); \ - type_di->di_tv.v_type = VAR_LIST; \ - type_di->di_tv.v_lock = 0; \ - type_di->di_tv.vval.v_list = (list_T *) eval_msgpack_type_lists[type]; \ - type_di->di_tv.vval.v_list->lv_refcount++; \ - dict_add(dict, type_di); \ - dictitem_T *const val_di = dictitem_alloc((char_u *) "_VAL"); \ - val_di->di_tv = val; \ - dict_add(dict, val_di); \ - tv->v_type = VAR_DICT; \ - dict->dv_refcount++; \ - tv->vval.v_dict = dict; \ - } while (0) switch (mobj.type) { case MSGPACK_OBJECT_NIL: { - INIT_SPECIAL_DICT(rettv, kMPNil, ((typval_T) { - .v_type = VAR_NUMBER, - .v_lock = 0, - .vval = { .v_number = 0 }, - })); + create_special_dict(rettv, kMPNil, ((typval_T) { + .v_type = VAR_NUMBER, + .v_lock = 0, + .vval = { .v_number = 0 }, + })); break; } case MSGPACK_OBJECT_BOOLEAN: { - INIT_SPECIAL_DICT(rettv, kMPBoolean, - ((typval_T) { - .v_type = VAR_NUMBER, - .v_lock = 0, - .vval = { - .v_number = (varnumber_T) mobj.via.boolean, - }, - })); + create_special_dict(rettv, kMPBoolean, ((typval_T) { + .v_type = VAR_NUMBER, + .v_lock = 0, + .vval = { + .v_number = (varnumber_T) mobj.via.boolean, + }, + })); break; } case MSGPACK_OBJECT_POSITIVE_INTEGER: { @@ -567,12 +578,11 @@ int msgpack_to_vim(const msgpack_object mobj, typval_T *const rettv) } else { list_T *const list = list_alloc(); list->lv_refcount++; - INIT_SPECIAL_DICT(rettv, kMPInteger, - ((typval_T) { - .v_type = VAR_LIST, - .v_lock = 0, - .vval = { .v_list = list }, - })); + create_special_dict(rettv, kMPInteger, ((typval_T) { + .v_type = VAR_LIST, + .v_lock = 0, + .vval = { .v_list = list }, + })); uint64_t n = mobj.via.u64; list_append_number(list, 1); list_append_number(list, (varnumber_T) ((n >> 62) & 0x3)); @@ -591,12 +601,11 @@ int msgpack_to_vim(const msgpack_object mobj, typval_T *const rettv) } else { list_T *const list = list_alloc(); list->lv_refcount++; - INIT_SPECIAL_DICT(rettv, kMPInteger, - ((typval_T) { - .v_type = VAR_LIST, - .v_lock = 0, - .vval = { .v_list = list }, - })); + create_special_dict(rettv, kMPInteger, ((typval_T) { + .v_type = VAR_LIST, + .v_lock = 0, + .vval = { .v_list = list }, + })); uint64_t n = -((uint64_t) mobj.via.i64); list_append_number(list, -1); list_append_number(list, (varnumber_T) ((n >> 62) & 0x3)); @@ -616,12 +625,11 @@ int msgpack_to_vim(const msgpack_object mobj, typval_T *const rettv) case MSGPACK_OBJECT_STR: { list_T *const list = list_alloc(); list->lv_refcount++; - INIT_SPECIAL_DICT(rettv, kMPString, - ((typval_T) { - .v_type = VAR_LIST, - .v_lock = 0, - .vval = { .v_list = list }, - })); + create_special_dict(rettv, kMPString, ((typval_T) { + .v_type = VAR_LIST, + .v_lock = 0, + .vval = { .v_list = list }, + })); if (encode_list_write((void *) list, mobj.via.str.ptr, mobj.via.str.size) == -1) { return FAIL; @@ -639,12 +647,11 @@ int msgpack_to_vim(const msgpack_object mobj, typval_T *const rettv) } list_T *const list = list_alloc(); list->lv_refcount++; - INIT_SPECIAL_DICT(rettv, kMPBinary, - ((typval_T) { - .v_type = VAR_LIST, - .v_lock = 0, - .vval = { .v_list = list }, - })); + create_special_dict(rettv, kMPBinary, ((typval_T) { + .v_type = VAR_LIST, + .v_lock = 0, + .vval = { .v_list = list }, + })); if (encode_list_write((void *) list, mobj.via.bin.ptr, mobj.via.bin.size) == -1) { return FAIL; @@ -705,12 +712,11 @@ int msgpack_to_vim(const msgpack_object mobj, typval_T *const rettv) msgpack_to_vim_generic_map: {} list_T *const list = list_alloc(); list->lv_refcount++; - INIT_SPECIAL_DICT(rettv, kMPMap, - ((typval_T) { - .v_type = VAR_LIST, - .v_lock = 0, - .vval = { .v_list = list }, - })); + create_special_dict(rettv, kMPMap, ((typval_T) { + .v_type = VAR_LIST, + .v_lock = 0, + .vval = { .v_list = list }, + })); for (size_t i = 0; i < mobj.via.map.size; i++) { list_T *const kv_pair = list_alloc(); list_append_list(list, kv_pair); @@ -735,12 +741,11 @@ msgpack_to_vim_generic_map: {} list_append_number(list, mobj.via.ext.type); list_T *const ext_val_list = list_alloc(); list_append_list(list, ext_val_list); - INIT_SPECIAL_DICT(rettv, kMPExt, - ((typval_T) { - .v_type = VAR_LIST, - .v_lock = 0, - .vval = { .v_list = list }, - })); + create_special_dict(rettv, kMPExt, ((typval_T) { + .v_type = VAR_LIST, + .v_lock = 0, + .vval = { .v_list = list }, + })); if (encode_list_write((void *) ext_val_list, mobj.via.ext.ptr, mobj.via.ext.size) == -1) { return FAIL; @@ -748,6 +753,5 @@ msgpack_to_vim_generic_map: {} break; } } -#undef INIT_SPECIAL_DICT return OK; } From cddd7d47c325ab0c06c21fd101efe4a9a1708fca Mon Sep 17 00:00:00 2001 From: ZyX Date: Wed, 3 Feb 2016 20:04:16 +0300 Subject: [PATCH 18/82] eval/decode: Make msgpackparse() function use new v: vars --- runtime/doc/eval.txt | 10 +++++++--- src/nvim/eval/decode.c | 14 ++------------ test/functional/eval/msgpack_functions_spec.lua | 15 ++++++--------- 3 files changed, 15 insertions(+), 24 deletions(-) diff --git a/runtime/doc/eval.txt b/runtime/doc/eval.txt index 25005885c3..4676080536 100644 --- a/runtime/doc/eval.txt +++ b/runtime/doc/eval.txt @@ -4907,9 +4907,13 @@ msgpackparse({list}) {Nvim} *msgpackparse()* contains name of the key from |v:msgpack_types|): Key Value ~ - nil Zero, ignored when dumping. - boolean One or zero. When dumping it is only checked that - value is a |Number|. + nil Zero, ignored when dumping. This value cannot + possibly appear in |msgpackparse()| output in Neovim + versions which have |v:null|. + boolean One or zero. When dumping it is only checked that + value is a |Number|. This value cannot possibly + appear in |msgpackparse()| output in Neovim versions + which have |v:true| and |v:false|. integer |List| with four numbers: sign (-1 or 1), highest two bits, number with bits from 62nd to 31st, lowest 31 bits. I.e. to get actual number one will need to use diff --git a/src/nvim/eval/decode.c b/src/nvim/eval/decode.c index 2d295eabab..23e7752ecc 100644 --- a/src/nvim/eval/decode.c +++ b/src/nvim/eval/decode.c @@ -551,21 +551,11 @@ int msgpack_to_vim(const msgpack_object mobj, typval_T *const rettv) { switch (mobj.type) { case MSGPACK_OBJECT_NIL: { - create_special_dict(rettv, kMPNil, ((typval_T) { - .v_type = VAR_NUMBER, - .v_lock = 0, - .vval = { .v_number = 0 }, - })); + *rettv = get_vim_var_tv(VV_NULL); break; } case MSGPACK_OBJECT_BOOLEAN: { - create_special_dict(rettv, kMPBoolean, ((typval_T) { - .v_type = VAR_NUMBER, - .v_lock = 0, - .vval = { - .v_number = (varnumber_T) mobj.via.boolean, - }, - })); + *rettv = get_vim_var_tv(mobj.via.boolean ? VV_TRUE : VV_FALSE); break; } case MSGPACK_OBJECT_POSITIVE_INTEGER: { diff --git a/test/functional/eval/msgpack_functions_spec.lua b/test/functional/eval/msgpack_functions_spec.lua index 2c73a144ca..3d539d855d 100644 --- a/test/functional/eval/msgpack_functions_spec.lua +++ b/test/functional/eval/msgpack_functions_spec.lua @@ -393,25 +393,22 @@ end) describe('msgpackparse() function', function() before_each(clear) - it('restores nil as special dict', function() + it('restores nil as v:null', function() execute('let dumped = ["\\xC0"]') execute('let parsed = msgpackparse(dumped)') - eq({{_TYPE={}, _VAL=0}}, eval('parsed')) - eq(1, eval('g:parsed[0]._TYPE is v:msgpack_types.nil')) + eq('[v:null]', eval('string(parsed)')) end) - it('restores boolean false as zero', function() + it('restores boolean false as v:false', function() execute('let dumped = ["\\xC2"]') execute('let parsed = msgpackparse(dumped)') - eq({{_TYPE={}, _VAL=0}}, eval('parsed')) - eq(1, eval('g:parsed[0]._TYPE is v:msgpack_types.boolean')) + eq({false}, eval('parsed')) end) - it('restores boolean true as one', function() + it('restores boolean true as v:true', function() execute('let dumped = ["\\xC3"]') execute('let parsed = msgpackparse(dumped)') - eq({{_TYPE={}, _VAL=1}}, eval('parsed')) - eq(1, eval('g:parsed[0]._TYPE is v:msgpack_types.boolean')) + eq({true}, eval('parsed')) end) it('restores FIXSTR as special dict', function() From ea82270d30eef2dd716cd158d989f96fbd503ba6 Mon Sep 17 00:00:00 2001 From: ZyX Date: Wed, 3 Feb 2016 21:01:21 +0300 Subject: [PATCH 19/82] eval/decode: Fail on control and invalid unicode characters --- src/nvim/eval/decode.c | 35 +++++++++-- test/functional/eval/json_functions_spec.lua | 61 ++++++++++++++++++++ 2 files changed, 91 insertions(+), 5 deletions(-) diff --git a/src/nvim/eval/decode.c b/src/nvim/eval/decode.c index 23e7752ecc..29841db1b6 100644 --- a/src/nvim/eval/decode.c +++ b/src/nvim/eval/decode.c @@ -264,8 +264,8 @@ int json_decode_string(const char *const buf, const size_t len, } case '"': { size_t len = 0; - const char *s; - for (s = ++p; p < e && *p != '"'; p++) { + const char *const s = ++p; + while (p < e && *p != '"') { if (*p == '\\') { p++; if (p == e) { @@ -285,9 +285,10 @@ int json_decode_string(const char *const buf, const size_t len, p - 1); goto json_decode_string_fail; } - // One UTF-8 character below U+10000 can take up to 3 bytes + // One UTF-8 character below U+10000 can take up to 3 bytes, + // above up to 6, but they are encoded using two \u escapes. len += 3; - p += 4; + p += 5; break; } case '\\': @@ -299,6 +300,7 @@ int json_decode_string(const char *const buf, const size_t len, case 'r': case 'f': { len++; + p++; break; } default: { @@ -307,7 +309,30 @@ int json_decode_string(const char *const buf, const size_t len, } } } else { - len++; + uint8_t p_byte = (uint8_t) *p; + // unescaped = %x20-21 / %x23-5B / %x5D-10FFFF + if (p_byte < 0x20) { + EMSG2(_("E474: ASCII control characters cannot be present " + "inside string: %s"), p); + goto json_decode_string_fail; + } + const int ch = utf_ptr2char((char_u *) p); + // All characters above U+007F are encoded using two or more bytes + // and thus cannot possibly be equal to *p. But utf_ptr2char({0xFF, + // 0}) will return 0xFF, even though 0xFF cannot start any UTF-8 + // code point at all. + if (ch >= 0x80 && p_byte == ch) { + EMSG2(_("E474: Only UTF-8 strings allowed: %s"), p); + goto json_decode_string_fail; + } else if (ch > 0x10FFFF) { + EMSG2(_("E474: Only UTF-8 code points up to U+10FFFF " + "are allowed to appear unescaped: %s"), p); + goto json_decode_string_fail; + } + const size_t ch_len = (size_t) utf_char2len(ch); + assert(ch_len == (size_t) (ch ? utf_ptr2len((char_u *) p) : 1)); + len += ch_len; + p += ch_len; } } if (*p != '"') { diff --git a/test/functional/eval/json_functions_spec.lua b/test/functional/eval/json_functions_spec.lua index f979a6dd7c..9167cb2fef 100644 --- a/test/functional/eval/json_functions_spec.lua +++ b/test/functional/eval/json_functions_spec.lua @@ -235,6 +235,67 @@ describe('jsondecode() function', function() eq('', funcs.jsondecode('""')) eq('\\/"\t\b\n\r\f', funcs.jsondecode([["\\\/\"\t\b\n\r\f"]])) eq('/a', funcs.jsondecode([["\/a"]])) + -- Unicode characters: 2-byte, 3-byte, 4-byte + eq({ + '«', + 'ફ', + '\xF0\x90\x80\x80', + }, funcs.jsondecode({ + '[', + '"«",', + '"ફ",', + '"\xF0\x90\x80\x80"', + ']', + })) + end) + + it('fails on strings with invalid bytes', function() + eq('Vim(call):E474: Only UTF-8 strings allowed: \255"', + exc_exec('call jsondecode("\\t\\"\\xFF\\"")')) + eq('Vim(call):E474: ASCII control characters cannot be present inside string: ', + exc_exec('call jsondecode(["\\"\\n\\""])')) + -- 0xC2 starts 2-byte unicode character + eq('Vim(call):E474: Only UTF-8 strings allowed: \194"', + exc_exec('call jsondecode("\\t\\"\\xC2\\"")')) + -- 0xE0 0xAA starts 3-byte unicode character + eq('Vim(call):E474: Only UTF-8 strings allowed: \224"', + exc_exec('call jsondecode("\\t\\"\\xE0\\"")')) + eq('Vim(call):E474: Only UTF-8 strings allowed: \224\170"', + exc_exec('call jsondecode("\\t\\"\\xE0\\xAA\\"")')) + -- 0xF0 0x90 0x80 starts 4-byte unicode character + eq('Vim(call):E474: Only UTF-8 strings allowed: \240"', + exc_exec('call jsondecode("\\t\\"\\xF0\\"")')) + eq('Vim(call):E474: Only UTF-8 strings allowed: \240\144"', + exc_exec('call jsondecode("\\t\\"\\xF0\\x90\\"")')) + eq('Vim(call):E474: Only UTF-8 strings allowed: \240\144\128"', + exc_exec('call jsondecode("\\t\\"\\xF0\\x90\\x80\\"")')) + -- 0xF9 0x80 0x80 0x80 starts 5-byte unicode character + eq('Vim(call):E474: Only UTF-8 strings allowed: \xF9"', + exc_exec('call jsondecode("\\t\\"\\xF9\\"")')) + eq('Vim(call):E474: Only UTF-8 strings allowed: \xF9\x80"', + exc_exec('call jsondecode("\\t\\"\\xF9\\x80\\"")')) + eq('Vim(call):E474: Only UTF-8 strings allowed: \xF9\x80\x80"', + exc_exec('call jsondecode("\\t\\"\\xF9\\x80\\x80\\"")')) + eq('Vim(call):E474: Only UTF-8 strings allowed: \xF9\x80\x80\x80"', + exc_exec('call jsondecode("\\t\\"\\xF9\\x80\\x80\\x80\\"")')) + -- 0xFC 0x90 0x80 0x80 0x80 starts 6-byte unicode character + eq('Vim(call):E474: Only UTF-8 strings allowed: \xFC"', + exc_exec('call jsondecode("\\t\\"\\xFC\\"")')) + eq('Vim(call):E474: Only UTF-8 strings allowed: \xFC\x90"', + exc_exec('call jsondecode("\\t\\"\\xFC\\x90\\"")')) + eq('Vim(call):E474: Only UTF-8 strings allowed: \xFC\x90\x80"', + exc_exec('call jsondecode("\\t\\"\\xFC\\x90\\x80\\"")')) + eq('Vim(call):E474: Only UTF-8 strings allowed: \xFC\x90\x80\x80"', + exc_exec('call jsondecode("\\t\\"\\xFC\\x90\\x80\\x80\\"")')) + eq('Vim(call):E474: Only UTF-8 strings allowed: \xFC\x90\x80\x80\x80"', + exc_exec('call jsondecode("\\t\\"\\xFC\\x90\\x80\\x80\\x80\\"")')) + -- Specification does not allow unquoted characters above 0x10FFFF + eq('Vim(call):E474: Only UTF-8 code points up to U+10FFFF are allowed to appear unescaped: \xF9\x80\x80\x80\x80"', + exc_exec('call jsondecode("\\t\\"\\xF9\\x80\\x80\\x80\\x80\\"")')) + eq('Vim(call):E474: Only UTF-8 code points up to U+10FFFF are allowed to appear unescaped: \xFC\x90\x80\x80\x80\x80"', + exc_exec('call jsondecode("\\t\\"\\xFC\\x90\\x80\\x80\\x80\\x80\\"")')) + -- '"\xF9\x80\x80\x80\x80"', + -- '"\xFC\x90\x80\x80\x80\x80"', end) end) From 5814e29cdbe370a417d654dbd18620849aa00a09 Mon Sep 17 00:00:00 2001 From: ZyX Date: Wed, 3 Feb 2016 21:46:01 +0300 Subject: [PATCH 20/82] eval/decode: Fix surrogate pairs processing --- src/nvim/eval/decode.c | 27 ++++++++++---------- src/nvim/eval/encode.c | 8 +++--- src/nvim/eval/encode.h | 15 +++++++++++ test/functional/eval/json_functions_spec.lua | 18 +++++++++++++ 4 files changed, 51 insertions(+), 17 deletions(-) diff --git a/src/nvim/eval/decode.c b/src/nvim/eval/decode.c index 29841db1b6..05dc1c97c4 100644 --- a/src/nvim/eval/decode.c +++ b/src/nvim/eval/decode.c @@ -340,12 +340,12 @@ int json_decode_string(const char *const buf, const size_t len, goto json_decode_string_fail; } char *str = xmalloc(len + 1); - uint16_t fst_in_pair = 0; + int fst_in_pair = 0; char *str_end = str; for (const char *t = s; t < p; t++) { if (t[0] != '\\' || t[1] != 'u') { if (fst_in_pair != 0) { - str_end += utf_char2bytes((int) fst_in_pair, (char_u *) str_end); + str_end += utf_char2bytes(fst_in_pair, (char_u *) str_end); fst_in_pair = 0; } } @@ -353,20 +353,21 @@ int json_decode_string(const char *const buf, const size_t len, t++; switch (*t) { case 'u': { - char ubuf[] = { t[1], t[2], t[3], t[4], 0 }; + const char ubuf[] = { t[1], t[2], t[3], t[4], 0 }; t += 4; unsigned long ch; vim_str2nr((char_u *) ubuf, NULL, NULL, 0, 0, 2, NULL, &ch); - if (0xD800UL <= ch && ch <= 0xDB7FUL) { - fst_in_pair = (uint16_t) ch; - } else if (0xDC00ULL <= ch && ch <= 0xDB7FUL) { - if (fst_in_pair != 0) { - int full_char = ( - (int) (ch - 0xDC00UL) - + (((int) (fst_in_pair - 0xD800)) << 10) - ); - str_end += utf_char2bytes(full_char, (char_u *) str_end); - } + if (SURROGATE_HI_START <= ch && ch <= SURROGATE_HI_END) { + fst_in_pair = (int) ch; + } else if (SURROGATE_LO_START <= ch && ch <= SURROGATE_LO_END + && fst_in_pair != 0) { + const int full_char = ( + (int) (ch - SURROGATE_LO_START) + + ((fst_in_pair - SURROGATE_HI_START) << 10) + + SURROGATE_FIRST_CHAR + ); + str_end += utf_char2bytes(full_char, (char_u *) str_end); + fst_in_pair = 0; } else { str_end += utf_char2bytes((int) ch, (char_u *) str_end); } diff --git a/src/nvim/eval/encode.c b/src/nvim/eval/encode.c index 359c9b3de7..e44512d803 100644 --- a/src/nvim/eval/encode.c +++ b/src/nvim/eval/encode.c @@ -970,7 +970,7 @@ static inline int convert_to_json_string(garray_T *const gap, default: { if (vim_isprintc(ch)) { ga_concat_len(gap, buf + i, shift); - } else if (ch <= 0xFFFF) { + } else if (ch < SURROGATE_FIRST_CHAR) { ga_concat_len(gap, ((const char []) { '\\', 'u', xdigits[(ch >> (4 * 3)) & 0xF], @@ -979,9 +979,9 @@ static inline int convert_to_json_string(garray_T *const gap, xdigits[(ch >> (4 * 0)) & 0xF], }), sizeof("\\u1234") - 1); } else { - uint32_t tmp = (uint32_t) ch - 0x010000; - uint16_t hi = 0xD800 + ((tmp >> 10) & 0x03FF); - uint16_t lo = 0xDC00 + ((tmp >> 0) & 0x03FF); + uint32_t tmp = (uint32_t) ch - SURROGATE_FIRST_CHAR; + uint16_t hi = SURROGATE_HI_START + ((tmp >> 10) & ((1 << 10) - 1)); + uint16_t lo = SURROGATE_LO_END + ((tmp >> 0) & ((1 << 10) - 1)); ga_concat_len(gap, ((const char []) { '\\', 'u', xdigits[(hi >> (4 * 3)) & 0xF], diff --git a/src/nvim/eval/encode.h b/src/nvim/eval/encode.h index 0e60c96155..9bc665253b 100644 --- a/src/nvim/eval/encode.h +++ b/src/nvim/eval/encode.h @@ -54,6 +54,21 @@ static inline ListReaderState encode_init_lrstate(const list_T *const list) /// Array mapping values from SpecialVarValue enum to names extern const char *const encode_special_var_names[]; +/// First codepoint in high surrogates block +#define SURROGATE_HI_START 0xD800 + +/// Last codepoint in high surrogates block +#define SURROGATE_HI_END 0xDBFF + +/// First codepoint in low surrogates block +#define SURROGATE_LO_START 0xDC00 + +/// Last codepoint in low surrogates block +#define SURROGATE_LO_END 0xDFFF + +/// First character that needs to be encoded as surrogate pair +#define SURROGATE_FIRST_CHAR 0x10000 + #ifdef INCLUDE_GENERATED_DECLARATIONS # include "eval/encode.h.generated.h" #endif diff --git a/test/functional/eval/json_functions_spec.lua b/test/functional/eval/json_functions_spec.lua index 9167cb2fef..651f6c27b3 100644 --- a/test/functional/eval/json_functions_spec.lua +++ b/test/functional/eval/json_functions_spec.lua @@ -297,6 +297,24 @@ describe('jsondecode() function', function() -- '"\xF9\x80\x80\x80\x80"', -- '"\xFC\x90\x80\x80\x80\x80"', end) + + it('parses surrogate pairs properly', function() + eq('\xF0\x90\x80\x80', funcs.jsondecode('"\\uD800\\uDC00"')) + eq('\xED\xA0\x80a\xED\xB0\x80', funcs.jsondecode('"\\uD800a\\uDC00"')) + eq('\xED\xA0\x80\t\xED\xB0\x80', funcs.jsondecode('"\\uD800\\t\\uDC00"')) + + eq('\xED\xA0\x80', funcs.jsondecode('"\\uD800"')) + eq('\xED\xA0\x80a', funcs.jsondecode('"\\uD800a"')) + eq('\xED\xA0\x80\t', funcs.jsondecode('"\\uD800\\t"')) + + eq('\xED\xB0\x80', funcs.jsondecode('"\\uDC00"')) + eq('\xED\xB0\x80a', funcs.jsondecode('"\\uDC00a"')) + eq('\xED\xB0\x80\t', funcs.jsondecode('"\\uDC00\\t"')) + + eq('\xED\xB0\x80', funcs.jsondecode('"\\uDC00"')) + eq('a\xED\xB0\x80', funcs.jsondecode('"a\\uDC00"')) + eq('\t\xED\xB0\x80', funcs.jsondecode('"\\t\\uDC00"')) + end) end) describe('jsonencode() function', function() From 57700def795156ce93526bf12e95a56d90c39206 Mon Sep 17 00:00:00 2001 From: ZyX Date: Wed, 3 Feb 2016 21:55:42 +0300 Subject: [PATCH 21/82] doc: Update documentation regarding converting from/to &encoding --- runtime/doc/eval.txt | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/runtime/doc/eval.txt b/runtime/doc/eval.txt index 4676080536..ee8ede2b91 100644 --- a/runtime/doc/eval.txt +++ b/runtime/doc/eval.txt @@ -4324,7 +4324,9 @@ join({list} [, {sep}]) *join()* jsondecode({expr}) *jsondecode()* Convert {expr} from JSON object. Accepts |readfile()|-style list as the input, as well as regular string. May output any - Vim value. In the following cases it will output + Vim value. When 'encoding' is not UTF-8 string is converted + from UTF-8 to 'encoding', failing conversion fails + jsondecode(). In the following cases it will output |msgpack-special-dict|: 1. Dictionary contains duplicate key. 2. Dictionary contains empty key. @@ -4334,8 +4336,9 @@ jsondecode({expr}) *jsondecode()* jsonencode({expr}) *jsonencode()* Convert {expr} into a JSON string. Accepts - |msgpack-special-dict| as the input. Will not convert - |Funcref|s, mappings with non-string keys (can be created as + |msgpack-special-dict| as the input. Converts from 'encoding' + to UTF-8 when encoding strings. Will not convert |Funcref|s, + mappings with non-string keys (can be created as |msgpack-special-dict|), values with self-referencing containers, strings which contain non-UTF-8 characters, pseudo-UTF-8 strings which contain codepoints reserved for From 634e51d12b90b00dd01b768904d7bf5ade0acbb0 Mon Sep 17 00:00:00 2001 From: ZyX Date: Wed, 3 Feb 2016 21:56:01 +0300 Subject: [PATCH 22/82] eval/*: Fix some linter errors --- src/nvim/eval/decode.c | 9 +++-- src/nvim/eval/encode.c | 79 ++++++++++++++++++++---------------------- 2 files changed, 42 insertions(+), 46 deletions(-) diff --git a/src/nvim/eval/decode.c b/src/nvim/eval/decode.c index 05dc1c97c4..5a3c5709ad 100644 --- a/src/nvim/eval/decode.c +++ b/src/nvim/eval/decode.c @@ -193,8 +193,8 @@ int json_decode_string(const char *const buf, const size_t len, } else if (didcolon) { EMSG2(_("E474: Comma after colon: %s"), p); goto json_decode_string_fail; - } if (last_container.container.v_type == VAR_DICT - && last_container.stack_index != kv_size(stack) - 1) { + } else if (last_container.container.v_type == VAR_DICT + && last_container.stack_index != kv_size(stack) - 1) { EMSG2(_("E474: Using comma in place of colon: %s"), p); goto json_decode_string_fail; } else if ((last_container.container.v_type == VAR_DICT @@ -364,8 +364,7 @@ int json_decode_string(const char *const buf, const size_t len, const int full_char = ( (int) (ch - SURROGATE_LO_START) + ((fst_in_pair - SURROGATE_HI_START) << 10) - + SURROGATE_FIRST_CHAR - ); + + SURROGATE_FIRST_CHAR); str_end += utf_char2bytes(full_char, (char_u *) str_end); fst_in_pair = 0; } else { @@ -763,7 +762,7 @@ msgpack_to_vim_generic_map: {} .vval = { .v_list = list }, })); if (encode_list_write((void *) ext_val_list, mobj.via.ext.ptr, - mobj.via.ext.size) == -1) { + mobj.via.ext.size) == -1) { return FAIL; } break; diff --git a/src/nvim/eval/encode.c b/src/nvim/eval/encode.c index e44512d803..5c843357f2 100644 --- a/src/nvim/eval/encode.c +++ b/src/nvim/eval/encode.c @@ -194,13 +194,13 @@ static int conv_error(const char *const msg, const MPConvStack *const mpstack, /// /// @param[in] list Converted list. /// @param[out] ret_len Resulting buffer length. -/// @param[out] ret_buf Allocated buffer with the result or NULL if ret_len is +/// @param[out] ret_buf Allocated buffer with the result or NULL if ret_len is /// zero. /// /// @return true in case of success, false in case of failure. bool encode_vim_list_to_buf(const list_T *const list, size_t *const ret_len, char **const ret_buf) - FUNC_ATTR_NONNULL_ARG(2,3) FUNC_ATTR_WARN_UNUSED_RESULT + FUNC_ATTR_NONNULL_ARG(2, 3) FUNC_ATTR_WARN_UNUSED_RESULT { size_t len = 0; if (list != NULL) { @@ -336,18 +336,15 @@ static int name##_convert_one_value(firstargtype firstargname, \ } \ CHECK_SELF_REFERENCE(tv->vval.v_list, lv_copyID, kMPConvList); \ CONV_LIST_START(tv->vval.v_list); \ - kv_push( \ - MPConvStackVal, \ - *mpstack, \ - ((MPConvStackVal) { \ - .type = kMPConvList, \ - .data = { \ - .l = { \ - .list = tv->vval.v_list, \ - .li = tv->vval.v_list->lv_first, \ - }, \ - }, \ - })); \ + kv_push(MPConvStackVal, *mpstack, ((MPConvStackVal) { \ + .type = kMPConvList, \ + .data = { \ + .l = { \ + .list = tv->vval.v_list, \ + .li = tv->vval.v_list->lv_first, \ + }, \ + }, \ + })); \ break; \ } \ case VAR_SPECIAL: { \ @@ -475,14 +472,14 @@ static int name##_convert_one_value(firstargtype firstargname, \ kMPConvList); \ CONV_LIST_START(val_di->di_tv.vval.v_list); \ kv_push(MPConvStackVal, *mpstack, ((MPConvStackVal) { \ - .type = kMPConvList, \ - .data = { \ - .l = { \ - .list = val_di->di_tv.vval.v_list, \ - .li = val_di->di_tv.vval.v_list->lv_first, \ - }, \ - }, \ - })); \ + .type = kMPConvList, \ + .data = { \ + .l = { \ + .list = val_di->di_tv.vval.v_list, \ + .li = val_di->di_tv.vval.v_list->lv_first, \ + }, \ + }, \ + })); \ break; \ } \ case kMPMap: { \ @@ -504,14 +501,14 @@ static int name##_convert_one_value(firstargtype firstargname, \ CHECK_SELF_REFERENCE(val_list, lv_copyID, kMPConvPairs); \ CONV_DICT_START(val_list->lv_len); \ kv_push(MPConvStackVal, *mpstack, ((MPConvStackVal) { \ - .type = kMPConvPairs, \ - .data = { \ - .l = { \ - .list = val_list, \ - .li = val_list->lv_first, \ - }, \ - }, \ - })); \ + .type = kMPConvPairs, \ + .data = { \ + .l = { \ + .list = val_list, \ + .li = val_list->lv_first, \ + }, \ + }, \ + })); \ break; \ } \ case kMPExt: { \ @@ -543,15 +540,15 @@ name##_convert_one_value_regular_dict: \ CHECK_SELF_REFERENCE(tv->vval.v_dict, dv_copyID, kMPConvDict); \ CONV_DICT_START(tv->vval.v_dict->dv_hashtab.ht_used); \ kv_push(MPConvStackVal, *mpstack, ((MPConvStackVal) { \ - .type = kMPConvDict, \ - .data = { \ - .d = { \ - .dict = tv->vval.v_dict, \ - .hi = tv->vval.v_dict->dv_hashtab.ht_array, \ - .todo = tv->vval.v_dict->dv_hashtab.ht_used, \ - }, \ - }, \ - })); \ + .type = kMPConvDict, \ + .data = { \ + .d = { \ + .dict = tv->vval.v_dict, \ + .hi = tv->vval.v_dict->dv_hashtab.ht_array, \ + .todo = tv->vval.v_dict->dv_hashtab.ht_used, \ + }, \ + }, \ + })); \ break; \ } \ case VAR_UNKNOWN: { \ @@ -971,7 +968,7 @@ static inline int convert_to_json_string(garray_T *const gap, if (vim_isprintc(ch)) { ga_concat_len(gap, buf + i, shift); } else if (ch < SURROGATE_FIRST_CHAR) { - ga_concat_len(gap, ((const char []) { + ga_concat_len(gap, ((const char[]) { '\\', 'u', xdigits[(ch >> (4 * 3)) & 0xF], xdigits[(ch >> (4 * 2)) & 0xF], @@ -982,7 +979,7 @@ static inline int convert_to_json_string(garray_T *const gap, uint32_t tmp = (uint32_t) ch - SURROGATE_FIRST_CHAR; uint16_t hi = SURROGATE_HI_START + ((tmp >> 10) & ((1 << 10) - 1)); uint16_t lo = SURROGATE_LO_END + ((tmp >> 0) & ((1 << 10) - 1)); - ga_concat_len(gap, ((const char []) { + ga_concat_len(gap, ((const char[]) { '\\', 'u', xdigits[(hi >> (4 * 3)) & 0xF], xdigits[(hi >> (4 * 2)) & 0xF], From 2c378fdfaf4927b7071b2e673c19c8acb8dcdfd4 Mon Sep 17 00:00:00 2001 From: ZyX Date: Fri, 5 Feb 2016 00:29:47 +0300 Subject: [PATCH 23/82] eval/decode: Parse strings with NUL to special dictionaries --- src/nvim/eval/decode.c | 39 ++++++++--- test/functional/eval/json_functions_spec.lua | 71 +++++++++++++++++++- 2 files changed, 100 insertions(+), 10 deletions(-) diff --git a/src/nvim/eval/decode.c b/src/nvim/eval/decode.c index 5a3c5709ad..8c95b34326 100644 --- a/src/nvim/eval/decode.c +++ b/src/nvim/eval/decode.c @@ -342,6 +342,7 @@ int json_decode_string(const char *const buf, const size_t len, char *str = xmalloc(len + 1); int fst_in_pair = 0; char *str_end = str; + bool hasnul = false; for (const char *t = s; t < p; t++) { if (t[0] != '\\' || t[1] != 'u') { if (fst_in_pair != 0) { @@ -357,6 +358,9 @@ int json_decode_string(const char *const buf, const size_t len, t += 4; unsigned long ch; vim_str2nr((char_u *) ubuf, NULL, NULL, 0, 0, 2, NULL, &ch); + if (ch == 0) { + hasnul = true; + } if (SURROGATE_HI_START <= ch && ch <= SURROGATE_HI_END) { fst_in_pair = (int) ch; } else if (SURROGATE_LO_START <= ch && ch <= SURROGATE_LO_END @@ -405,9 +409,9 @@ int json_decode_string(const char *const buf, const size_t len, str_end += utf_char2bytes((int) fst_in_pair, (char_u *) str_end); } if (conv.vc_type != CONV_NONE) { - size_t len = (size_t) (str_end - str); + size_t str_len = (size_t) (str_end - str); char *const new_str = (char *) string_convert(&conv, (char_u *) str, - &len); + &str_len); if (new_str == NULL) { EMSG2(_("E474: Failed to convert string \"%s\" from UTF-8"), str); xfree(str); @@ -415,14 +419,31 @@ int json_decode_string(const char *const buf, const size_t len, } xfree(str); str = new_str; - str_end = new_str + len; + str_end = new_str + str_len; + } + if (hasnul) { + typval_T obj; + list_T *const list = list_alloc(); + list->lv_refcount++; + create_special_dict(&obj, kMPString, ((typval_T) { + .v_type = VAR_LIST, + .v_lock = 0, + .vval = { .v_list = list }, + })); + if (encode_list_write((void *) list, str, (size_t) (str_end - str)) + == -1) { + clear_tv(&obj); + goto json_decode_string_fail; + } + POP(obj); + } else { + *str_end = NUL; + // TODO: return special string in case of NUL bytes + POP(((typval_T) { + .v_type = VAR_STRING, + .vval = { .v_string = (char_u *) str, }, + })); } - *str_end = NUL; - // TODO: return special string in case of NUL bytes - POP(((typval_T) { - .v_type = VAR_STRING, - .vval = { .v_string = (char_u *) str, }, - })); break; } case '-': diff --git a/test/functional/eval/json_functions_spec.lua b/test/functional/eval/json_functions_spec.lua index 651f6c27b3..7916bc829c 100644 --- a/test/functional/eval/json_functions_spec.lua +++ b/test/functional/eval/json_functions_spec.lua @@ -1,13 +1,63 @@ local helpers = require('test.functional.helpers') local clear = helpers.clear local funcs = helpers.funcs +local meths = helpers.meths local eq = helpers.eq local eval = helpers.eval local execute = helpers.execute local exc_exec = helpers.exc_exec describe('jsondecode() function', function() - before_each(clear) + before_each(function() + clear() + execute([[ + function Eq(exp, act) + let act = a:act + let exp = a:exp + if type(exp) != type(act) + return 0 + endif + if type(exp) == type({}) + if sort(keys(exp)) !=# sort(keys(act)) + return 0 + endif + if sort(keys(exp)) ==# ['_TYPE', '_VAL'] + let exp_typ = v:msgpack_types[exp._TYPE] + let act_typ = act._TYPE + if exp_typ isnot act_typ + return 0 + endif + return Eq(exp._VAL, act._VAL) + else + return empty(filter(copy(exp), '!Eq(v:val, act[v:key])')) + endif + else + if type(exp) == type([]) + if len(exp) != len(act) + return 0 + endif + return empty(filter(copy(exp), '!Eq(v:val, act[v:key])')) + endif + return exp ==# act + endif + return 1 + endfunction + ]]) + execute([[ + function EvalEq(exp, act_expr) + let act = eval(a:act_expr) + if Eq(a:exp, act) + return 1 + else + return string(act) + endif + endfunction + ]]) + end) + + local speq = function(expected, actual_expr) + eq(1, funcs.EvalEq(expected, actual_expr)) + end it('accepts readfile()-style list', function() eq({Test=1}, funcs.jsondecode({ @@ -221,6 +271,14 @@ describe('jsondecode() function', function() exc_exec('call jsondecode("\\t\\"abc\\\\u00")')) eq('Vim(call):E474: Unfinished unicode escape sequence: \t"abc\\u000', exc_exec('call jsondecode("\\t\\"abc\\\\u000")')) + eq('Vim(call):E474: Expected four hex digits after \\u: \\u" ', + exc_exec('call jsondecode("\\t\\"abc\\\\u\\" ")')) + eq('Vim(call):E474: Expected four hex digits after \\u: \\u0" ', + exc_exec('call jsondecode("\\t\\"abc\\\\u0\\" ")')) + eq('Vim(call):E474: Expected four hex digits after \\u: \\u00" ', + exc_exec('call jsondecode("\\t\\"abc\\\\u00\\" ")')) + eq('Vim(call):E474: Expected four hex digits after \\u: \\u000" ', + exc_exec('call jsondecode("\\t\\"abc\\\\u000\\" ")')) eq('Vim(call):E474: Expected string end: \t"abc\\u0000', exc_exec('call jsondecode("\\t\\"abc\\\\u0000")')) end) @@ -315,6 +373,17 @@ describe('jsondecode() function', function() eq('a\xED\xB0\x80', funcs.jsondecode('"a\\uDC00"')) eq('\t\xED\xB0\x80', funcs.jsondecode('"\\t\\uDC00"')) end) + + local sp_decode_eq = function(expected, json) + meths.set_var('__json', json) + speq(expected, 'jsondecode(g:__json)') + execute('unlet! g:__json') + end + + it('parses strings with NUL properly', function() + sp_decode_eq({_TYPE='string', _VAL={'\n'}}, '"\\u0000"') + sp_decode_eq({_TYPE='string', _VAL={'\n', '\n'}}, '"\\u0000\\n\\u0000"') + end) end) describe('jsonencode() function', function() From e303ea8a19bcd385eb7829beb7f2ef691c064b35 Mon Sep 17 00:00:00 2001 From: ZyX Date: Fri, 5 Feb 2016 02:29:10 +0300 Subject: [PATCH 24/82] eval/decode: Add support for special maps Special dictionaries representing map are created when encountering duplicate key or when key is empty or contains NUL. Also checks that values are separated by a comma/colon properly. --- src/nvim/eval/decode.c | 241 ++++++++++++++----- src/nvim/eval/encode.c | 1 + test/functional/eval/json_functions_spec.lua | 52 ++++ 3 files changed, 233 insertions(+), 61 deletions(-) diff --git a/src/nvim/eval/decode.c b/src/nvim/eval/decode.c index 8c95b34326..7fffe1c48b 100644 --- a/src/nvim/eval/decode.c +++ b/src/nvim/eval/decode.c @@ -13,12 +13,28 @@ /// Helper structure for container_struct typedef struct { - size_t stack_index; ///< Index of current container in stack. - typval_T container; ///< Container. Either VAR_LIST, VAR_DICT or VAR_LIST - ///< which is _VAL from special dictionary. + size_t stack_index; ///< Index of current container in stack. + list_T *special_val; ///< _VAL key contents for special maps. + ///< When container is not a special dictionary it is + ///< NULL. + const char *s; ///< Location where container starts. + typval_T container; ///< Container. Either VAR_LIST, VAR_DICT or VAR_LIST + ///< which is _VAL from special dictionary. } ContainerStackItem; -typedef kvec_t(typval_T) ValuesStack; +/// Helper structure for values struct +typedef struct { + bool is_special_string; ///< Indicates that current value is a special + ///< dictionary with string. + bool didcomma; ///< True if previous token was comma. + bool didcolon; ///< True if previous token was colon. + typval_T val; ///< Actual value. +} ValuesStackItem; + +/// Vector containing values not yet saved in any container +typedef kvec_t(ValuesStackItem) ValuesStack; + +/// Vector containing containers, each next container is located inside previous typedef kvec_t(ContainerStackItem) ContainerStack; #ifdef INCLUDE_GENERATED_DECLARATIONS @@ -58,59 +74,119 @@ static inline void create_special_dict(typval_T *const rettv, /// @param[in] obj New object. /// @param[out] stack Object stack. /// @param[out] container_stack Container objects stack. -/// @param[in] p Position in string which is currently being parsed. +/// @param[in,out] pp Position in string which is currently being parsed. Used +/// for error reporting and is also set when decoding is +/// restarted due to the necessity of converting regular +/// dictionary to a special map. +/// @param[out] next_map_special Is set to true when dictionary is converted +/// to a special map, otherwise not touched. +/// @param[out] didcomma True if previous token was comma. Is set to recorded +/// value when decoder is restarted, otherwise unused. +/// @param[out] didcolon True if previous token was colon. Is set to recorded +/// value when decoder is restarted, otherwise unused. /// /// @return OK in case of success, FAIL in case of error. -static inline int json_decoder_pop(typval_T obj, ValuesStack *const stack, +static inline int json_decoder_pop(ValuesStackItem obj, + ValuesStack *const stack, ContainerStack *const container_stack, - const char *const p) + const char **const pp, + bool *const next_map_special, + bool *const didcomma, + bool *const didcolon) FUNC_ATTR_NONNULL_ALL { if (kv_size(*container_stack) == 0) { - kv_push(typval_T, *stack, obj); + kv_push(ValuesStackItem, *stack, obj); return OK; } ContainerStackItem last_container = kv_last(*container_stack); - if (obj.v_type == last_container.container.v_type + const char *val_location = *pp; + if (obj.val.v_type == last_container.container.v_type // vval.v_list and vval.v_dict should have the same size and offset - && ((void *) obj.vval.v_list + && ((void *) obj.val.vval.v_list == (void *) last_container.container.vval.v_list)) { kv_pop(*container_stack); + val_location = last_container.s; last_container = kv_last(*container_stack); } if (last_container.container.v_type == VAR_LIST) { + if (last_container.container.vval.v_list->lv_len != 0 + && !obj.didcomma) { + EMSG2(_("E474: Expected comma before list item: %s"), val_location); + clear_tv(&obj.val); + return FAIL; + } + assert(last_container.special_val == NULL); listitem_T *obj_li = listitem_alloc(); - obj_li->li_tv = obj; + obj_li->li_tv = obj.val; list_append(last_container.container.vval.v_list, obj_li); } else if (last_container.stack_index == kv_size(*stack) - 2) { - typval_T key = kv_pop(*stack); - if (key.v_type != VAR_STRING) { - assert(false); - } else if (key.vval.v_string == NULL || *key.vval.v_string == NUL) { - // TODO: fall back to special dict in case of empty key - EMSG(_("E474: Empty key")); - clear_tv(&obj); + if (!obj.didcolon) { + EMSG2(_("E474: Expected colon before dictionary value: %s"), + val_location); + clear_tv(&obj.val); return FAIL; } - dictitem_T *obj_di = dictitem_alloc(key.vval.v_string); - clear_tv(&key); - if (dict_add(last_container.container.vval.v_dict, obj_di) - == FAIL) { - // TODO: fall back to special dict in case of duplicate keys - EMSG(_("E474: Duplicate key")); - dictitem_free(obj_di); - clear_tv(&obj); - return FAIL; + ValuesStackItem key = kv_pop(*stack); + if (last_container.special_val == NULL) { + // These cases should have already been handled. + assert(!(key.is_special_string + || key.val.vval.v_string == NULL + || *key.val.vval.v_string == NUL)); + dictitem_T *obj_di = dictitem_alloc(key.val.vval.v_string); + clear_tv(&key.val); + if (dict_add(last_container.container.vval.v_dict, obj_di) + == FAIL) { + assert(false); + } + obj_di->di_tv = obj.val; + } else { + list_T *const kv_pair = list_alloc(); + list_append_list(last_container.special_val, kv_pair); + listitem_T *const key_li = listitem_alloc(); + key_li->li_tv = key.val; + list_append(kv_pair, key_li); + listitem_T *const val_li = listitem_alloc(); + val_li->li_tv = obj.val; + list_append(kv_pair, val_li); } - obj_di->di_tv = obj; } else { // Object with key only - if (obj.v_type != VAR_STRING) { - EMSG2(_("E474: Expected string key: %s"), p); - clear_tv(&obj); + if (!obj.is_special_string && obj.val.v_type != VAR_STRING) { + EMSG2(_("E474: Expected string key: %s"), *pp); + clear_tv(&obj.val); + return FAIL; + } else if (!obj.didcomma + && (last_container.special_val == NULL + && (last_container.container.vval.v_dict->dv_hashtab.ht_used + != 0))) { + EMSG2(_("E474: Expected comma before dictionary key: %s"), val_location); + clear_tv(&obj.val); return FAIL; } - kv_push(typval_T, *stack, obj); + // Handle empty key and key represented as special dictionary + if (last_container.special_val == NULL + && (obj.is_special_string + || obj.val.vval.v_string == NULL + || *obj.val.vval.v_string == NUL + || dict_find(last_container.container.vval.v_dict, + obj.val.vval.v_string, -1))) { + clear_tv(&obj.val); + + // Restart + kv_pop(*container_stack); + ValuesStackItem last_container_val = + kv_A(*stack, last_container.stack_index); + while (kv_size(*stack) > last_container.stack_index) { + clear_tv(&(kv_pop(*stack).val)); + } + *pp = last_container.s; + *didcomma = last_container_val.didcomma; + *didcolon = last_container_val.didcolon; + *next_map_special = true; + return OK; + } + kv_push(ValuesStackItem, *stack, obj); } return OK; } @@ -126,7 +202,7 @@ int json_decode_string(const char *const buf, const size_t len, typval_T *const rettv) FUNC_ATTR_NONNULL_ALL FUNC_ATTR_WARN_UNUSED_RESULT { - vimconv_T conv; + vimconv_T conv = { .vc_type = CONV_NONE }; convert_setup(&conv, (char_u *) "utf-8", p_enc); conv.vc_fail = true; int ret = OK; @@ -138,14 +214,29 @@ int json_decode_string(const char *const buf, const size_t len, const char *const e = buf + len; bool didcomma = false; bool didcolon = false; -#define POP(obj) \ + bool next_map_special = false; +#define OBJ(obj_tv, is_sp_string) \ + ((ValuesStackItem) { \ + .is_special_string = (is_sp_string), \ + .val = (obj_tv), \ + .didcomma = didcomma, \ + .didcolon = didcolon, \ + }) +#define POP(obj_tv, is_sp_string) \ do { \ - if (json_decoder_pop(obj, &stack, &container_stack, p) == FAIL) { \ + if (json_decoder_pop(OBJ(obj_tv, is_sp_string), &stack, &container_stack, \ + &p, &next_map_special, &didcomma, &didcolon) \ + == FAIL) { \ goto json_decode_string_fail; \ } \ + if (next_map_special) { \ + goto json_decode_string_cycle_start; \ + } \ } while (0) const char *p = buf; for (; p < e; p++) { +json_decode_string_cycle_start: + assert(*p == '{' || next_map_special == false); switch (*p) { case '}': case ']': { @@ -176,8 +267,11 @@ int json_decode_string(const char *const buf, const size_t len, kv_pop(container_stack); goto json_decode_string_after_cycle; } else { - typval_T obj = kv_pop(stack); - POP(obj); + if (json_decoder_pop(kv_pop(stack), &stack, &container_stack, &p, + &next_map_special, &didcomma, &didcolon) == FAIL) { + goto json_decode_string_fail; + } + assert(!next_map_special); break; } } @@ -197,11 +291,12 @@ int json_decode_string(const char *const buf, const size_t len, && last_container.stack_index != kv_size(stack) - 1) { EMSG2(_("E474: Using comma in place of colon: %s"), p); goto json_decode_string_fail; - } else if ((last_container.container.v_type == VAR_DICT - && (last_container.container.vval.v_dict->dv_hashtab.ht_used - == 0)) - || (last_container.container.v_type == VAR_LIST - && last_container.container.vval.v_list->lv_len == 0)) { + } else if (last_container.special_val == NULL + ? (last_container.container.v_type == VAR_DICT + ? (last_container.container.vval.v_dict->dv_hashtab.ht_used + == 0) + : (last_container.container.vval.v_list->lv_len == 0)) + : (last_container.special_val->lv_len == 0)) { EMSG2(_("E474: Leading comma: %s"), p); goto json_decode_string_fail; } @@ -241,7 +336,7 @@ int json_decode_string(const char *const buf, const size_t len, goto json_decode_string_fail; } p += 3; - POP(get_vim_var_tv(VV_NULL)); + POP(get_vim_var_tv(VV_NULL), false); break; } case 't': { @@ -250,7 +345,7 @@ int json_decode_string(const char *const buf, const size_t len, goto json_decode_string_fail; } p += 3; - POP(get_vim_var_tv(VV_TRUE)); + POP(get_vim_var_tv(VV_TRUE), false); break; } case 'f': { @@ -259,7 +354,7 @@ int json_decode_string(const char *const buf, const size_t len, goto json_decode_string_fail; } p += 4; - POP(get_vim_var_tv(VV_FALSE)); + POP(get_vim_var_tv(VV_FALSE), false); break; } case '"': { @@ -339,6 +434,13 @@ int json_decode_string(const char *const buf, const size_t len, EMSG2(_("E474: Expected string end: %s"), buf); goto json_decode_string_fail; } + if (len == 0) { + POP(((typval_T) { + .v_type = VAR_STRING, + .vval = { .v_string = NULL }, + }), false); + break; + } char *str = xmalloc(len + 1); int fst_in_pair = 0; char *str_end = str; @@ -435,14 +537,13 @@ int json_decode_string(const char *const buf, const size_t len, clear_tv(&obj); goto json_decode_string_fail; } - POP(obj); + POP(obj, true); } else { *str_end = NUL; - // TODO: return special string in case of NUL bytes POP(((typval_T) { .v_type = VAR_STRING, - .vval = { .v_string = (char_u *) str, }, - })); + .vval = { .v_string = (char_u *) str }, + }), false); } break; } @@ -510,7 +611,7 @@ int json_decode_string(const char *const buf, const size_t len, vim_str2nr((char_u *) s, NULL, NULL, 0, 0, 0, &nr, NULL); tv.vval.v_number = (varnumber_T) nr; } - POP(tv); + POP(tv, false); p--; break; } @@ -524,24 +625,41 @@ int json_decode_string(const char *const buf, const size_t len, }; kv_push(ContainerStackItem, container_stack, ((ContainerStackItem) { .stack_index = kv_size(stack), + .s = p, .container = tv, + .special_val = NULL, })); - kv_push(typval_T, stack, tv); + kv_push(ValuesStackItem, stack, OBJ(tv, false)); break; } case '{': { - dict_T *dict = dict_alloc(); - dict->dv_refcount++; - typval_T tv = { - .v_type = VAR_DICT, - .v_lock = VAR_UNLOCKED, - .vval = { .v_dict = dict }, - }; + typval_T tv; + list_T *val_list = NULL; + if (next_map_special) { + next_map_special = false; + val_list = list_alloc(); + val_list->lv_refcount++; + create_special_dict(&tv, kMPMap, ((typval_T) { + .v_type = VAR_LIST, + .v_lock = VAR_UNLOCKED, + .vval = { .v_list = val_list }, + })); + } else { + dict_T *dict = dict_alloc(); + dict->dv_refcount++; + tv = (typval_T) { + .v_type = VAR_DICT, + .v_lock = VAR_UNLOCKED, + .vval = { .v_dict = dict }, + }; + } kv_push(ContainerStackItem, container_stack, ((ContainerStackItem) { .stack_index = kv_size(stack), + .s = p, .container = tv, + .special_val = val_list, })); - kv_push(typval_T, stack, tv); + kv_push(ValuesStackItem, stack, OBJ(tv, false)); break; } default: { @@ -557,6 +675,7 @@ int json_decode_string(const char *const buf, const size_t len, } } #undef POP +#undef OBJ json_decode_string_after_cycle: for (; p < e; p++) { switch (*p) { @@ -579,12 +698,12 @@ json_decode_string_after_cycle: json_decode_string_fail: ret = FAIL; while (kv_size(stack)) { - clear_tv(&kv_pop(stack)); + clear_tv(&(kv_pop(stack).val)); } json_decode_string_ret: if (ret != FAIL) { assert(kv_size(stack) == 1); - *rettv = kv_pop(stack); + *rettv = kv_pop(stack).val; } kv_destroy(stack); kv_destroy(container_stack); diff --git a/src/nvim/eval/encode.c b/src/nvim/eval/encode.c index 5c843357f2..0fc975ed42 100644 --- a/src/nvim/eval/encode.c +++ b/src/nvim/eval/encode.c @@ -897,6 +897,7 @@ static inline int convert_to_json_string(garray_T *const gap, size_t len_ = len; char *tofree = NULL; if (last_p_enc != (const void *) p_enc) { + p_enc_conv.vc_type = CONV_NONE; convert_setup(&p_enc_conv, p_enc, "utf-8"); p_enc_conv.vc_fail = true; last_p_enc = p_enc; diff --git a/test/functional/eval/json_functions_spec.lua b/test/functional/eval/json_functions_spec.lua index 7916bc829c..6f81a36479 100644 --- a/test/functional/eval/json_functions_spec.lua +++ b/test/functional/eval/json_functions_spec.lua @@ -203,6 +203,19 @@ describe('jsondecode() function', function() exc_exec('call jsondecode("[}")')) end) + it('fails to parse concat inside container', function() + eq('Vim(call):E474: Expected comma before list item: []]', + exc_exec('call jsondecode("[[][]]")')) + eq('Vim(call):E474: Expected comma before list item: {}]', + exc_exec('call jsondecode("[{}{}]")')) + eq('Vim(call):E474: Expected comma before list item: ]', + exc_exec('call jsondecode("[1 2]")')) + eq('Vim(call):E474: Expected comma before dictionary key: ": 4}', + exc_exec('call jsondecode("{\\"1\\": 2 \\"3\\": 4}")')) + eq('Vim(call):E474: Expected colon before dictionary value: , "3" 4}', + exc_exec('call jsondecode("{\\"1\\" 2, \\"3\\" 4}")')) + end) + it('fails to parse containers with leading comma or colon', function() eq('Vim(call):E474: Leading comma: ,}', exc_exec('call jsondecode("{,}")')) @@ -384,6 +397,45 @@ describe('jsondecode() function', function() sp_decode_eq({_TYPE='string', _VAL={'\n'}}, '"\\u0000"') sp_decode_eq({_TYPE='string', _VAL={'\n', '\n'}}, '"\\u0000\\n\\u0000"') end) + + it('parses dictionaries with duplicate keys to special maps', function() + sp_decode_eq({_TYPE='map', _VAL={{'a', 1}, {'a', 2}}}, + '{"a": 1, "a": 2}') + sp_decode_eq({_TYPE='map', _VAL={{'b', 3}, {'a', 1}, {'a', 2}}}, + '{"b": 3, "a": 1, "a": 2}') + sp_decode_eq({_TYPE='map', _VAL={{'b', 3}, {'a', 1}, {'c', 4}, {'a', 2}}}, + '{"b": 3, "a": 1, "c": 4, "a": 2}') + sp_decode_eq({_TYPE='map', _VAL={{'b', 3}, {'a', 1}, {'c', 4}, {'a', 2}, {'c', 4}}}, + '{"b": 3, "a": 1, "c": 4, "a": 2, "c": 4}') + sp_decode_eq({{_TYPE='map', _VAL={{'b', 3}, {'a', 1}, {'c', 4}, {'a', 2}, {'c', 4}}}}, + '[{"b": 3, "a": 1, "c": 4, "a": 2, "c": 4}]') + sp_decode_eq({{d={_TYPE='map', _VAL={{'b', 3}, {'a', 1}, {'c', 4}, {'a', 2}, {'c', 4}}}}}, + '[{"d": {"b": 3, "a": 1, "c": 4, "a": 2, "c": 4}}]') + sp_decode_eq({1, {d={_TYPE='map', _VAL={{'b', 3}, {'a', 1}, {'c', 4}, {'a', 2}, {'c', 4}}}}}, + '[1, {"d": {"b": 3, "a": 1, "c": 4, "a": 2, "c": 4}}]') + sp_decode_eq({1, {a={}, d={_TYPE='map', _VAL={{'b', 3}, {'a', 1}, {'c', 4}, {'a', 2}, {'c', 4}}}}}, + '[1, {"a": [], "d": {"b": 3, "a": 1, "c": 4, "a": 2, "c": 4}}]') + end) + + it('parses dictionaries with empty keys to special maps', function() + sp_decode_eq({_TYPE='map', _VAL={{'', 4}}}, + '{"": 4}') + sp_decode_eq({_TYPE='map', _VAL={{'b', 3}, {'a', 1}, {'c', 4}, {'d', 2}, {'', 4}}}, + '{"b": 3, "a": 1, "c": 4, "d": 2, "": 4}') + sp_decode_eq({_TYPE='map', _VAL={{'', 3}, {'a', 1}, {'c', 4}, {'d', 2}, {'', 4}}}, + '{"": 3, "a": 1, "c": 4, "d": 2, "": 4}') + sp_decode_eq({{_TYPE='map', _VAL={{'', 3}, {'a', 1}, {'c', 4}, {'d', 2}, {'', 4}}}}, + '[{"": 3, "a": 1, "c": 4, "d": 2, "": 4}]') + end) + + it('parses dictionaries with keys with NUL bytes to special maps', function() + sp_decode_eq({_TYPE='map', _VAL={{{_TYPE='string', _VAL={'a\n', 'b'}}, 4}}}, + '{"a\\u0000\\nb": 4}') + sp_decode_eq({_TYPE='map', _VAL={{{_TYPE='string', _VAL={'a\n', 'b', ''}}, 4}}}, + '{"a\\u0000\\nb\\n": 4}') + sp_decode_eq({_TYPE='map', _VAL={{'b', 3}, {'a', 1}, {'c', 4}, {'d', 2}, {{_TYPE='string', _VAL={'\n'}}, 4}}}, + '{"b": 3, "a": 1, "c": 4, "d": 2, "\\u0000": 4}') + end) end) describe('jsonencode() function', function() From a3b87fc19b652065d96cec8f571d3245f1fc2446 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sat, 6 Feb 2016 02:36:07 +0300 Subject: [PATCH 25/82] eval: Remove get_vim_var_tv function --- src/nvim/eval.c | 12 ------------ src/nvim/eval/decode.c | 32 +++++++++++++++++++++++++++----- 2 files changed, 27 insertions(+), 17 deletions(-) diff --git a/src/nvim/eval.c b/src/nvim/eval.c index 38dc7c1034..dc7737e4ce 100644 --- a/src/nvim/eval.c +++ b/src/nvim/eval.c @@ -17060,18 +17060,6 @@ long get_vim_var_nr(int idx) FUNC_ATTR_PURE return vimvars[idx].vv_nr; } -/// Get typval_T representing v: variable -/// -/// @warning if v: variable has reference counter it is not increased. -/// -/// @param[in] idx Variable index, @see VimVarIndex. -typval_T get_vim_var_tv(VimVarIndex idx) FUNC_ATTR_PURE -{ - typval_T ret = vimvars[idx].vv_di.di_tv; - ret.v_lock = VAR_UNLOCKED; - return ret; -} - /* * Get string v: variable value. Uses a static buffer, can only be used once. */ diff --git a/src/nvim/eval/decode.c b/src/nvim/eval/decode.c index 7fffe1c48b..de89f9c132 100644 --- a/src/nvim/eval/decode.c +++ b/src/nvim/eval/decode.c @@ -336,7 +336,11 @@ json_decode_string_cycle_start: goto json_decode_string_fail; } p += 3; - POP(get_vim_var_tv(VV_NULL), false); + POP(((typval_T) { + .v_type = VAR_SPECIAL, + .v_lock = VAR_UNLOCKED, + .vval = { .v_special = kSpecialVarNull }, + }), false); break; } case 't': { @@ -345,7 +349,11 @@ json_decode_string_cycle_start: goto json_decode_string_fail; } p += 3; - POP(get_vim_var_tv(VV_TRUE), false); + POP(((typval_T) { + .v_type = VAR_SPECIAL, + .v_lock = VAR_UNLOCKED, + .vval = { .v_special = kSpecialVarTrue }, + }), false); break; } case 'f': { @@ -354,7 +362,11 @@ json_decode_string_cycle_start: goto json_decode_string_fail; } p += 4; - POP(get_vim_var_tv(VV_FALSE), false); + POP(((typval_T) { + .v_type = VAR_SPECIAL, + .v_lock = VAR_UNLOCKED, + .vval = { .v_special = kSpecialVarFalse }, + }), false); break; } case '"': { @@ -716,11 +728,21 @@ int msgpack_to_vim(const msgpack_object mobj, typval_T *const rettv) { switch (mobj.type) { case MSGPACK_OBJECT_NIL: { - *rettv = get_vim_var_tv(VV_NULL); + *rettv = (typval_T) { + .v_type = VAR_SPECIAL, + .v_lock = VAR_UNLOCKED, + .vval = { .v_special = kSpecialVarNull }, + }; break; } case MSGPACK_OBJECT_BOOLEAN: { - *rettv = get_vim_var_tv(mobj.via.boolean ? VV_TRUE : VV_FALSE); + *rettv = (typval_T) { + .v_type = VAR_SPECIAL, + .v_lock = VAR_UNLOCKED, + .vval = { + .v_special = mobj.via.boolean ? kSpecialVarTrue : kSpecialVarFalse + }, + }; break; } case MSGPACK_OBJECT_POSITIVE_INTEGER: { From 6167ce6df2753d5474ad49aea19f5957128ab015 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sat, 6 Feb 2016 02:46:23 +0300 Subject: [PATCH 26/82] eval: Remove v:none To get v:none back just rever this commit. This will not make json*() functions compatible with Vim though. --- runtime/doc/eval.txt | 9 +----- runtime/doc/vim_diff.txt | 4 +++ src/nvim/api/private/helpers.c | 3 +- src/nvim/eval.c | 4 --- src/nvim/eval.h | 1 - src/nvim/eval/encode.c | 17 ---------- src/nvim/eval_defs.h | 3 +- .../eval/msgpack_functions_spec.lua | 7 +--- test/functional/eval/special_vars_spec.lua | 32 ------------------- test/functional/eval/string_spec.lua | 1 - 10 files changed, 8 insertions(+), 73 deletions(-) diff --git a/runtime/doc/eval.txt b/runtime/doc/eval.txt index ee8ede2b91..f085dd2972 100644 --- a/runtime/doc/eval.txt +++ b/runtime/doc/eval.txt @@ -1569,13 +1569,6 @@ v:null Special value used to put "null" in JSON and NIL in msgpack. See |jsonencode()|. This value is converted to "null" when used as a String (e.g. in |expr5| with string concatenation operator) and to zero when used as a Number (e.g. in |expr5| - or |expr7| when used with numeric operators). - - *v:none* *none-variable* -v:none Special value used to put an empty item in JSON. See - |jsonencode()|. This value is converted to "none" when used - as a String (e.g. in |expr5| with string concatenation - operator) and to zero when used as a Number (e.g. in |expr5| or |expr7| when used with numeric operators). *v:oldfiles* *oldfiles-variable* @@ -4876,7 +4869,7 @@ msgpackdump({list}) {Nvim} *msgpackdump()* messagepack). Limitations: *E951* *E952* *E953* - 1. |Funcref|s and |v:none| cannot be dumped. + 1. |Funcref|s cannot be dumped. 2. Containers that reference themselves cannot be dumped. 3. Dictionary keys are always dumped as STR strings. 4. Other strings are always dumped as BIN strings. diff --git a/runtime/doc/vim_diff.txt b/runtime/doc/vim_diff.txt index ddb1fd3b8f..2d03b4ff45 100644 --- a/runtime/doc/vim_diff.txt +++ b/runtime/doc/vim_diff.txt @@ -106,6 +106,10 @@ are always available and may be used simultaneously in separate plugins. The |jsonencode()| behaviour slightly changed: now |msgpack-special-dict| values are accepted. +*v:none* variable is absent. In Vim it represents “no value” in non-JSON +strings like "{"a": }" parsed as "{'a': v:none}". See |jsondecode()| and +|jsonencode()| incompatibilities above. + Viminfo text files were replaced with binary (messagepack) ShaDa files. Additional differences: diff --git a/src/nvim/api/private/helpers.c b/src/nvim/api/private/helpers.c index a8082655fd..9082dfd759 100644 --- a/src/nvim/api/private/helpers.c +++ b/src/nvim/api/private/helpers.c @@ -659,8 +659,7 @@ static Object vim_to_object_rec(typval_T *obj, PMap(ptr_t) *lookup) rv.data.boolean = (obj->vval.v_special == kSpecialVarTrue); break; } - case kSpecialVarNull: - case kSpecialVarNone: { + case kSpecialVarNull: { rv.type = kObjectTypeNil; break; } diff --git a/src/nvim/eval.c b/src/nvim/eval.c index dc7737e4ce..4919cc2ce8 100644 --- a/src/nvim/eval.c +++ b/src/nvim/eval.c @@ -368,7 +368,6 @@ static struct vimvar { { VV_NAME("false", VAR_SPECIAL), VV_RO }, { VV_NAME("true", VAR_SPECIAL), VV_RO }, { VV_NAME("null", VAR_SPECIAL), VV_RO }, - { VV_NAME("none", VAR_SPECIAL), VV_RO }, }; /* shorthand */ @@ -512,7 +511,6 @@ void eval_init(void) set_vim_var_special(VV_FALSE, kSpecialVarFalse); set_vim_var_special(VV_TRUE, kSpecialVarTrue); - set_vim_var_special(VV_NONE, kSpecialVarNone); set_vim_var_special(VV_NULL, kSpecialVarNull); set_reg_var(0); // default for v:register is not 0 but '"' @@ -16204,7 +16202,6 @@ static void f_type(typval_T *argvars, typval_T *rettv) n = 6; break; } - case kSpecialVarNone: case kSpecialVarNull: { n = 7; break; @@ -17520,7 +17517,6 @@ long get_tv_number_chk(typval_T *varp, int *denote) return 1; } case kSpecialVarFalse: - case kSpecialVarNone: case kSpecialVarNull: { return 0; } diff --git a/src/nvim/eval.h b/src/nvim/eval.h index 40111abf8d..c2b67cd12e 100644 --- a/src/nvim/eval.h +++ b/src/nvim/eval.h @@ -124,7 +124,6 @@ typedef enum { VV_FALSE, VV_TRUE, VV_NULL, - VV_NONE, } VimVarIndex; /// All recognized msgpack types diff --git a/src/nvim/eval/encode.c b/src/nvim/eval/encode.c index 0fc975ed42..1550a61f7f 100644 --- a/src/nvim/eval/encode.c +++ b/src/nvim/eval/encode.c @@ -55,7 +55,6 @@ typedef kvec_t(MPConvStackVal) MPConvStack; const char *const encode_special_var_names[] = { [kSpecialVarNull] = "null", - [kSpecialVarNone] = "none", [kSpecialVarTrue] = "true", [kSpecialVarFalse] = "false", }; @@ -358,10 +357,6 @@ static int name##_convert_one_value(firstargtype firstargname, \ CONV_BOOL(tv->vval.v_special == kSpecialVarTrue); \ break; \ } \ - case kSpecialVarNone: { \ - CONV_NONE_VAL(); \ - break; \ - } \ } \ break; \ } \ @@ -726,9 +721,6 @@ encode_vim_to_##name##_error_ret: \ #define CONV_BOOL(num) \ ga_concat(gap, ((num)? "v:true": "v:false")) -#define CONV_NONE_VAL() \ - ga_concat(gap, "v:none") - #define CONV_UNSIGNED_NUMBER(num) #define CONV_DICT_START(len) \ @@ -1074,9 +1066,6 @@ static inline bool check_json_key(const typval_T *const tv) } \ } while (0) -#undef CONV_NONE_VAL -#define CONV_NONE_VAL() - DEFINE_VIML_CONV_FUNCTIONS(static, json, garray_T *const, gap) #undef CONV_STRING @@ -1090,7 +1079,6 @@ DEFINE_VIML_CONV_FUNCTIONS(static, json, garray_T *const, gap) #undef CONV_EMPTY_DICT #undef CONV_NIL #undef CONV_BOOL -#undef CONV_NONE_VAL #undef CONV_UNSIGNED_NUMBER #undef CONV_DICT_START #undef CONV_DICT_END @@ -1226,10 +1214,6 @@ char *encode_tv2json(typval_T *tv, size_t *len) #define CONV_NIL() \ msgpack_pack_nil(packer) -#define CONV_NONE_VAL() \ - return conv_error(_("E953: Attempt to convert v:none in %s, %s"), \ - mpstack, objname) - #define CONV_BOOL(num) \ do { \ if ((num)) { \ @@ -1277,7 +1261,6 @@ DEFINE_VIML_CONV_FUNCTIONS(, msgpack, msgpack_packer *const, packer) #undef CONV_EMPTY_DICT #undef CONV_NIL #undef CONV_BOOL -#undef CONV_NONE_VAL #undef CONV_UNSIGNED_NUMBER #undef CONV_DICT_START #undef CONV_DICT_END diff --git a/src/nvim/eval_defs.h b/src/nvim/eval_defs.h index bcd9e80f9a..3c119c44e1 100644 --- a/src/nvim/eval_defs.h +++ b/src/nvim/eval_defs.h @@ -20,7 +20,6 @@ typedef struct dictvar_S dict_T; typedef enum { kSpecialVarFalse, ///< v:false kSpecialVarTrue, ///< v:true - kSpecialVarNone, ///< v:none kSpecialVarNull, ///< v:null } SpecialVarValue; @@ -40,7 +39,7 @@ typedef enum { VAR_LIST, ///< List, .v_list is used. VAR_DICT, ///< Dictionary, .v_dict is used. VAR_FLOAT, ///< Floating-point value, .v_float is used. - VAR_SPECIAL, ///< Special value (true, false, null, none), .v_special + VAR_SPECIAL, ///< Special value (true, false, null), .v_special ///< is used. } VarType; diff --git a/test/functional/eval/msgpack_functions_spec.lua b/test/functional/eval/msgpack_functions_spec.lua index 3d539d855d..a602bad86f 100644 --- a/test/functional/eval/msgpack_functions_spec.lua +++ b/test/functional/eval/msgpack_functions_spec.lua @@ -645,11 +645,6 @@ describe('msgpackdump() function', function() exc_exec('call msgpackdump([todump])')) end) - it('fails to dump v:none', function() - eq('Vim(call):E953: Attempt to convert v:none in msgpackdump() argument, index 0, itself', - exc_exec('call msgpackdump([v:none])')) - end) - it('fails when called with no arguments', function() eq('Vim(call):E119: Not enough arguments for function: msgpackdump', exc_exec('call msgpackdump()')) @@ -686,7 +681,7 @@ describe('msgpackdump() function', function() end) it('fails to dump special value', function() - for _, val in ipairs({'v:true', 'v:false', 'v:null', 'v:none'}) do + for _, val in ipairs({'v:true', 'v:false', 'v:null'}) do eq('Vim(call):E686: Argument of msgpackdump() must be a List', exc_exec('call msgpackdump(' .. val .. ')')) end diff --git a/test/functional/eval/special_vars_spec.lua b/test/functional/eval/special_vars_spec.lua index c7df847946..b5c65d23d9 100644 --- a/test/functional/eval/special_vars_spec.lua +++ b/test/functional/eval/special_vars_spec.lua @@ -28,21 +28,15 @@ describe('Special values', function() eq(0, funcs.empty(true)) eq(1, funcs.empty(false)) eq(1, eval('empty(v:null)')) - eq(1, eval('empty(v:none)')) end) it('can be stringified and eval’ed back', function() eq(true, funcs.eval(funcs.string(true))) eq(false, funcs.eval(funcs.string(false))) eq(nil, eval('eval(string(v:null))')) - eq(1, eval('eval(string(v:none)) is# v:none')) end) it('work with is/isnot properly', function() - eq(1, eval('v:none is v:none')) - eq(0, eval('v:none is v:null')) - eq(0, eval('v:none is v:true')) - eq(0, eval('v:none is v:false')) eq(1, eval('v:null is v:null')) eq(0, eval('v:null is v:true')) eq(0, eval('v:null is v:false')) @@ -50,35 +44,26 @@ describe('Special values', function() eq(0, eval('v:true is v:false')) eq(1, eval('v:false is v:false')) - eq(0, eval('v:none is 0')) eq(0, eval('v:null is 0')) eq(0, eval('v:true is 0')) eq(0, eval('v:false is 0')) - eq(0, eval('v:none is 1')) eq(0, eval('v:null is 1')) eq(0, eval('v:true is 1')) eq(0, eval('v:false is 1')) - eq(0, eval('v:none is ""')) eq(0, eval('v:null is ""')) eq(0, eval('v:true is ""')) eq(0, eval('v:false is ""')) - eq(0, eval('v:none is "none"')) eq(0, eval('v:null is "null"')) eq(0, eval('v:true is "true"')) eq(0, eval('v:false is "false"')) - eq(0, eval('v:none is []')) eq(0, eval('v:null is []')) eq(0, eval('v:true is []')) eq(0, eval('v:false is []')) - eq(0, eval('v:none isnot v:none')) - eq(1, eval('v:none isnot v:null')) - eq(1, eval('v:none isnot v:true')) - eq(1, eval('v:none isnot v:false')) eq(0, eval('v:null isnot v:null')) eq(1, eval('v:null isnot v:true')) eq(1, eval('v:null isnot v:false')) @@ -86,27 +71,22 @@ describe('Special values', function() eq(1, eval('v:true isnot v:false')) eq(0, eval('v:false isnot v:false')) - eq(1, eval('v:none isnot 0')) eq(1, eval('v:null isnot 0')) eq(1, eval('v:true isnot 0')) eq(1, eval('v:false isnot 0')) - eq(1, eval('v:none isnot 1')) eq(1, eval('v:null isnot 1')) eq(1, eval('v:true isnot 1')) eq(1, eval('v:false isnot 1')) - eq(1, eval('v:none isnot ""')) eq(1, eval('v:null isnot ""')) eq(1, eval('v:true isnot ""')) eq(1, eval('v:false isnot ""')) - eq(1, eval('v:none isnot "none"')) eq(1, eval('v:null isnot "null"')) eq(1, eval('v:true isnot "true"')) eq(1, eval('v:false isnot "false"')) - eq(1, eval('v:none isnot []')) eq(1, eval('v:null isnot []')) eq(1, eval('v:true isnot []')) eq(1, eval('v:false isnot []')) @@ -114,17 +94,14 @@ describe('Special values', function() it('work with +/-/* properly', function() eq(1, eval('0 + v:true')) - eq(0, eval('0 + v:none')) eq(0, eval('0 + v:null')) eq(0, eval('0 + v:false')) eq(-1, eval('0 - v:true')) - eq( 0, eval('0 - v:none')) eq( 0, eval('0 - v:null')) eq( 0, eval('0 - v:false')) eq(1, eval('1 * v:true')) - eq(0, eval('1 * v:none')) eq(0, eval('1 * v:null')) eq(0, eval('1 * v:false')) end) @@ -132,28 +109,23 @@ describe('Special values', function() it('does not work with +=/-=/.=', function() meths.set_var('true', true) meths.set_var('false', false) - execute('let none = v:none') execute('let null = v:null') eq('Vim(let):E734: Wrong variable type for +=', exc_exec('let true += 1')) eq('Vim(let):E734: Wrong variable type for +=', exc_exec('let false += 1')) - eq('Vim(let):E734: Wrong variable type for +=', exc_exec('let none += 1')) eq('Vim(let):E734: Wrong variable type for +=', exc_exec('let null += 1')) eq('Vim(let):E734: Wrong variable type for -=', exc_exec('let true -= 1')) eq('Vim(let):E734: Wrong variable type for -=', exc_exec('let false -= 1')) - eq('Vim(let):E734: Wrong variable type for -=', exc_exec('let none -= 1')) eq('Vim(let):E734: Wrong variable type for -=', exc_exec('let null -= 1')) eq('Vim(let):E734: Wrong variable type for .=', exc_exec('let true .= 1')) eq('Vim(let):E734: Wrong variable type for .=', exc_exec('let false .= 1')) - eq('Vim(let):E734: Wrong variable type for .=', exc_exec('let none .= 1')) eq('Vim(let):E734: Wrong variable type for .=', exc_exec('let null .= 1')) end) it('work with . (concat) properly', function() eq("true", eval('"" . v:true')) - eq("none", eval('"" . v:none')) eq("null", eval('"" . v:null')) eq("false", eval('"" . v:false')) end) @@ -162,25 +134,21 @@ describe('Special values', function() eq(6, funcs.type(true)) eq(6, funcs.type(false)) eq(7, eval('type(v:null)')) - eq(7, eval('type(v:none)')) end) it('work with copy() and deepcopy()', function() eq(true, funcs.deepcopy(true)) eq(false, funcs.deepcopy(false)) eq(nil, eval('deepcopy(v:null)')) - eq(nil, eval('deepcopy(v:none)')) eq(true, funcs.copy(true)) eq(false, funcs.copy(false)) eq(nil, eval('copy(v:null)')) - eq(nil, eval('copy(v:none)')) end) it('fails in index', function() eq('Vim(echo):E15: Cannot index a special value', exc_exec('echo v:true[0]')) eq('Vim(echo):E15: Cannot index a special value', exc_exec('echo v:false[0]')) - eq('Vim(echo):E15: Cannot index a special value', exc_exec('echo v:none[0]')) eq('Vim(echo):E15: Cannot index a special value', exc_exec('echo v:null[0]')) end) end) diff --git a/test/functional/eval/string_spec.lua b/test/functional/eval/string_spec.lua index fe79708910..0c4ff87231 100644 --- a/test/functional/eval/string_spec.lua +++ b/test/functional/eval/string_spec.lua @@ -31,7 +31,6 @@ describe('string() function', function() it('dumps special v: values', function() eq('v:true', eval('string(v:true)')) eq('v:false', eval('string(v:false)')) - eq('v:none', eval('string(v:none)')) eq('v:null', eval('string(v:null)')) end) From 33778c36ccc62d83d24ab30181926ba44fa4eecf Mon Sep 17 00:00:00 2001 From: ZyX Date: Sat, 6 Feb 2016 03:14:10 +0300 Subject: [PATCH 27/82] *: Fix linter errors --- src/nvim/eval.c | 12 +++++---- src/nvim/eval.h | 4 ++- src/nvim/eval/decode.c | 55 +++++++++++++++++++++++------------------- src/nvim/message.h | 2 +- 4 files changed, 41 insertions(+), 32 deletions(-) diff --git a/src/nvim/eval.c b/src/nvim/eval.c index 4919cc2ce8..22a12d353a 100644 --- a/src/nvim/eval.c +++ b/src/nvim/eval.c @@ -5482,8 +5482,9 @@ static int list_join_inner(garray_T *const gap, list_T *const l, char *s; size_t len; s = encode_tv2echo(&item->li_tv, &len); - if (s == NULL) + if (s == NULL) { return FAIL; + } sumlen += (int) len; @@ -8317,9 +8318,9 @@ static void f_deepcopy(typval_T *argvars, typval_T *rettv) if (argvars[1].v_type != VAR_UNKNOWN) noref = get_tv_number_chk(&argvars[1], NULL); - if (noref < 0 || noref > 1) + if (noref < 0 || noref > 1) { EMSG(_(e_invarg)); - else { + } else { var_item_copy(NULL, &argvars[0], rettv, true, (noref == 0 ? get_copyID() : 0)); @@ -11960,8 +11961,9 @@ static void find_some_match(typval_T *argvars, typval_T *rettv, int type) } xfree(tofree); tofree = str = (char_u *) encode_tv2echo(&li->li_tv, NULL); - if (str == NULL) + if (str == NULL) { break; + } } match = vim_regexec_nl(®match, str, (colnr_T)startcol); @@ -17400,7 +17402,7 @@ void free_tv(typval_T *varp) switch (varp->v_type) { case VAR_FUNC: func_unref(varp->vval.v_string); - /*FALLTHROUGH*/ + // FALLTHROUGH case VAR_STRING: xfree(varp->vval.v_string); break; diff --git a/src/nvim/eval.h b/src/nvim/eval.h index c2b67cd12e..50c11011c6 100644 --- a/src/nvim/eval.h +++ b/src/nvim/eval.h @@ -53,7 +53,9 @@ EXTERN ufunc_T dumuf; #define HIKEY2UF(p) ((ufunc_T *)(p - (dumuf.uf_name - (char_u *)&dumuf))) #define HI2UF(hi) HIKEY2UF((hi)->hi_key) -/* Defines for Vim variables. These must match vimvars[] in eval.c! */ +/// Defines for Vim variables +/// +/// Order must match order in vimvars[] table in eval.c. typedef enum { VV_COUNT, VV_COUNT1, diff --git a/src/nvim/eval/decode.c b/src/nvim/eval/decode.c index de89f9c132..29a1b2a82a 100644 --- a/src/nvim/eval/decode.c +++ b/src/nvim/eval/decode.c @@ -69,6 +69,8 @@ static inline void create_special_dict(typval_T *const rettv, }; } +#define DICT_LEN(dict) (dict)->dv_hashtab.ht_used + /// Helper function used for working with stack vectors used by JSON decoder /// /// @param[in] obj New object. @@ -158,8 +160,7 @@ static inline int json_decoder_pop(ValuesStackItem obj, return FAIL; } else if (!obj.didcomma && (last_container.special_val == NULL - && (last_container.container.vval.v_dict->dv_hashtab.ht_used - != 0))) { + && (DICT_LEN(last_container.container.vval.v_dict) != 0))) { EMSG2(_("E474: Expected comma before dictionary key: %s"), val_location); clear_tv(&obj.val); return FAIL; @@ -191,6 +192,25 @@ static inline int json_decoder_pop(ValuesStackItem obj, return OK; } +#define OBJ(obj_tv, is_sp_string) \ + ((ValuesStackItem) { \ + .is_special_string = (is_sp_string), \ + .val = (obj_tv), \ + .didcomma = didcomma, \ + .didcolon = didcolon, \ + }) +#define POP(obj_tv, is_sp_string) \ + do { \ + if (json_decoder_pop(OBJ(obj_tv, is_sp_string), &stack, &container_stack, \ + &p, &next_map_special, &didcomma, &didcolon) \ + == FAIL) { \ + goto json_decode_string_fail; \ + } \ + if (next_map_special) { \ + goto json_decode_string_cycle_start; \ + } \ + } while (0) + /// Convert JSON string into VimL object /// /// @param[in] buf String to convert. UTF-8 encoding is assumed. @@ -215,24 +235,6 @@ int json_decode_string(const char *const buf, const size_t len, bool didcomma = false; bool didcolon = false; bool next_map_special = false; -#define OBJ(obj_tv, is_sp_string) \ - ((ValuesStackItem) { \ - .is_special_string = (is_sp_string), \ - .val = (obj_tv), \ - .didcomma = didcomma, \ - .didcolon = didcolon, \ - }) -#define POP(obj_tv, is_sp_string) \ - do { \ - if (json_decoder_pop(OBJ(obj_tv, is_sp_string), &stack, &container_stack, \ - &p, &next_map_special, &didcomma, &didcolon) \ - == FAIL) { \ - goto json_decode_string_fail; \ - } \ - if (next_map_special) { \ - goto json_decode_string_cycle_start; \ - } \ - } while (0) const char *p = buf; for (; p < e; p++) { json_decode_string_cycle_start: @@ -268,7 +270,8 @@ json_decode_string_cycle_start: goto json_decode_string_after_cycle; } else { if (json_decoder_pop(kv_pop(stack), &stack, &container_stack, &p, - &next_map_special, &didcomma, &didcolon) == FAIL) { + &next_map_special, &didcomma, &didcolon) + == FAIL) { goto json_decode_string_fail; } assert(!next_map_special); @@ -293,8 +296,7 @@ json_decode_string_cycle_start: goto json_decode_string_fail; } else if (last_container.special_val == NULL ? (last_container.container.v_type == VAR_DICT - ? (last_container.container.vval.v_dict->dv_hashtab.ht_used - == 0) + ? (DICT_LEN(last_container.container.vval.v_dict) == 0) : (last_container.container.vval.v_list->lv_len == 0)) : (last_container.special_val->lv_len == 0)) { EMSG2(_("E474: Leading comma: %s"), p); @@ -686,8 +688,6 @@ json_decode_string_cycle_start: break; } } -#undef POP -#undef OBJ json_decode_string_after_cycle: for (; p < e; p++) { switch (*p) { @@ -722,6 +722,11 @@ json_decode_string_ret: return ret; } +#undef POP +#undef OBJ + +#undef DICT_LEN + /// Convert msgpack object to a VimL one int msgpack_to_vim(const msgpack_object mobj, typval_T *const rettv) FUNC_ATTR_NONNULL_ALL FUNC_ATTR_WARN_UNUSED_RESULT diff --git a/src/nvim/message.h b/src/nvim/message.h index 9249596bec..0cd0ede2c4 100644 --- a/src/nvim/message.h +++ b/src/nvim/message.h @@ -41,7 +41,7 @@ /// Like #EMSG, but for messages with two "%s" inside #define EMSG3(s, p, q) emsg3((char_u *)(s), (char_u *)(p), \ - (char_u *)(q)) + (char_u *)(q)) /// Like #EMSG, but for messages with one "%" PRId64 inside #define EMSGN(s, n) emsgn((char_u *)(s), (int64_t)(n)) From f4ea114c672dbd62088b3107775060c58209a9b7 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sat, 6 Feb 2016 03:19:55 +0300 Subject: [PATCH 28/82] eval/decode: Fix vim_str2nr invocation --- src/nvim/eval/decode.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/nvim/eval/decode.c b/src/nvim/eval/decode.c index 29a1b2a82a..8a5684136b 100644 --- a/src/nvim/eval/decode.c +++ b/src/nvim/eval/decode.c @@ -470,10 +470,11 @@ json_decode_string_cycle_start: t++; switch (*t) { case 'u': { - const char ubuf[] = { t[1], t[2], t[3], t[4], 0 }; + const char ubuf[] = { t[1], t[2], t[3], t[4] }; t += 4; unsigned long ch; - vim_str2nr((char_u *) ubuf, NULL, NULL, 0, 0, 2, NULL, &ch); + vim_str2nr((char_u *) ubuf, NULL, NULL, + STR2NR_HEX | STR2NR_FORCE, NULL, &ch, 4); if (ch == 0) { hasnul = true; } @@ -622,7 +623,7 @@ json_decode_string_cycle_start: } else { // Convert integer long nr; - vim_str2nr((char_u *) s, NULL, NULL, 0, 0, 0, &nr, NULL); + vim_str2nr((char_u *) s, NULL, NULL, 0, &nr, NULL, (int) (p - s)); tv.vval.v_number = (varnumber_T) nr; } POP(tv, false); From 6cdf45e298631f0fba19d25cebba906a6e5fabcf Mon Sep 17 00:00:00 2001 From: ZyX Date: Sat, 6 Feb 2016 03:21:59 +0300 Subject: [PATCH 29/82] eval: Change dv_lock type to VarLockStatus --- src/nvim/eval_defs.h | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/src/nvim/eval_defs.h b/src/nvim/eval_defs.h index 3c119c44e1..8ffc0c98ce 100644 --- a/src/nvim/eval_defs.h +++ b/src/nvim/eval_defs.h @@ -120,19 +120,18 @@ typedef struct dictitem_S dictitem_T; #define DI_FLAGS_LOCK 8 // "di_flags" value: locked variable #define DI_FLAGS_ALLOC 16 // "di_flags" value: separately allocated -/* - * Structure to hold info about a Dictionary. - */ +/// Structure representing a Dictionary struct dictvar_S { - char dv_lock; /* zero, VAR_LOCKED, VAR_FIXED */ - char dv_scope; /* zero, VAR_SCOPE, VAR_DEF_SCOPE */ - int dv_refcount; /* reference count */ - int dv_copyID; /* ID used by deepcopy() */ - hashtab_T dv_hashtab; /* hashtab that refers to the items */ - dict_T *dv_copydict; /* copied dict used by deepcopy() */ - dict_T *dv_used_next; /* next dict in used dicts list */ - dict_T *dv_used_prev; /* previous dict in used dicts list */ - QUEUE watchers; // dictionary key watchers set by user code + VarLockStatus dv_lock; ///< Whole dictionary lock status. + char dv_scope; ///< Non-zero (#VAR_SCOPE, #VAR_DEF_SCOPE) if + ///< dictionary represents a scope (i.e. g:, l: …). + int dv_refcount; ///< Reference count. + int dv_copyID; ///< ID used when recursivery traversing a value. + hashtab_T dv_hashtab; ///< Hashtab containing all items. + dict_T *dv_copydict; ///< Copied dict used by deepcopy(). + dict_T *dv_used_next; ///< Next dictionary in used dictionaries list. + dict_T *dv_used_prev; ///< Previous dictionary in used dictionaries list. + QUEUE watchers; ///< Dictionary key watchers set by user code. }; // structure used for explicit stack while garbage collecting hash tables From c91c0171dd7c72717866569be96e48bf838cdf0f Mon Sep 17 00:00:00 2001 From: ZyX Date: Sat, 6 Feb 2016 03:31:45 +0300 Subject: [PATCH 30/82] *: Fix gcc warnings --- src/nvim/eval/decode.c | 6 +++--- src/nvim/eval/encode.c | 12 ++++++------ src/nvim/garray.c | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/nvim/eval/decode.c b/src/nvim/eval/decode.c index 8a5684136b..a89a9b8920 100644 --- a/src/nvim/eval/decode.c +++ b/src/nvim/eval/decode.c @@ -107,7 +107,7 @@ static inline int json_decoder_pop(ValuesStackItem obj, // vval.v_list and vval.v_dict should have the same size and offset && ((void *) obj.val.vval.v_list == (void *) last_container.container.vval.v_list)) { - kv_pop(*container_stack); + (void) kv_pop(*container_stack); val_location = last_container.s; last_container = kv_last(*container_stack); } @@ -175,7 +175,7 @@ static inline int json_decoder_pop(ValuesStackItem obj, clear_tv(&obj.val); // Restart - kv_pop(*container_stack); + (void) kv_pop(*container_stack); ValuesStackItem last_container_val = kv_A(*stack, last_container.stack_index); while (kv_size(*stack) > last_container.stack_index) { @@ -266,7 +266,7 @@ json_decode_string_cycle_start: } if (kv_size(stack) == 1) { p++; - kv_pop(container_stack); + (void) kv_pop(container_stack); goto json_decode_string_after_cycle; } else { if (json_decoder_pop(kv_pop(stack), &stack, &container_stack, &p, diff --git a/src/nvim/eval/encode.c b/src/nvim/eval/encode.c index 1550a61f7f..48fbc44b0c 100644 --- a/src/nvim/eval/encode.c +++ b/src/nvim/eval/encode.c @@ -755,7 +755,7 @@ encode_vim_to_##name##_error_ret: \ char ebuf[NUMBUFLEN + 7]; \ size_t backref = 0; \ for (; backref < kv_size(*mpstack); backref++) { \ - const MPConvStackVal mpval = kv_a(MPConvStackVal, *mpstack, backref); \ + const MPConvStackVal mpval = kv_A(*mpstack, backref); \ if (mpval.type == conv_type) { \ if (conv_type == kMPConvDict) { \ if ((void *) mpval.data.d.dict == (void *) (val)) { \ @@ -783,7 +783,7 @@ DEFINE_VIML_CONV_FUNCTIONS(static, string, garray_T *const, gap) char ebuf[NUMBUFLEN + 7]; \ size_t backref = 0; \ for (; backref < kv_size(*mpstack); backref++) { \ - const MPConvStackVal mpval = kv_a(MPConvStackVal, *mpstack, backref); \ + const MPConvStackVal mpval = kv_A(*mpstack, backref); \ if (mpval.type == conv_type) { \ if (conv_type == kMPConvDict) { \ if ((void *) mpval.data.d.dict == (void *) val) { \ @@ -932,7 +932,7 @@ static inline int convert_to_json_string(garray_T *const gap, } else if (vim_isprintc(ch)) { str_len += shift; } else { - str_len += ((sizeof("\\u1234") - 1) * (1 + (ch > 0xFFFF))); + str_len += ((sizeof("\\u1234") - 1) * (size_t) (1 + (ch > 0xFFFF))); } break; } @@ -969,9 +969,9 @@ static inline int convert_to_json_string(garray_T *const gap, xdigits[(ch >> (4 * 0)) & 0xF], }), sizeof("\\u1234") - 1); } else { - uint32_t tmp = (uint32_t) ch - SURROGATE_FIRST_CHAR; - uint16_t hi = SURROGATE_HI_START + ((tmp >> 10) & ((1 << 10) - 1)); - uint16_t lo = SURROGATE_LO_END + ((tmp >> 0) & ((1 << 10) - 1)); + const int tmp = ch - SURROGATE_FIRST_CHAR; + const int hi = SURROGATE_HI_START + ((tmp >> 10) & ((1 << 10) - 1)); + const int lo = SURROGATE_LO_END + ((tmp >> 0) & ((1 << 10) - 1)); ga_concat_len(gap, ((const char[]) { '\\', 'u', xdigits[(hi >> (4 * 3)) & 0xF], diff --git a/src/nvim/garray.c b/src/nvim/garray.c index 47a370b18c..98cec69b54 100644 --- a/src/nvim/garray.c +++ b/src/nvim/garray.c @@ -204,7 +204,7 @@ void ga_concat_len(garray_T *const gap, const char *restrict s, ga_grow(gap, (int) len); char *data = gap->ga_data; memcpy(data + gap->ga_len, s, len); - gap->ga_len += len; + gap->ga_len += (int) len; } } From b7cb8f0597c42f209ec460a1850df53bd655c81c Mon Sep 17 00:00:00 2001 From: ZyX Date: Sat, 6 Feb 2016 19:09:24 +0300 Subject: [PATCH 31/82] eval: Make assert_true and assert_false accept v:true and v:false --- src/nvim/eval.c | 12 ++++++++---- test/functional/eval/special_vars_spec.lua | 17 +++++++++++++++++ 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/src/nvim/eval.c b/src/nvim/eval.c index 22a12d353a..63fd392a3a 100644 --- a/src/nvim/eval.c +++ b/src/nvim/eval.c @@ -7587,16 +7587,20 @@ static void f_assert_equal(typval_T *argvars, typval_T *rettv) } // Common for assert_true() and assert_false(). -static void assert_bool(typval_T *argvars, bool isTrue) +static void assert_bool(typval_T *argvars, bool is_true) { int error = (int)false; garray_T ga; - if (argvars[0].v_type != VAR_NUMBER || - (get_tv_number_chk(&argvars[0], &error) == 0) == isTrue || error) { + if ((argvars[0].v_type != VAR_NUMBER || + (get_tv_number_chk(&argvars[0], &error) == 0) == is_true || error) + && (argvars[0].v_type != VAR_SPECIAL + || argvars[0].vval.v_special != (is_true + ?kSpecialVarTrue + :kSpecialVarFalse))) { prepare_assert_error(&ga); fill_assert_error(&ga, &argvars[1], - (char_u *)(isTrue ? "True" : "False"), + (char_u *)(is_true ? "True" : "False"), NULL, &argvars[0]); assert_error(&ga); ga_clear(&ga); diff --git a/test/functional/eval/special_vars_spec.lua b/test/functional/eval/special_vars_spec.lua index b5c65d23d9..7539261d49 100644 --- a/test/functional/eval/special_vars_spec.lua +++ b/test/functional/eval/special_vars_spec.lua @@ -151,4 +151,21 @@ describe('Special values', function() eq('Vim(echo):E15: Cannot index a special value', exc_exec('echo v:false[0]')) eq('Vim(echo):E15: Cannot index a special value', exc_exec('echo v:null[0]')) end) + + it('is accepted by assert_true and assert_false', function() + funcs.assert_false(false) + funcs.assert_false(true) + eval('assert_false(v:null)') + + funcs.assert_true(false) + funcs.assert_true(true) + eval('assert_true(v:null)') + + eq({ + 'Expected False but got v:true', + 'Expected False but got v:null', + 'Expected True but got v:false', + 'Expected True but got v:null', + }, meths.get_vvar('errors')) + end) end) From 0aa3e7b7ceb259680f9da31bd247b42cdc934449 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sat, 6 Feb 2016 20:54:42 +0300 Subject: [PATCH 32/82] eval: Port parts of 7.4.1267 that are not already present --- runtime/doc/vim_diff.txt | 2 + src/nvim/eval.c | 79 +++++++++++++++------- src/nvim/version.c | 2 +- test/functional/eval/special_vars_spec.lua | 6 +- 4 files changed, 59 insertions(+), 30 deletions(-) diff --git a/runtime/doc/vim_diff.txt b/runtime/doc/vim_diff.txt index 2d03b4ff45..b42b91140c 100644 --- a/runtime/doc/vim_diff.txt +++ b/runtime/doc/vim_diff.txt @@ -99,6 +99,8 @@ are always available and may be used simultaneously in separate plugins. The error out. 4. Stringifyed infinite and NaN values now use |str2float()| and can be evaled back. +5. (internal) Trying to print or stringify VAR_UNKNOWN in Vim results in + nothing, |E908|, in Neovim it is internal error. |jsondecode()| behaviour changed: 1. It may output |msgpack-special-dict|. diff --git a/src/nvim/eval.c b/src/nvim/eval.c index 63fd392a3a..9f5a3985d3 100644 --- a/src/nvim/eval.c +++ b/src/nvim/eval.c @@ -4319,19 +4319,37 @@ eval_index ( char_u *s; char_u *key = NULL; - if (rettv->v_type == VAR_FUNC) { - if (verbose) - EMSG(_("E695: Cannot index a Funcref")); - return FAIL; - } else if (rettv->v_type == VAR_FLOAT) { - if (verbose) - EMSG(_(e_float_as_string)); - return FAIL; - } else if (rettv->v_type == VAR_SPECIAL) { - if (verbose) { - EMSG(_("E15: Cannot index a special value")); + switch (rettv->v_type) { + case VAR_FUNC: { + if (verbose) { + EMSG(_("E695: Cannot index a Funcref")); + } + return FAIL; + } + case VAR_FLOAT: { + if (verbose) { + EMSG(_(e_float_as_string)); + } + return FAIL; + } + case VAR_SPECIAL: { + if (verbose) { + EMSG(_("E909: Cannot index a special variable")); + } + return FAIL; + } + case VAR_UNKNOWN: { + if (evaluate) { + return FAIL; + } + // fallthrough + } + case VAR_STRING: + case VAR_NUMBER: + case VAR_LIST: + case VAR_DICT: { + break; } - return FAIL; } init_tv(&var1); @@ -4522,11 +4540,11 @@ eval_index ( *rettv = var1; } break; + case VAR_SPECIAL: case VAR_FUNC: case VAR_FLOAT: case VAR_UNKNOWN: - case VAR_SPECIAL: - assert(false); + break; // Not evaluating, skipping over subscript } } @@ -5076,11 +5094,12 @@ tv_equal ( return tv1->vval.v_special == tv2->vval.v_special; case VAR_UNKNOWN: - break; + // VAR_UNKNOWN can be the result of an invalid expression, let’s say it does + // not equal anything, not even self. + return false; } - EMSG2(_(e_intern2), "tv_equal()"); - return TRUE; + assert(false); } /* @@ -8516,7 +8535,7 @@ static void f_diff_hlID(typval_T *argvars, typval_T *rettv) */ static void f_empty(typval_T *argvars, typval_T *rettv) { - int n; + bool n; switch (argvars[0].v_type) { case VAR_STRING: @@ -8542,8 +8561,9 @@ static void f_empty(typval_T *argvars, typval_T *rettv) n = argvars[0].vval.v_special != kSpecialVarTrue; break; case VAR_UNKNOWN: - EMSG2(_(e_intern2), "f_empty()"); - n = 0; + EMSG2(_(e_intern2), "f_empty(UNKNOWN)"); + n = true; + break; } rettv->vval.v_number = n; @@ -11641,7 +11661,10 @@ static void f_len(typval_T *argvars, typval_T *rettv) case VAR_DICT: rettv->vval.v_number = dict_len(argvars[0].vval.v_dict); break; - default: + case VAR_UNKNOWN: + case VAR_SPECIAL: + case VAR_FLOAT: + case VAR_FUNC: EMSG(_("E701: Invalid type for len()")); break; } @@ -16215,7 +16238,11 @@ static void f_type(typval_T *argvars, typval_T *rettv) } break; } - case VAR_UNKNOWN: EMSG2(_(e_intern2), "f_type()"); n = 0; break; + case VAR_UNKNOWN: { + EMSG2(_(e_intern2), "f_type(UNKNOWN)"); + n = -1; + break; + } } rettv->vval.v_number = n; } @@ -17529,7 +17556,7 @@ long get_tv_number_chk(typval_T *varp, int *denote) } break; case VAR_UNKNOWN: - EMSG2(_(e_intern2), "get_tv_number()"); + EMSG2(_(e_intern2), "get_tv_number(UNKNOWN)"); break; } if (denote == NULL) { @@ -17638,7 +17665,7 @@ static char_u *get_tv_string_buf_chk(const typval_T *varp, char_u *buf) STRCPY(buf, encode_special_var_names[varp->vval.v_special]); return buf; case VAR_UNKNOWN: - EMSG2(_(e_intern2), "get_tv_string_buf()"); + EMSG(_("E908: using an invalid value as a String")); break; } return NULL; @@ -18212,7 +18239,7 @@ void copy_tv(typval_T *from, typval_T *to) } break; case VAR_UNKNOWN: - EMSG2(_(e_intern2), "copy_tv()"); + EMSG2(_(e_intern2), "copy_tv(UNKNOWN)"); break; } } @@ -18301,7 +18328,7 @@ int var_item_copy(const vimconv_T *const conv, ret = FAIL; break; case VAR_UNKNOWN: - EMSG2(_(e_intern2), "var_item_copy()"); + EMSG2(_(e_intern2), "var_item_copy(UNKNOWN)"); ret = FAIL; } --recurse; diff --git a/src/nvim/version.c b/src/nvim/version.c index 262adcba34..78f3857256 100644 --- a/src/nvim/version.c +++ b/src/nvim/version.c @@ -95,7 +95,7 @@ static int included_patches[] = { // 1270 // 1269 // 1268 - // 1267 + 1267, // 1266 // 1265 // 1264 diff --git a/test/functional/eval/special_vars_spec.lua b/test/functional/eval/special_vars_spec.lua index 7539261d49..b003dceb04 100644 --- a/test/functional/eval/special_vars_spec.lua +++ b/test/functional/eval/special_vars_spec.lua @@ -147,9 +147,9 @@ describe('Special values', function() end) it('fails in index', function() - eq('Vim(echo):E15: Cannot index a special value', exc_exec('echo v:true[0]')) - eq('Vim(echo):E15: Cannot index a special value', exc_exec('echo v:false[0]')) - eq('Vim(echo):E15: Cannot index a special value', exc_exec('echo v:null[0]')) + eq('Vim(echo):E909: Cannot index a special variable', exc_exec('echo v:true[0]')) + eq('Vim(echo):E909: Cannot index a special variable', exc_exec('echo v:false[0]')) + eq('Vim(echo):E909: Cannot index a special variable', exc_exec('echo v:null[0]')) end) it('is accepted by assert_true and assert_false', function() From d4106f6df335b9609939d11289dd9faae2c85bb6 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sat, 6 Feb 2016 21:03:33 +0300 Subject: [PATCH 33/82] shada: Make sure that NIL and EXT values can also be parsed back MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Note: currently they are both *dumped*, but parsing them produces an error. This is inappropriate: variables should either be skipped with error message when dumping or should be read back properly. It also appears that I did not have test for “has wrong variable value type” error, so nothing got removed from errors_spec. --- src/nvim/shada.c | 6 ------ test/functional/shada/variables_spec.lua | 13 +++++++++++-- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/nvim/shada.c b/src/nvim/shada.c index a18fd725d2..84880d1a99 100644 --- a/src/nvim/shada.c +++ b/src/nvim/shada.c @@ -3883,12 +3883,6 @@ shada_read_next_item_hist_no_conv: initial_fpos); goto shada_read_next_item_error; } - if (unpacked.data.via.array.ptr[1].type == MSGPACK_OBJECT_NIL - || unpacked.data.via.array.ptr[1].type == MSGPACK_OBJECT_EXT) { - emsgu(_(READERR("variable", "has wrong variable value type")), - initial_fpos); - goto shada_read_next_item_error; - } entry->data.global_var.name = xmemdupz(unpacked.data.via.array.ptr[0].via.bin.ptr, unpacked.data.via.array.ptr[0].via.bin.size); diff --git a/test/functional/shada/variables_spec.lua b/test/functional/shada/variables_spec.lua index 3becf1bc32..7ceeafdc71 100644 --- a/test/functional/shada/variables_spec.lua +++ b/test/functional/shada/variables_spec.lua @@ -22,12 +22,17 @@ describe('ShaDa support code', function() eq('foo', meths.get_var('STRVAR')) end) - local autotest = function(tname, varname, varval) + local autotest = function(tname, varname, varval, val_is_expr) it('is able to dump and read back ' .. tname .. ' variable automatically', function() set_additional_cmd('set shada+=!') reset() - meths.set_var(varname, varval) + if val_is_expr then + nvim_command('let g:' .. varname .. ' = ' .. varval) + varval = meths.get_var(varname) + else + meths.set_var(varname, varval) + end -- Exit during `reset` is not a regular exit: it does not write shada -- automatically nvim_command('qall') @@ -41,6 +46,10 @@ describe('ShaDa support code', function() autotest('float', 'FLTVAR', 42.5) autotest('dictionary', 'DCTVAR', {a=10}) autotest('list', 'LSTVAR', {{a=10}, {b=10.5}, {c='str'}}) + autotest('true', 'TRUEVAR', true) + autotest('false', 'FALSEVAR', false) + autotest('null', 'NULLVAR', 'v:null', true) + autotest('ext', 'EXTVAR', '{"_TYPE": v:msgpack_types.ext, "_VAL": [2, ["", ""]]}', true) it('does not read back variables without `!` in &shada', function() meths.set_var('STRVAR', 'foo') From 7124329bd915e3896b7f09083ff394cd7f598cb8 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sat, 6 Feb 2016 22:19:23 +0300 Subject: [PATCH 34/82] *: Fix memory leaks found by clang sanitizer --- src/nvim/eval/decode.c | 1 + src/nvim/eval/encode.c | 10 +++++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/nvim/eval/decode.c b/src/nvim/eval/decode.c index a89a9b8920..c6706eb0dd 100644 --- a/src/nvim/eval/decode.c +++ b/src/nvim/eval/decode.c @@ -552,6 +552,7 @@ json_decode_string_cycle_start: clear_tv(&obj); goto json_decode_string_fail; } + xfree(str); POP(obj, true); } else { *str_end = NUL; diff --git a/src/nvim/eval/encode.c b/src/nvim/eval/encode.c index 48fbc44b0c..2df689990a 100644 --- a/src/nvim/eval/encode.c +++ b/src/nvim/eval/encode.c @@ -613,7 +613,7 @@ scope int encode_vim_to_##name(firstargtype firstargname, typval_T *const tv, \ CONV_DICT_BETWEEN_ITEMS(); \ } \ const list_T *const kv_pair = cur_mpsv->data.l.li->li_tv.vval.v_list; \ - CONV_SPECIAL_DICT_KEY_CHECK(kv_pair); \ + CONV_SPECIAL_DICT_KEY_CHECK(name, kv_pair); \ if (name##_convert_one_value(firstargname, &mpstack, \ &kv_pair->lv_first->li_tv, copyID, \ objname) == FAIL) { \ @@ -735,7 +735,7 @@ encode_vim_to_##name##_error_ret: \ #define CONV_DICT_BETWEEN_ITEMS() \ ga_concat(gap, ", ") -#define CONV_SPECIAL_DICT_KEY_CHECK(kv_pair) +#define CONV_SPECIAL_DICT_KEY_CHECK(name, kv_pair) #define CONV_LIST_END(lst) \ ga_append(gap, ']') @@ -1058,11 +1058,11 @@ static inline bool check_json_key(const typval_T *const tv) } #undef CONV_SPECIAL_DICT_KEY_CHECK -#define CONV_SPECIAL_DICT_KEY_CHECK(kv_pair) \ +#define CONV_SPECIAL_DICT_KEY_CHECK(name, kv_pair) \ do { \ if (!check_json_key(&kv_pair->lv_first->li_tv)) { \ EMSG(_("E474: Invalid key in special dictionary")); \ - return FAIL; \ + goto encode_vim_to_##name##_error_ret; \ } \ } while (0) @@ -1235,7 +1235,7 @@ char *encode_tv2json(typval_T *tv, size_t *len) #define CONV_DICT_BETWEEN_ITEMS() -#define CONV_SPECIAL_DICT_KEY_CHECK(kv_pair) +#define CONV_SPECIAL_DICT_KEY_CHECK(name, kv_pair) #define CONV_LIST_END(lst) From 569e404622900222d88d856adbc6421734146bea Mon Sep 17 00:00:00 2001 From: ZyX Date: Sat, 6 Feb 2016 23:07:53 +0300 Subject: [PATCH 35/82] eval/encode: Fix non-utf-8 &encoding handling, add tests --- src/nvim/eval/encode.c | 45 ++++++++++++-------- test/functional/eval/json_functions_spec.lua | 27 ++++++++++-- 2 files changed, 52 insertions(+), 20 deletions(-) diff --git a/src/nvim/eval/encode.c b/src/nvim/eval/encode.c index 2df689990a..b29a4c6f21 100644 --- a/src/nvim/eval/encode.c +++ b/src/nvim/eval/encode.c @@ -882,11 +882,11 @@ static inline int convert_to_json_string(garray_T *const gap, const size_t len) FUNC_ATTR_NONNULL_ARG(1) FUNC_ATTR_ALWAYS_INLINE { - const char *buf_ = buf; - if (buf_ == NULL) { + const char *utf_buf = buf; + if (utf_buf == NULL) { ga_concat(gap, "\"\""); } else { - size_t len_ = len; + size_t utf_len = len; char *tofree = NULL; if (last_p_enc != (const void *) p_enc) { p_enc_conv.vc_type = CONV_NONE; @@ -895,17 +895,28 @@ static inline int convert_to_json_string(garray_T *const gap, last_p_enc = p_enc; } if (p_enc_conv.vc_type != CONV_NONE) { - tofree = string_convert(&p_enc_conv, buf_, &len_); + tofree = string_convert(&p_enc_conv, buf, &utf_len); if (tofree == NULL) { - EMSG2(_("E474: Failed to convert string \"%s\" to UTF-8"), buf_); + EMSG2(_("E474: Failed to convert string \"%s\" to UTF-8"), utf_buf); return FAIL; } - buf_ = tofree; + utf_buf = tofree; } size_t str_len = 0; - for (size_t i = 0; i < len_;) { - const int ch = utf_ptr2char(buf + i); - const size_t shift = (ch == 0? 1: utf_ptr2len(buf + i)); + // Encode character as \u0000 if + // 1. It is an ASCII control character (0x0 .. 0x1F, 0x7F). + // 2. &encoding is not UTF-8 and code point is above 0x7F. + // 3. &encoding is UTF-8 and code point is not printable according to + // utf_printable(). + // This is done to make it possible to :echo values when &encoding is not + // UTF-8. +#define ENCODE_RAW(p_enc_conv, ch) \ + (ch >= 0x20 && (p_enc_conv.vc_type == CONV_NONE \ + ? utf_printable(ch) \ + : ch < 0x7F)) + for (size_t i = 0; i < utf_len;) { + const int ch = utf_ptr2char(utf_buf + i); + const size_t shift = (ch == 0? 1: utf_ptr2len(utf_buf + i)); assert(shift > 0); i += shift; switch (ch) { @@ -922,14 +933,14 @@ static inline int convert_to_json_string(garray_T *const gap, default: { if (ch > 0x7F && shift == 1) { EMSG2(_("E474: String \"%s\" contains byte that does not start any " - "UTF-8 character"), buf_); + "UTF-8 character"), utf_buf); return FAIL; } else if ((0xD800 <= ch && ch <= 0xDB7F) || (0xDC00 <= ch && ch <= 0xDFFF)) { EMSG2(_("E474: UTF-8 string contains code point which belongs " - "to surrogate pairs"), buf_); + "to surrogate pairs: %s"), utf_buf + i); return FAIL; - } else if (vim_isprintc(ch)) { + } else if (ENCODE_RAW(p_enc_conv, ch)) { str_len += shift; } else { str_len += ((sizeof("\\u1234") - 1) * (size_t) (1 + (ch > 0xFFFF))); @@ -940,12 +951,12 @@ static inline int convert_to_json_string(garray_T *const gap, } ga_append(gap, '"'); ga_grow(gap, (int) str_len); - for (size_t i = 0; i < len_;) { - const int ch = utf_ptr2char(buf + i); + for (size_t i = 0; i < utf_len;) { + const int ch = utf_ptr2char(utf_buf + i); const size_t shift = (ch == 0? 1: utf_char2len(ch)); assert(shift > 0); // Is false on invalid unicode, but this should already be handled. - assert(ch == 0 || shift == utf_ptr2len(buf + i)); + assert(ch == 0 || shift == utf_ptr2len(utf_buf + i)); switch (ch) { case BS: case TAB: @@ -958,8 +969,8 @@ static inline int convert_to_json_string(garray_T *const gap, break; } default: { - if (vim_isprintc(ch)) { - ga_concat_len(gap, buf + i, shift); + if (ENCODE_RAW(p_enc_conv, ch)) { + ga_concat_len(gap, utf_buf + i, shift); } else if (ch < SURROGATE_FIRST_CHAR) { ga_concat_len(gap, ((const char[]) { '\\', 'u', diff --git a/test/functional/eval/json_functions_spec.lua b/test/functional/eval/json_functions_spec.lua index 6f81a36479..13597eb7a0 100644 --- a/test/functional/eval/json_functions_spec.lua +++ b/test/functional/eval/json_functions_spec.lua @@ -8,8 +8,8 @@ local execute = helpers.execute local exc_exec = helpers.exc_exec describe('jsondecode() function', function() - before_each(function() - clear() + local restart = function(cmd) + clear(cmd) execute([[ function Eq(exp, act) let act = a:act @@ -53,7 +53,8 @@ describe('jsondecode() function', function() endif endfunction ]]) - end) + end + before_each(restart) local speq = function(expected, actual_expr) eq(1, funcs.EvalEq(expected, actual_expr)) @@ -396,6 +397,7 @@ describe('jsondecode() function', function() it('parses strings with NUL properly', function() sp_decode_eq({_TYPE='string', _VAL={'\n'}}, '"\\u0000"') sp_decode_eq({_TYPE='string', _VAL={'\n', '\n'}}, '"\\u0000\\n\\u0000"') + sp_decode_eq({_TYPE='string', _VAL={'\n«\n'}}, '"\\u0000\\u00AB\\u0000"') end) it('parses dictionaries with duplicate keys to special maps', function() @@ -436,6 +438,12 @@ describe('jsondecode() function', function() sp_decode_eq({_TYPE='map', _VAL={{'b', 3}, {'a', 1}, {'c', 4}, {'d', 2}, {{_TYPE='string', _VAL={'\n'}}, 4}}}, '{"b": 3, "a": 1, "c": 4, "d": 2, "\\u0000": 4}') end) + + it('converts strings to latin1 when &encoding is latin1', function() + restart('set encoding=latin1') + eq('\xAB', funcs.jsondecode('"\\u00AB"')) + sp_decode_eq({_TYPE='string', _VAL={'\n\xAB\n'}}, '"\\u0000\\u00AB\\u0000"') + end) end) describe('jsonencode() function', function() @@ -447,6 +455,7 @@ describe('jsonencode() function', function() eq('"\\t"', funcs.jsonencode('\t')) eq('"\\n"', funcs.jsonencode('\n')) eq('"\\u001B"', funcs.jsonencode('\27')) + eq('"þÿþ"', funcs.jsonencode('þÿþ')) end) it('dumps numbers', function() @@ -642,4 +651,16 @@ describe('jsonencode() function', function() eq('Vim(call):E118: Too many arguments for function: jsonencode', exc_exec('call jsonencode(["", ""], 1)')) end) + + it('converts strings from latin1 when &encoding is latin1', function() + clear('set encoding=latin1') + eq('"\\u00AB"', funcs.jsonencode('\xAB')) + eq('"\\u0000\\u00AB\\u0000"', eval('jsonencode({"_TYPE": v:msgpack_types.string, "_VAL": ["\\n\xAB\\n"]})')) + end) + + it('ignores improper values in &isprint', function() + meths.set_option('isprint', '1') + eq(1, eval('"\x01" =~# "\\\\p"')) + eq('"\\u0001"', funcs.jsonencode('\x01')) + end) end) From fa26eee85b7da5ab562ff9e57d6303a423a8c850 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sat, 6 Feb 2016 23:10:36 +0300 Subject: [PATCH 36/82] version: Record that vim-7.4.1271 is kinda of there --- src/nvim/version.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nvim/version.c b/src/nvim/version.c index 78f3857256..8814a592cf 100644 --- a/src/nvim/version.c +++ b/src/nvim/version.c @@ -91,7 +91,7 @@ static int included_patches[] = { // 1274 // 1273 // 1272 - // 1271 + 1271, // 1270 // 1269 // 1268 From c27395ddc84952b94118de94af4c33f56f6beca5 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 7 Feb 2016 00:20:45 +0300 Subject: [PATCH 37/82] eval: Fix QuickBuild failures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Compiler used by one VM in QuickBuild has found a number of false positives. Everything is fine on travis. List of failures: From [QuickBuild][1], build [7429][2]: 14:38:19,945 WARN - /home/quickbuild/buildagent/workspace/root/neovim/pull-requests-automated/src/nvim/eval.c: In function ‘assert_bool’: 14:38:19,945 WARN - /home/quickbuild/buildagent/workspace/root/neovim/pull-requests-automated/src/nvim/eval.c:7551:40: error: comparison between signed and unsigned integer expressions [-Werror=sign-compare] 14:38:20,058 WARN - cc1: all warnings being treated as errors . This is not making much sense (7551:40 is `!=` in `{SpecialVarValue} != ({bool}?{SpecialVarValue}:{SpecialVarValue})`), but this error is present. --- Also fail from [build][3] [4930][4]: 15:47:00,853 WARN - /home/quickbuild/buildagent/workspace/root/neovim/pull-requests-automated/src/nvim/eval/encode.c: In function ‘encode_read_from_list’: 15:47:00,853 WARN - /home/quickbuild/buildagent/workspace/root/neovim/pull-requests-automated/src/nvim/eval/encode.c:258:30: error: conversion to ‘char’ from ‘int’ may alter its value [-Werror=conversion] , pointing to `:` in `{char} = ({char} == {const} ? {const} : {char})` where `{const}` is character constant like `'\n'`. I have no idea where exactly it saw conversion, so simply casted everything to (char). --- [Build][5] error: 08:32:03,472 WARN - /home/quickbuild/buildagent/workspace/root/neovim/pull-requests-automated/src/nvim/eval.c: In function ‘tv_equal’: 08:32:03,472 WARN - /home/quickbuild/buildagent/workspace/root/neovim/pull-requests-automated/src/nvim/eval.c:5077:1: error: control reaches end of non-void function [-Werror=return-type] --- Build [4949][7]: 11:28:00,578 WARN - /home/quickbuild/buildagent/workspace/root/neovim/pull-requests-automated/src/nvim/eval.c: In function ‘f_type’: 11:28:00,578 WARN - /home/quickbuild/buildagent/workspace/root/neovim/pull-requests-automated/src/nvim/eval.c:16085:24: error: ‘n’ may be used uninitialized in this function [-Werror=uninitialized] 11:28:00,581 WARN - /home/quickbuild/buildagent/workspace/root/neovim/pull-requests-automated/src/nvim/eval.c: In function ‘f_empty’: 11:28:00,581 WARN - /home/quickbuild/buildagent/workspace/root/neovim/pull-requests-automated/src/nvim/eval.c:8505:24: error: ‘n’ may be used uninitialized in this function [-Werror=uninitialized] [1]: http://neovim-qb.szakmeister.net/wicket/page?5-1.ILinkListener-content-buildTab-panel-masterStep-body-children-0-step-body-children-2-body-children-3-step-body-children-0-step-body-children-0-step-head-logLink [2]: http://neovim-qb.szakmeister.net/build/4929 [3]: http://neovim-qb.szakmeister.net/build/4930 [4]: http://neovim-qb.szakmeister.net/wicket/page?1-1.ILinkListener-content-buildTab-panel-masterStep-body-children-0-step-body-children-1-body-children-3-step-body-children-0-step-body-children-0-step-head-logLink [5]: http://neovim-qb.szakmeister.net/build/4948/step_status [7]: http://neovim-qb.szakmeister.net/build/4949 --- src/nvim/eval.c | 14 +++++++------- src/nvim/eval/encode.c | 2 +- src/nvim/version.c | 1 + 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/nvim/eval.c b/src/nvim/eval.c index 9f5a3985d3..d2a7aeb74e 100644 --- a/src/nvim/eval.c +++ b/src/nvim/eval.c @@ -5100,6 +5100,7 @@ tv_equal ( } assert(false); + return false; } /* @@ -7614,9 +7615,10 @@ static void assert_bool(typval_T *argvars, bool is_true) if ((argvars[0].v_type != VAR_NUMBER || (get_tv_number_chk(&argvars[0], &error) == 0) == is_true || error) && (argvars[0].v_type != VAR_SPECIAL - || argvars[0].vval.v_special != (is_true - ?kSpecialVarTrue - :kSpecialVarFalse))) { + || (argvars[0].vval.v_special + != (SpecialVarValue) (is_true + ? kSpecialVarTrue + : kSpecialVarFalse)))) { prepare_assert_error(&ga); fill_assert_error(&ga, &argvars[1], (char_u *)(is_true ? "True" : "False"), @@ -8535,7 +8537,7 @@ static void f_diff_hlID(typval_T *argvars, typval_T *rettv) */ static void f_empty(typval_T *argvars, typval_T *rettv) { - bool n; + bool n = true; switch (argvars[0].v_type) { case VAR_STRING: @@ -8562,7 +8564,6 @@ static void f_empty(typval_T *argvars, typval_T *rettv) break; case VAR_UNKNOWN: EMSG2(_(e_intern2), "f_empty(UNKNOWN)"); - n = true; break; } @@ -16215,7 +16216,7 @@ static void f_trunc(typval_T *argvars, typval_T *rettv) */ static void f_type(typval_T *argvars, typval_T *rettv) { - int n; + int n = -1; switch (argvars[0].v_type) { case VAR_NUMBER: n = 0; break; @@ -16240,7 +16241,6 @@ static void f_type(typval_T *argvars, typval_T *rettv) } case VAR_UNKNOWN: { EMSG2(_(e_intern2), "f_type(UNKNOWN)"); - n = -1; break; } } diff --git a/src/nvim/eval/encode.c b/src/nvim/eval/encode.c index b29a4c6f21..8280889fbe 100644 --- a/src/nvim/eval/encode.c +++ b/src/nvim/eval/encode.c @@ -255,7 +255,7 @@ int encode_read_from_list(ListReaderState *const state, char *const buf, while (p < buf_end) { for (size_t i = state->offset; i < state->li_length && p < buf_end; i++) { const char ch = (char) state->li->li_tv.vval.v_string[state->offset++]; - *p++ = (ch == NL ? NUL : ch); + *p++ = (char) ((char) ch == (char) NL ? (char) NUL : (char) ch); } if (p < buf_end) { state->li = state->li->li_next; diff --git a/src/nvim/version.c b/src/nvim/version.c index 8814a592cf..106cc8d1d8 100644 --- a/src/nvim/version.c +++ b/src/nvim/version.c @@ -78,6 +78,7 @@ static int included_patches[] = { 1511, 1425, 1366, + 1292, 1284, // 1283 1282, From 77776b09c684bc2a0c42114fce5a8b04409ec91d Mon Sep 17 00:00:00 2001 From: ZyX Date: Tue, 9 Feb 2016 03:20:16 +0300 Subject: [PATCH 38/82] eval/encode: Fix writing strings starting with NL to list Error [found][1] by oni-link. [1]: https://github.com/neovim/neovim/pull/4131/files#r52239384 --- src/nvim/eval.h | 2 - src/nvim/eval/encode.c | 24 ++++---- test/unit/eval/encode_spec.lua | 100 +++++++++++++++++++++++++++++++++ test/unit/eval/helpers.lua | 72 ++++++++++++++++++++++++ test/unit/helpers.lua | 11 +++- 5 files changed, 192 insertions(+), 17 deletions(-) create mode 100644 test/unit/eval/encode_spec.lua create mode 100644 test/unit/eval/helpers.lua diff --git a/src/nvim/eval.h b/src/nvim/eval.h index 50c11011c6..9d45b780a9 100644 --- a/src/nvim/eval.h +++ b/src/nvim/eval.h @@ -1,8 +1,6 @@ #ifndef NVIM_EVAL_H #define NVIM_EVAL_H -#include - #include "nvim/profile.h" #include "nvim/hashtab.h" // For hashtab_T #include "nvim/garray.h" // For garray_T diff --git a/src/nvim/eval/encode.c b/src/nvim/eval/encode.c index 8280889fbe..6026189235 100644 --- a/src/nvim/eval/encode.c +++ b/src/nvim/eval/encode.c @@ -79,11 +79,9 @@ int encode_list_write(void *data, const char *buf, size_t len) do { const char *line_start = line_end; line_end = xmemscan(line_start, NL, (size_t) (end - line_start)); - if (line_end == line_start) { - list_append_allocated_string(list, NULL); - } else { + char *str = NULL; + if (line_end != line_start) { const size_t line_length = (size_t) (line_end - line_start); - char *str; if (li == NULL) { str = xmemdupz(line_start, line_length); } else { @@ -93,7 +91,7 @@ int encode_list_write(void *data, const char *buf, size_t len) li->li_tv.vval.v_string = xrealloc(li->li_tv.vval.v_string, li_len + line_length + 1); str = (char *) li->li_tv.vval.v_string + li_len; - memmove(str, line_start, line_length); + memcpy(str, line_start, line_length); str[line_length] = 0; } for (size_t i = 0; i < line_length; i++) { @@ -101,14 +99,14 @@ int encode_list_write(void *data, const char *buf, size_t len) str[i] = NL; } } - if (li == NULL) { - list_append_allocated_string(list, str); - } else { - li = NULL; - } - if (line_end == end - 1) { - list_append_allocated_string(list, NULL); - } + } + if (li == NULL) { + list_append_allocated_string(list, str); + } else { + li = NULL; + } + if (line_end == end - 1) { + list_append_allocated_string(list, NULL); } line_end++; } while (line_end < end); diff --git a/test/unit/eval/encode_spec.lua b/test/unit/eval/encode_spec.lua new file mode 100644 index 0000000000..f151a191fb --- /dev/null +++ b/test/unit/eval/encode_spec.lua @@ -0,0 +1,100 @@ +local helpers = require('test.unit.helpers') +local eval_helpers = require('test.unit.eval.helpers') + +local cimport = helpers.cimport +local to_cstr = helpers.to_cstr +local eq = helpers.eq + +local list = eval_helpers.list +local lst2tbl = eval_helpers.lst2tbl +local type_key = eval_helpers.type_key +local list_type = eval_helpers.list_type +local null_string = eval_helpers.null_string + +local encode = cimport('./src/nvim/eval/encode.h') + +describe('encode_list_write()', function() + local encode_list_write = function(l, s) + return encode.encode_list_write(l, to_cstr(s), #s) + end + + it('writes empty string', function() + local l = list() + eq(0, encode_list_write(l, '')) + eq({[type_key]=list_type}, lst2tbl(l)) + end) + + it('writes ASCII string literal with printable characters', function() + local l = list() + eq(0, encode_list_write(l, 'abc')) + eq({[type_key]=list_type, 'abc'}, lst2tbl(l)) + end) + + it('writes string starting with NL', function() + local l = list() + eq(0, encode_list_write(l, '\nabc')) + eq({[type_key]=list_type, null_string, 'abc'}, lst2tbl(l)) + end) + + it('writes string starting with NL twice', function() + local l = list() + eq(0, encode_list_write(l, '\nabc')) + eq({[type_key]=list_type, null_string, 'abc'}, lst2tbl(l)) + eq(0, encode_list_write(l, '\nabc')) + eq({[type_key]=list_type, null_string, 'abc', 'abc'}, lst2tbl(l)) + end) + + it('writes string ending with NL', function() + local l = list() + eq(0, encode_list_write(l, 'abc\n')) + eq({[type_key]=list_type, 'abc', null_string}, lst2tbl(l)) + end) + + it('writes string ending with NL twice', function() + local l = list() + eq(0, encode_list_write(l, 'abc\n')) + eq({[type_key]=list_type, 'abc', null_string}, lst2tbl(l)) + eq(0, encode_list_write(l, 'abc\n')) + eq({[type_key]=list_type, 'abc', 'abc', null_string}, lst2tbl(l)) + end) + + it('writes string starting, ending and containing NL twice', function() + local l = list() + eq(0, encode_list_write(l, '\na\nb\n')) + eq({[type_key]=list_type, null_string, 'a', 'b', null_string}, lst2tbl(l)) + eq(0, encode_list_write(l, '\na\nb\n')) + eq({[type_key]=list_type, null_string, 'a', 'b', null_string, 'a', 'b', null_string}, lst2tbl(l)) + end) + + it('writes string starting, ending and containing NUL with NL between twice', function() + local l = list() + eq(0, encode_list_write(l, '\0\n\0\n\0')) + eq({[type_key]=list_type, '\n', '\n', '\n'}, lst2tbl(l)) + eq(0, encode_list_write(l, '\0\n\0\n\0')) + eq({[type_key]=list_type, '\n', '\n', '\n\n', '\n', '\n'}, lst2tbl(l)) + end) + + it('writes string starting, ending and containing NL with NUL between twice', function() + local l = list() + eq(0, encode_list_write(l, '\n\0\n\0\n')) + eq({[type_key]=list_type, null_string, '\n', '\n', null_string}, lst2tbl(l)) + eq(0, encode_list_write(l, '\n\0\n\0\n')) + eq({[type_key]=list_type, null_string, '\n', '\n', null_string, '\n', '\n', null_string}, lst2tbl(l)) + end) + + it('writes string containing a single NL twice', function() + local l = list() + eq(0, encode_list_write(l, '\n')) + eq({[type_key]=list_type, null_string, null_string}, lst2tbl(l)) + eq(0, encode_list_write(l, '\n')) + eq({[type_key]=list_type, null_string, null_string, null_string}, lst2tbl(l)) + end) + + it('writes string containing a few NLs twice', function() + local l = list() + eq(0, encode_list_write(l, '\n\n\n')) + eq({[type_key]=list_type, null_string, null_string, null_string, null_string}, lst2tbl(l)) + eq(0, encode_list_write(l, '\n\n\n')) + eq({[type_key]=list_type, null_string, null_string, null_string, null_string, null_string, null_string, null_string}, lst2tbl(l)) + end) +end) diff --git a/test/unit/eval/helpers.lua b/test/unit/eval/helpers.lua new file mode 100644 index 0000000000..da2f5626ff --- /dev/null +++ b/test/unit/eval/helpers.lua @@ -0,0 +1,72 @@ +local helpers = require('test.unit.helpers') + +local cimport = helpers.cimport +local to_cstr = helpers.to_cstr +local ffi = helpers.ffi +local eq = helpers.eq + +local eval = cimport('./src/nvim/eval.h', './src/nvim/eval_defs.h') + +local null_string = {[true]='NULL string'} +local null_list = {[true]='NULL list'} +local type_key = {[true]='type key'} +local list_type = {[true]='list type'} + +local list = function(...) + local ret = ffi.gc(eval.list_alloc(), eval.list_unref) + eq(0, ret.lv_refcount) + ret.lv_refcount = 1 + for i = 1, select('#', ...) do + local val = select(i, ...) + local typ = type(val) + if typ == 'string' then + eval.list_append_string(ret, to_cstr(val)) + elseif typ == 'table' and val == null_string then + eval.list_append_string(ret, nil) + elseif typ == 'table' and val == null_list then + eval.list_append_list(ret, nil) + elseif typ == 'table' and val[type_key] == list_type then + local itemlist = ffi.gc(list(table.unpack(val)), nil) + eq(1, itemlist.lv_refcount) + itemlist.lv_refcount = 0 + eval.list_append_list(ret, itemlist) + else + assert(false, 'Not implemented yet') + end + end + return ret +end + +local lst2tbl = function(l) + local ret = {[type_key]=list_type} + if l == nil then + return ret + end + local li = l.lv_first + -- (listitem_T *) NULL is equal to nil, but yet it is not false. + while li ~= nil do + local typ = li.li_tv.v_type + if typ == eval.VAR_STRING then + str = li.li_tv.vval.v_string + if str == nil then + ret[#ret + 1] = null_string + else + ret[#ret + 1] = ffi.string(str) + end + else + assert(false, 'Not implemented yet') + end + li = li.li_next + end + return ret +end + +return { + null_string=null_string, + null_list=null_list, + list_type=list_type, + type_key=type_key, + + list=list, + lst2tbl=lst2tbl, +} diff --git a/test/unit/helpers.lua b/test/unit/helpers.lua index 7b43b2218c..9b9c1fef0f 100644 --- a/test/unit/helpers.lua +++ b/test/unit/helpers.lua @@ -28,8 +28,10 @@ local function filter_complex_blocks(body) local result = {} for line in body:gmatch("[^\r\n]+") do - if not (string.find(line, "(^)", 1, true) ~= nil or - string.find(line, "_ISwupper", 1, true)) then + if not (string.find(line, "(^)", 1, true) ~= nil + or string.find(line, "_ISwupper", 1, true) + or string.find(line, "msgpack_zone_push_finalizer") + or string.find(line, "msgpack_unpacker_reserve_buffer")) then result[#result + 1] = line end end @@ -103,6 +105,11 @@ local function cimport(...) -- request a sorted version of the new lines (same relative order as the -- original preprocessed file) and feed that to the LuaJIT ffi local new_lines = new_cdefs:to_table() + if os.getenv('NVIM_TEST_PRINT_CDEF') == '1' then + for lnum, line in ipairs(new_lines) do + print(lnum, line) + end + end ffi.cdef(table.concat(new_lines, "\n")) return libnvim From f0bd4a149408e75ebf887530964e0948518938dc Mon Sep 17 00:00:00 2001 From: ZyX Date: Thu, 11 Feb 2016 01:29:09 +0300 Subject: [PATCH 39/82] eval/encode: Fix invalid UTF-8 strings handling: 1. Do not allow reading past buffer end when creating error messages. 2. Fix surrogate pairs range, avoid magic constants. --- src/nvim/eval/encode.c | 20 +++++++++++------- src/nvim/message.c | 22 +++++++++++++++++--- test/functional/eval/json_functions_spec.lua | 7 +++++++ 3 files changed, 38 insertions(+), 11 deletions(-) diff --git a/src/nvim/eval/encode.c b/src/nvim/eval/encode.c index 6026189235..6fa22bfc5c 100644 --- a/src/nvim/eval/encode.c +++ b/src/nvim/eval/encode.c @@ -895,7 +895,8 @@ static inline int convert_to_json_string(garray_T *const gap, if (p_enc_conv.vc_type != CONV_NONE) { tofree = string_convert(&p_enc_conv, buf, &utf_len); if (tofree == NULL) { - EMSG2(_("E474: Failed to convert string \"%s\" to UTF-8"), utf_buf); + emsgf(_("E474: Failed to convert string \"%.*s\" to UTF-8"), + utf_len, utf_buf); return FAIL; } utf_buf = tofree; @@ -930,18 +931,21 @@ static inline int convert_to_json_string(garray_T *const gap, } default: { if (ch > 0x7F && shift == 1) { - EMSG2(_("E474: String \"%s\" contains byte that does not start any " - "UTF-8 character"), utf_buf); + emsgf(_("E474: String \"%.*s\" contains byte that does not start " + "any UTF-8 character"), + utf_len - (i - shift), utf_buf + i - shift); return FAIL; - } else if ((0xD800 <= ch && ch <= 0xDB7F) - || (0xDC00 <= ch && ch <= 0xDFFF)) { - EMSG2(_("E474: UTF-8 string contains code point which belongs " - "to surrogate pairs: %s"), utf_buf + i); + } else if ((SURROGATE_HI_START <= ch && ch <= SURROGATE_HI_END) + || (SURROGATE_LO_START <= ch && ch <= SURROGATE_LO_END)) { + emsgf(_("E474: UTF-8 string contains code point which belongs " + "to a surrogate pair: %.*s"), + utf_len - (i - shift), utf_buf + i - shift); return FAIL; } else if (ENCODE_RAW(p_enc_conv, ch)) { str_len += shift; } else { - str_len += ((sizeof("\\u1234") - 1) * (size_t) (1 + (ch > 0xFFFF))); + str_len += ((sizeof("\\u1234") - 1) + * (size_t) (1 + (ch >= SURROGATE_FIRST_CHAR))); } break; } diff --git a/src/nvim/message.c b/src/nvim/message.c index 1dd71baaa4..c4207fbe9e 100644 --- a/src/nvim/message.c +++ b/src/nvim/message.c @@ -609,6 +609,21 @@ int emsgu(char_u *s, uint64_t n) return emsg(IObuff); } +/// Print an error message with unknown number of arguments +bool emsgf(const char *const fmt, ...) +{ + if (emsg_not_now()) { + return true; + } + + va_list ap; + va_start(ap, fmt); + vim_vsnprintf((char *) IObuff, IOSIZE, fmt, ap, NULL); + va_end(ap); + + return emsg(IObuff); +} + /* * Like msg(), but truncate to a single line if p_shm contains 't', or when * "force" is TRUE. This truncates in another way as for normal messages. @@ -3097,11 +3112,12 @@ int vim_snprintf(char *str, size_t str_m, char *fmt, ...) return str_l; } -int vim_vsnprintf(char *str, size_t str_m, char *fmt, va_list ap, typval_T *tvs) +int vim_vsnprintf(char *str, size_t str_m, const char *fmt, va_list ap, + typval_T *tvs) { size_t str_l = 0; bool str_avail = str_l < str_m; - char *p = fmt; + const char *p = fmt; int arg_idx = 1; if (!p) { @@ -3135,7 +3151,7 @@ int vim_vsnprintf(char *str, size_t str_m, char *fmt, va_list ap, typval_T *tvs) char tmp[TMP_LEN]; // string address in case of string argument - char *str_arg; + const char *str_arg; // natural field width of arg without padding and sign size_t str_arg_l; diff --git a/test/functional/eval/json_functions_spec.lua b/test/functional/eval/json_functions_spec.lua index 13597eb7a0..398fab6c4b 100644 --- a/test/functional/eval/json_functions_spec.lua +++ b/test/functional/eval/json_functions_spec.lua @@ -663,4 +663,11 @@ describe('jsonencode() function', function() eq(1, eval('"\x01" =~# "\\\\p"')) eq('"\\u0001"', funcs.jsonencode('\x01')) end) + + it('fails when using surrogate character in a UTF-8 string', function() + eq('Vim(call):E474: UTF-8 string contains code point which belongs to a surrogate pair: \xED\xA0\x80', + exc_exec('call jsonencode("\xED\xA0\x80")')) + eq('Vim(call):E474: UTF-8 string contains code point which belongs to a surrogate pair: \xED\xAF\xBF', + exc_exec('call jsonencode("\xED\xAF\xBF")')) + end) end) From 209427e97224ea7fdd49eb53fa41e0b26c55369f Mon Sep 17 00:00:00 2001 From: ZyX Date: Thu, 11 Feb 2016 01:34:08 +0300 Subject: [PATCH 40/82] eval/encode: Reduce length of encode_list_write Changes suggested by oni-link. --- src/nvim/eval/encode.c | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/src/nvim/eval/encode.c b/src/nvim/eval/encode.c index 6fa22bfc5c..4d6a0afe9c 100644 --- a/src/nvim/eval/encode.c +++ b/src/nvim/eval/encode.c @@ -72,9 +72,6 @@ int encode_list_write(void *data, const char *buf, size_t len) list_T *const list = (list_T *) data; const char *const end = buf + len; const char *line_end = buf; - if (list->lv_last == NULL) { - list_append_string(list, NULL, 0); - } listitem_T *li = list->lv_last; do { const char *line_start = line_end; @@ -94,11 +91,7 @@ int encode_list_write(void *data, const char *buf, size_t len) memcpy(str, line_start, line_length); str[line_length] = 0; } - for (size_t i = 0; i < line_length; i++) { - if (str[i] == NUL) { - str[i] = NL; - } - } + memchrsub(str, NUL, NL, line_length); } if (li == NULL) { list_append_allocated_string(list, str); From 4913a25dec5edc8888579a0f09a1b2f5f783c911 Mon Sep 17 00:00:00 2001 From: ZyX Date: Thu, 11 Feb 2016 01:38:58 +0300 Subject: [PATCH 41/82] eval/encode: Free memory just in case After string_convert() with .vc_fail=true these blocks should never be entered because they indicate invalid unicode. --- src/nvim/eval/encode.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/nvim/eval/encode.c b/src/nvim/eval/encode.c index 4d6a0afe9c..0096d9172b 100644 --- a/src/nvim/eval/encode.c +++ b/src/nvim/eval/encode.c @@ -927,12 +927,14 @@ static inline int convert_to_json_string(garray_T *const gap, emsgf(_("E474: String \"%.*s\" contains byte that does not start " "any UTF-8 character"), utf_len - (i - shift), utf_buf + i - shift); + xfree(tofree); return FAIL; } else if ((SURROGATE_HI_START <= ch && ch <= SURROGATE_HI_END) || (SURROGATE_LO_START <= ch && ch <= SURROGATE_LO_END)) { emsgf(_("E474: UTF-8 string contains code point which belongs " "to a surrogate pair: %.*s"), utf_len - (i - shift), utf_buf + i - shift); + xfree(tofree); return FAIL; } else if (ENCODE_RAW(p_enc_conv, ch)) { str_len += shift; From af6603a6b4c9b1cb4a65eb2dc581295d8990c5ef Mon Sep 17 00:00:00 2001 From: ZyX Date: Thu, 11 Feb 2016 01:40:40 +0300 Subject: [PATCH 42/82] eval/encode: Remove unneeded variable, add missing include --- src/nvim/eval/encode.c | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/nvim/eval/encode.c b/src/nvim/eval/encode.c index 0096d9172b..e3d0bf69b2 100644 --- a/src/nvim/eval/encode.c +++ b/src/nvim/eval/encode.c @@ -15,6 +15,7 @@ #include "nvim/garray.h" #include "nvim/mbyte.h" #include "nvim/message.h" +#include "nvim/memory.h" #include "nvim/charset.h" // vim_isprintc() #include "nvim/macros.h" #include "nvim/ascii.h" @@ -636,17 +637,10 @@ encode_vim_to_##name##_error_ret: \ ga_concat(gap, "''"); \ } else { \ const size_t len_ = (len); \ - size_t num_quotes = 0; \ - for (size_t i = 0; i < len_; i++) { \ - if (buf_[i] == '\'') { \ - num_quotes++; \ - } \ - } \ - ga_grow(gap, (int) (2 + len_ + num_quotes)); \ + ga_grow(gap, (int) (2 + len_ + memcnt(buf_, '\'', len_))); \ ga_append(gap, '\''); \ for (size_t i = 0; i < len_; i++) { \ if (buf_[i] == '\'') { \ - num_quotes++; \ ga_append(gap, '\''); \ } \ ga_append(gap, buf_[i]); \ From f1ced96c28b7db7b3dad9b0ca2f71f8d046ef732 Mon Sep 17 00:00:00 2001 From: ZyX Date: Thu, 11 Feb 2016 02:01:17 +0300 Subject: [PATCH 43/82] api: Replace set_var(name, NIL) with del_var(name) --- src/nvim/api/buffer.c | 21 +++++++++++++++++++-- src/nvim/api/private/helpers.c | 9 ++++++--- src/nvim/api/tabpage.c | 21 +++++++++++++++++++-- src/nvim/api/vim.c | 14 ++++++++++++-- src/nvim/api/window.c | 21 +++++++++++++++++++-- src/nvim/eval.c | 4 ++-- src/nvim/terminal.c | 1 + test/functional/api/buffer_spec.lua | 6 +++++- test/functional/api/tabpage_spec.lua | 8 +++++++- test/functional/api/vim_spec.lua | 15 ++++++++++++++- test/functional/api/window_spec.lua | 7 ++++++- 11 files changed, 110 insertions(+), 17 deletions(-) diff --git a/src/nvim/api/buffer.c b/src/nvim/api/buffer.c index c25a9789c5..075f101f61 100644 --- a/src/nvim/api/buffer.c +++ b/src/nvim/api/buffer.c @@ -423,7 +423,7 @@ Object buffer_get_var(Buffer buffer, String name, Error *err) return dict_get_value(buf->b_vars, name, err); } -/// Sets a buffer-scoped (b:) variable. 'nil' value deletes the variable. +/// Sets a buffer-scoped (b:) variable /// /// @param buffer The buffer handle /// @param name The variable name @@ -438,7 +438,24 @@ Object buffer_set_var(Buffer buffer, String name, Object value, Error *err) return (Object) OBJECT_INIT; } - return dict_set_value(buf->b_vars, name, value, err); + return dict_set_value(buf->b_vars, name, value, false, err); +} + +/// Removes a buffer-scoped (b:) variable +/// +/// @param buffer The buffer handle +/// @param name The variable name +/// @param[out] err Details of an error that may have occurred +/// @return The old value +Object buffer_del_var(Buffer buffer, String name, Error *err) +{ + buf_T *buf = find_buffer_by_handle(buffer, err); + + if (!buf) { + return (Object) OBJECT_INIT; + } + + return dict_set_value(buf->b_vars, name, NIL, true, err); } /// Gets a buffer option value diff --git a/src/nvim/api/private/helpers.c b/src/nvim/api/private/helpers.c index 9082dfd759..db3e499427 100644 --- a/src/nvim/api/private/helpers.c +++ b/src/nvim/api/private/helpers.c @@ -90,14 +90,17 @@ Object dict_get_value(dict_T *dict, String key, Error *err) } /// Set a value in a dict. Objects are recursively expanded into their -/// vimscript equivalents. Passing 'nil' as value deletes the key. +/// vimscript equivalents. /// /// @param dict The vimscript dict /// @param key The key /// @param value The new value +/// @param del Delete key in place of setting it. Argument `value` is ignored in +/// this case. /// @param[out] err Details of an error that may have occurred /// @return the old value, if any -Object dict_set_value(dict_T *dict, String key, Object value, Error *err) +Object dict_set_value(dict_T *dict, String key, Object value, bool del, + Error *err) { Object rv = OBJECT_INIT; @@ -118,7 +121,7 @@ Object dict_set_value(dict_T *dict, String key, Object value, Error *err) dictitem_T *di = dict_find(dict, (uint8_t *)key.data, (int)key.size); - if (value.type == kObjectTypeNil) { + if (del) { // Delete the key if (di == NULL) { // Doesn't exist, fail diff --git a/src/nvim/api/tabpage.c b/src/nvim/api/tabpage.c index 126ee4072d..475b75b571 100644 --- a/src/nvim/api/tabpage.c +++ b/src/nvim/api/tabpage.c @@ -54,7 +54,7 @@ Object tabpage_get_var(Tabpage tabpage, String name, Error *err) return dict_get_value(tab->tp_vars, name, err); } -/// Sets a tab-scoped (t:) variable. 'nil' value deletes the variable. +/// Sets a tab-scoped (t:) variable /// /// @param tabpage handle /// @param name The variable name @@ -69,7 +69,24 @@ Object tabpage_set_var(Tabpage tabpage, String name, Object value, Error *err) return (Object) OBJECT_INIT; } - return dict_set_value(tab->tp_vars, name, value, err); + return dict_set_value(tab->tp_vars, name, value, false, err); +} + +/// Removes a tab-scoped (t:) variable +/// +/// @param tabpage handle +/// @param name The variable name +/// @param[out] err Details of an error that may have occurred +/// @return The tab page handle +Object tabpage_del_var(Tabpage tabpage, String name, Error *err) +{ + tabpage_T *tab = find_tab_by_handle(tabpage, err); + + if (!tab) { + return (Object) OBJECT_INIT; + } + + return dict_set_value(tab->tp_vars, name, NIL, true, err); } /// Gets the current window in a tab page diff --git a/src/nvim/api/vim.c b/src/nvim/api/vim.c index 9279f6b469..09d5f1b6f8 100644 --- a/src/nvim/api/vim.c +++ b/src/nvim/api/vim.c @@ -331,7 +331,7 @@ Object vim_get_var(String name, Error *err) return dict_get_value(&globvardict, name, err); } -/// Sets a global variable. Passing 'nil' as value deletes the variable. +/// Sets a global variable /// /// @param name The variable name /// @param value The variable value @@ -339,7 +339,17 @@ Object vim_get_var(String name, Error *err) /// @return the old value if any Object vim_set_var(String name, Object value, Error *err) { - return dict_set_value(&globvardict, name, value, err); + return dict_set_value(&globvardict, name, value, false, err); +} + +/// Removes a global variable +/// +/// @param name The variable name +/// @param[out] err Details of an error that may have occurred +/// @return the old value if any +Object vim_del_var(String name, Error *err) +{ + return dict_set_value(&globvardict, name, NIL, true, err); } /// Gets a vim variable diff --git a/src/nvim/api/window.c b/src/nvim/api/window.c index aad616c7bf..58218af09d 100644 --- a/src/nvim/api/window.c +++ b/src/nvim/api/window.c @@ -197,7 +197,7 @@ Object window_get_var(Window window, String name, Error *err) return dict_get_value(win->w_vars, name, err); } -/// Sets a window-scoped (w:) variable. 'nil' value deletes the variable. +/// Sets a window-scoped (w:) variable /// /// @param window The window handle /// @param name The variable name @@ -212,7 +212,24 @@ Object window_set_var(Window window, String name, Object value, Error *err) return (Object) OBJECT_INIT; } - return dict_set_value(win->w_vars, name, value, err); + return dict_set_value(win->w_vars, name, value, false, err); +} + +/// Removes a window-scoped (w:) variable +/// +/// @param window The window handle +/// @param name The variable name +/// @param[out] err Details of an error that may have occurred +/// @return The old value +Object window_del_var(Window window, String name, Error *err) +{ + win_T *win = find_window_by_handle(window, err); + + if (!win) { + return (Object) OBJECT_INIT; + } + + return dict_set_value(win->w_vars, name, NIL, true, err); } /// Gets a window option value diff --git a/src/nvim/eval.c b/src/nvim/eval.c index d2a7aeb74e..755b05e591 100644 --- a/src/nvim/eval.c +++ b/src/nvim/eval.c @@ -16028,9 +16028,9 @@ static void f_termopen(typval_T *argvars, typval_T *rettv) // Save the job id and pid in b:terminal_job_{id,pid} Error err; dict_set_value(curbuf->b_vars, cstr_as_string("terminal_job_id"), - INTEGER_OBJ(rettv->vval.v_number), &err); + INTEGER_OBJ(rettv->vval.v_number), false, &err); dict_set_value(curbuf->b_vars, cstr_as_string("terminal_job_pid"), - INTEGER_OBJ(pid), &err); + INTEGER_OBJ(pid), false, &err); Terminal *term = terminal_open(topts); data->term = term; diff --git a/src/nvim/terminal.c b/src/nvim/terminal.c index 63a7e20880..0440272eb9 100644 --- a/src/nvim/terminal.c +++ b/src/nvim/terminal.c @@ -627,6 +627,7 @@ static int term_settermprop(VTermProp prop, VTermValue *val, void *data) api_free_object(dict_set_value(buf->b_vars, cstr_as_string("term_title"), STRING_OBJ(cstr_as_string(val->string)), + false, &err)); break; } diff --git a/test/functional/api/buffer_spec.lua b/test/functional/api/buffer_spec.lua index a15d489a1f..0eefa25a13 100644 --- a/test/functional/api/buffer_spec.lua +++ b/test/functional/api/buffer_spec.lua @@ -3,6 +3,7 @@ local helpers = require('test.functional.helpers') local clear, nvim, buffer = helpers.clear, helpers.nvim, helpers.buffer local curbuf, curwin, eq = helpers.curbuf, helpers.curwin, helpers.eq local curbufmeths, ok = helpers.curbufmeths, helpers.ok +local funcs = helpers.funcs describe('buffer_* functions', function() before_each(clear) @@ -234,11 +235,14 @@ describe('buffer_* functions', function() end) - describe('{get,set}_var', function() + describe('{get,set,del}_var', function() it('works', function() curbuf('set_var', 'lua', {1, 2, {['3'] = 1}}) eq({1, 2, {['3'] = 1}}, curbuf('get_var', 'lua')) eq({1, 2, {['3'] = 1}}, nvim('eval', 'b:lua')) + eq(1, funcs.exists('b:lua')) + curbufmeths.del_var('lua') + eq(0, funcs.exists('b:lua')) end) end) diff --git a/test/functional/api/tabpage_spec.lua b/test/functional/api/tabpage_spec.lua index 9937e0c72e..0f59bbea7b 100644 --- a/test/functional/api/tabpage_spec.lua +++ b/test/functional/api/tabpage_spec.lua @@ -3,6 +3,9 @@ local helpers = require('test.functional.helpers') local clear, nvim, tabpage, curtab, eq, ok = helpers.clear, helpers.nvim, helpers.tabpage, helpers.curtab, helpers.eq, helpers.ok +local wait = helpers.wait +local curtabmeths = helpers.curtabmeths +local funcs = helpers.funcs describe('tabpage_* functions', function() before_each(clear) @@ -21,11 +24,14 @@ describe('tabpage_* functions', function() end) end) - describe('{get,set}_var', function() + describe('{get,set,del}_var', function() it('works', function() curtab('set_var', 'lua', {1, 2, {['3'] = 1}}) eq({1, 2, {['3'] = 1}}, curtab('get_var', 'lua')) eq({1, 2, {['3'] = 1}}, nvim('eval', 't:lua')) + eq(1, funcs.exists('t:lua')) + curtabmeths.del_var('lua') + eq(0, funcs.exists('t:lua')) end) end) diff --git a/test/functional/api/vim_spec.lua b/test/functional/api/vim_spec.lua index 699a0bd588..d7faaa9d2d 100644 --- a/test/functional/api/vim_spec.lua +++ b/test/functional/api/vim_spec.lua @@ -4,6 +4,8 @@ local Screen = require('test.functional.ui.screen') local clear, nvim, eq, neq = helpers.clear, helpers.nvim, helpers.eq, helpers.neq local ok, nvim_async, feed = helpers.ok, helpers.nvim_async, helpers.feed local os_name = helpers.os_name +local meths = helpers.meths +local funcs = helpers.funcs describe('vim_* functions', function() before_each(clear) @@ -70,11 +72,14 @@ describe('vim_* functions', function() end) end) - describe('{get,set}_var', function() + describe('{get,set,del}_var', function() it('works', function() nvim('set_var', 'lua', {1, 2, {['3'] = 1}}) eq({1, 2, {['3'] = 1}}, nvim('get_var', 'lua')) eq({1, 2, {['3'] = 1}}, nvim('eval', 'g:lua')) + eq(1, funcs.exists('g:lua')) + meths.del_var('lua') + eq(0, funcs.exists('g:lua')) end) it('set_var returns the old value', function() @@ -84,6 +89,14 @@ describe('vim_* functions', function() eq(val1, nvim('set_var', 'lua', val2)) end) + it('del_var returns the old value', function() + local val1 = {1, 2, {['3'] = 1}} + local val2 = {4, 7} + eq(nil, meths.set_var('lua', val1)) + eq(val1, meths.set_var('lua', val2)) + eq(val2, meths.del_var('lua')) + end) + it('truncates values with NULs in them', function() nvim('set_var', 'xxx', 'ab\0cd') eq('ab', nvim('get_var', 'xxx')) diff --git a/test/functional/api/window_spec.lua b/test/functional/api/window_spec.lua index 17aacafe9b..92a33b4cdb 100644 --- a/test/functional/api/window_spec.lua +++ b/test/functional/api/window_spec.lua @@ -5,6 +5,8 @@ local clear, nvim, curbuf, curbuf_contents, window, curwin, eq, neq, helpers.curbuf_contents, helpers.window, helpers.curwin, helpers.eq, helpers.neq, helpers.ok, helpers.feed, helpers.insert, helpers.eval local wait = helpers.wait +local curwinmeths = helpers.curwinmeths +local funcs = helpers.funcs -- check if str is visible at the beginning of some line local function is_visible(str) @@ -126,11 +128,14 @@ describe('window_* functions', function() end) end) - describe('{get,set}_var', function() + describe('{get,set,del}_var', function() it('works', function() curwin('set_var', 'lua', {1, 2, {['3'] = 1}}) eq({1, 2, {['3'] = 1}}, curwin('get_var', 'lua')) eq({1, 2, {['3'] = 1}}, nvim('eval', 'w:lua')) + eq(1, funcs.exists('w:lua')) + curwinmeths.del_var('lua') + eq(0, funcs.exists('w:lua')) end) end) From 2f67786796d5fb4237f4b0258ec3db0982cc7f53 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sat, 13 Feb 2016 21:39:28 +0300 Subject: [PATCH 44/82] eval: Rename json* functions to json_* --- runtime/doc/eval.txt | 16 +- runtime/doc/vim_diff.txt | 18 +- src/nvim/eval.c | 12 +- src/nvim/eval/encode.c | 2 +- src/nvim/version.c | 11 +- test/functional/eval/json_functions_spec.lua | 368 +++++++++---------- 6 files changed, 216 insertions(+), 211 deletions(-) diff --git a/runtime/doc/eval.txt b/runtime/doc/eval.txt index f085dd2972..fea9e669e0 100644 --- a/runtime/doc/eval.txt +++ b/runtime/doc/eval.txt @@ -1420,7 +1420,7 @@ v:exception The value of the exception most recently caught and not *v:false* *false-variable* v:false Special value used to put "false" in JSON and msgpack. See - |jsonencode()|. This value is converted to "false" when used + |json_encode()|. This value is converted to "false" when used as a String (e.g. in |expr5| with string concatenation operator) and to zero when used as a Number (e.g. in |expr5| or |expr7| when used with numeric operators). @@ -1566,7 +1566,7 @@ v:msgpack_types Dictionary containing msgpack types used by |msgpackparse()| *v:null* *null-variable* v:null Special value used to put "null" in JSON and NIL in msgpack. - See |jsonencode()|. This value is converted to "null" when + See |json_encode()|. This value is converted to "null" when used as a String (e.g. in |expr5| with string concatenation operator) and to zero when used as a Number (e.g. in |expr5| or |expr7| when used with numeric operators). @@ -1738,7 +1738,7 @@ v:throwpoint The point where the exception most recently caught and not *v:true* *true-variable* v:true Special value used to put "true" in JSON and msgpack. See - |jsonencode()|. This value is converted to "true" when used + |json_encode()|. This value is converted to "true" when used as a String (e.g. in |expr5| with string concatenation operator) and to one when used as a Number (e.g. in |expr5| or |expr7| when used with numeric operators). @@ -1953,8 +1953,8 @@ jobstart( {cmd}[, {opts}]) Number Spawns {cmd} as a job jobstop( {job}) Number Stops a job jobwait( {ids}[, {timeout}]) Number Wait for a set of jobs join( {list} [, {sep}]) String join {list} items into one String -jsondecode( {expr}) any Convert {expr} from JSON -jsonencode( {expr}) String Convert {expr} to JSON +json_decode( {expr}) any Convert {expr} from JSON +json_encode( {expr}) String Convert {expr} to JSON keys( {dict}) List keys in {dict} len( {expr}) Number the length of {expr} libcall( {lib}, {func}, {arg}) String call {func} in library {lib} with {arg} @@ -4314,12 +4314,12 @@ join({list} [, {sep}]) *join()* converted into a string like with |string()|. The opposite function is |split()|. -jsondecode({expr}) *jsondecode()* +json_decode({expr}) *json_decode()* Convert {expr} from JSON object. Accepts |readfile()|-style list as the input, as well as regular string. May output any Vim value. When 'encoding' is not UTF-8 string is converted from UTF-8 to 'encoding', failing conversion fails - jsondecode(). In the following cases it will output + json_decode(). In the following cases it will output |msgpack-special-dict|: 1. Dictionary contains duplicate key. 2. Dictionary contains empty key. @@ -4327,7 +4327,7 @@ jsondecode({expr}) *jsondecode()* dictionary and for string will be emitted in case string with NUL byte was a dictionary key. -jsonencode({expr}) *jsonencode()* +json_encode({expr}) *json_encode()* Convert {expr} into a JSON string. Accepts |msgpack-special-dict| as the input. Converts from 'encoding' to UTF-8 when encoding strings. Will not convert |Funcref|s, diff --git a/runtime/doc/vim_diff.txt b/runtime/doc/vim_diff.txt index b42b91140c..508712ca75 100644 --- a/runtime/doc/vim_diff.txt +++ b/runtime/doc/vim_diff.txt @@ -102,15 +102,19 @@ are always available and may be used simultaneously in separate plugins. The 5. (internal) Trying to print or stringify VAR_UNKNOWN in Vim results in nothing, |E908|, in Neovim it is internal error. -|jsondecode()| behaviour changed: +|json_decode()| behaviour changed: 1. It may output |msgpack-special-dict|. -2. It accepts only valid JSON. |v:none| is never emitted. -|jsonencode()| behaviour slightly changed: now |msgpack-special-dict| values -are accepted. +2. |msgpack-special-dict| is emitted also in case of duplicate keys, while in + Vim it errors out. +3. It accepts only valid JSON. Trailing commas are not accepted. -*v:none* variable is absent. In Vim it represents “no value” in non-JSON -strings like "{"a": }" parsed as "{'a': v:none}". See |jsondecode()| and -|jsonencode()| incompatibilities above. +|json_encode()| behaviour slightly changed: now |msgpack-special-dict| values +are accepted, but |v:none| is not. + +*v:none* variable is absent. In Vim it represents “no value” in “js” strings +like "[,]" parsed as "[v:none]" by |js_decode()|. + +*js_encode()* and *js_decode()* functions are also absent. Viminfo text files were replaced with binary (messagepack) ShaDa files. Additional differences: diff --git a/src/nvim/eval.c b/src/nvim/eval.c index 755b05e591..c23613aeab 100644 --- a/src/nvim/eval.c +++ b/src/nvim/eval.c @@ -6784,8 +6784,8 @@ static struct fst { { "jobstop", 1, 1, f_jobstop }, { "jobwait", 1, 2, f_jobwait }, { "join", 1, 2, f_join }, - { "jsondecode", 1, 1, f_jsondecode }, - { "jsonencode", 1, 1, f_jsonencode }, + { "json_decode", 1, 1, f_json_decode }, + { "json_encode", 1, 1, f_json_encode }, { "keys", 1, 1, f_keys }, { "last_buffer_nr", 0, 0, f_last_buffer_nr }, // obsolete { "len", 1, 1, f_len }, @@ -11583,8 +11583,8 @@ static void f_join(typval_T *argvars, typval_T *rettv) rettv->vval.v_string = NULL; } -/// jsondecode() function -static void f_jsondecode(typval_T *argvars, typval_T *rettv) +/// json_decode() function +static void f_json_decode(typval_T *argvars, typval_T *rettv) { char numbuf[NUMBUFLEN]; char *s = NULL; @@ -11614,8 +11614,8 @@ static void f_jsondecode(typval_T *argvars, typval_T *rettv) xfree(tofree); } -/// jsonencode() function -static void f_jsonencode(typval_T *argvars, typval_T *rettv) +/// json_encode() function +static void f_json_encode(typval_T *argvars, typval_T *rettv) { rettv->v_type = VAR_STRING; rettv->vval.v_string = (char_u *) encode_tv2json(&argvars[0], NULL); diff --git a/src/nvim/eval/encode.c b/src/nvim/eval/encode.c index e3d0bf69b2..a131f5c3c1 100644 --- a/src/nvim/eval/encode.c +++ b/src/nvim/eval/encode.c @@ -1020,7 +1020,7 @@ static inline int convert_to_json_string(garray_T *const gap, "attempt to dump function reference"), \ mpstack, objname) -/// Check whether given key can be used in jsonencode() +/// Check whether given key can be used in json_encode() /// /// @param[in] tv Key to check. static inline bool check_json_key(const typval_T *const tv) diff --git a/src/nvim/version.c b/src/nvim/version.c index 106cc8d1d8..e0d5e984cd 100644 --- a/src/nvim/version.c +++ b/src/nvim/version.c @@ -78,14 +78,15 @@ static int included_patches[] = { 1511, 1425, 1366, + 1304, 1292, 1284, // 1283 1282, // 1281 // 1280 - // 1279 - // 1278 + // 1279 NA + // 1278 NA // 1277 // 1276 // 1275 @@ -94,7 +95,7 @@ static int included_patches[] = { // 1272 1271, // 1270 - // 1269 + 1269, // 1268 1267, // 1266 @@ -120,7 +121,7 @@ static int included_patches[] = { // 1246 // 1245 // 1244 - // 1243 + // 1243 NA // 1242 // 1241 // 1240 @@ -132,7 +133,7 @@ static int included_patches[] = { // 1234 // 1233 // 1232 - // 1231 + // 1231 NA // 1230 // 1229 1228, diff --git a/test/functional/eval/json_functions_spec.lua b/test/functional/eval/json_functions_spec.lua index 398fab6c4b..0b0403ce8e 100644 --- a/test/functional/eval/json_functions_spec.lua +++ b/test/functional/eval/json_functions_spec.lua @@ -7,7 +7,7 @@ local eval = helpers.eval local execute = helpers.execute local exc_exec = helpers.exc_exec -describe('jsondecode() function', function() +describe('json_decode() function', function() local restart = function(cmd) clear(cmd) execute([[ @@ -61,7 +61,7 @@ describe('jsondecode() function', function() end it('accepts readfile()-style list', function() - eq({Test=1}, funcs.jsondecode({ + eq({Test=1}, funcs.json_decode({ '{', '\t"Test": 1', '}', @@ -69,7 +69,7 @@ describe('jsondecode() function', function() end) it('accepts strings with newlines', function() - eq({Test=1}, funcs.jsondecode([[ + eq({Test=1}, funcs.json_decode([[ { "Test": 1 } @@ -77,242 +77,242 @@ describe('jsondecode() function', function() end) it('parses null, true, false', function() - eq(nil, funcs.jsondecode('null')) - eq(true, funcs.jsondecode('true')) - eq(false, funcs.jsondecode('false')) + eq(nil, funcs.json_decode('null')) + eq(true, funcs.json_decode('true')) + eq(false, funcs.json_decode('false')) end) it('fails to parse incomplete null, true, false', function() eq('Vim(call):E474: Expected null: n', - exc_exec('call jsondecode("n")')) + exc_exec('call json_decode("n")')) eq('Vim(call):E474: Expected null: nu', - exc_exec('call jsondecode("nu")')) + exc_exec('call json_decode("nu")')) eq('Vim(call):E474: Expected null: nul', - exc_exec('call jsondecode("nul")')) + exc_exec('call json_decode("nul")')) eq('Vim(call):E474: Expected null: nul\n\t', - exc_exec('call jsondecode("nul\\n\\t")')) + exc_exec('call json_decode("nul\\n\\t")')) eq('Vim(call):E474: Expected true: t', - exc_exec('call jsondecode("t")')) + exc_exec('call json_decode("t")')) eq('Vim(call):E474: Expected true: tr', - exc_exec('call jsondecode("tr")')) + exc_exec('call json_decode("tr")')) eq('Vim(call):E474: Expected true: tru', - exc_exec('call jsondecode("tru")')) + exc_exec('call json_decode("tru")')) eq('Vim(call):E474: Expected true: tru\t\n', - exc_exec('call jsondecode("tru\\t\\n")')) + exc_exec('call json_decode("tru\\t\\n")')) eq('Vim(call):E474: Expected false: f', - exc_exec('call jsondecode("f")')) + exc_exec('call json_decode("f")')) eq('Vim(call):E474: Expected false: fa', - exc_exec('call jsondecode("fa")')) + exc_exec('call json_decode("fa")')) eq('Vim(call):E474: Expected false: fal', - exc_exec('call jsondecode("fal")')) + exc_exec('call json_decode("fal")')) eq('Vim(call):E474: Expected false: fal <', - exc_exec('call jsondecode(" fal <")')) + exc_exec('call json_decode(" fal <")')) eq('Vim(call):E474: Expected false: fals', - exc_exec('call jsondecode("fals")')) + exc_exec('call json_decode("fals")')) end) it('parses integer numbers', function() - eq(100000, funcs.jsondecode('100000')) - eq(-100000, funcs.jsondecode('-100000')) - eq(100000, funcs.jsondecode(' 100000 ')) - eq(-100000, funcs.jsondecode(' -100000 ')) + eq(100000, funcs.json_decode('100000')) + eq(-100000, funcs.json_decode('-100000')) + eq(100000, funcs.json_decode(' 100000 ')) + eq(-100000, funcs.json_decode(' -100000 ')) end) it('fails to parse +numbers', function() eq('Vim(call):E474: Unidentified byte: +1000', - exc_exec('call jsondecode("+1000")')) + exc_exec('call json_decode("+1000")')) end) it('fails to parse negative numbers with space after -', function() eq('Vim(call):E474: Missing number after minus sign: - 1000', - exc_exec('call jsondecode("- 1000")')) + exc_exec('call json_decode("- 1000")')) end) it('fails to parse -', function() eq('Vim(call):E474: Missing number after minus sign: -', - exc_exec('call jsondecode("-")')) + exc_exec('call json_decode("-")')) end) it('parses floating-point numbers', function() - eq('100000.0', eval('string(jsondecode("100000.0"))')) - eq(100000.5, funcs.jsondecode('100000.5')) - eq(-100000.5, funcs.jsondecode('-100000.5')) - eq(-100000.5e50, funcs.jsondecode('-100000.5e50')) - eq(100000.5e50, funcs.jsondecode('100000.5e50')) - eq(100000.5e50, funcs.jsondecode('100000.5e+50')) - eq(-100000.5e-50, funcs.jsondecode('-100000.5e-50')) - eq(100000.5e-50, funcs.jsondecode('100000.5e-50')) + eq('100000.0', eval('string(json_decode("100000.0"))')) + eq(100000.5, funcs.json_decode('100000.5')) + eq(-100000.5, funcs.json_decode('-100000.5')) + eq(-100000.5e50, funcs.json_decode('-100000.5e50')) + eq(100000.5e50, funcs.json_decode('100000.5e50')) + eq(100000.5e50, funcs.json_decode('100000.5e+50')) + eq(-100000.5e-50, funcs.json_decode('-100000.5e-50')) + eq(100000.5e-50, funcs.json_decode('100000.5e-50')) end) it('fails to parse incomplete floating-point numbers', function() eq('Vim(call):E474: Missing number after decimal dot: 0.', - exc_exec('call jsondecode("0.")')) + exc_exec('call json_decode("0.")')) eq('Vim(call):E474: Missing exponent: 0.0e', - exc_exec('call jsondecode("0.0e")')) + exc_exec('call json_decode("0.0e")')) eq('Vim(call):E474: Missing exponent: 0.0e+', - exc_exec('call jsondecode("0.0e+")')) + exc_exec('call json_decode("0.0e+")')) eq('Vim(call):E474: Missing exponent: 0.0e-', - exc_exec('call jsondecode("0.0e-")')) + exc_exec('call json_decode("0.0e-")')) end) it('fails to parse floating-point numbers with spaces inside', function() eq('Vim(call):E474: Missing number after decimal dot: 0. ', - exc_exec('call jsondecode("0. ")')) + exc_exec('call json_decode("0. ")')) eq('Vim(call):E474: Missing number after decimal dot: 0. 0', - exc_exec('call jsondecode("0. 0")')) + exc_exec('call json_decode("0. 0")')) eq('Vim(call):E474: Missing exponent: 0.0e 1', - exc_exec('call jsondecode("0.0e 1")')) + exc_exec('call json_decode("0.0e 1")')) eq('Vim(call):E474: Missing exponent: 0.0e+ 1', - exc_exec('call jsondecode("0.0e+ 1")')) + exc_exec('call json_decode("0.0e+ 1")')) eq('Vim(call):E474: Missing exponent: 0.0e- 1', - exc_exec('call jsondecode("0.0e- 1")')) + exc_exec('call json_decode("0.0e- 1")')) end) it('fails to parse "," and ":"', function() eq('Vim(call):E474: Comma not inside container: , ', - exc_exec('call jsondecode(" , ")')) + exc_exec('call json_decode(" , ")')) eq('Vim(call):E474: Colon not inside container: : ', - exc_exec('call jsondecode(" : ")')) + exc_exec('call json_decode(" : ")')) end) it('parses empty containers', function() - eq({}, funcs.jsondecode('[]')) - eq('[]', eval('string(jsondecode("[]"))')) + eq({}, funcs.json_decode('[]')) + eq('[]', eval('string(json_decode("[]"))')) end) it('fails to parse "[" and "{"', function() eq('Vim(call):E474: Unexpected end of input: {', - exc_exec('call jsondecode("{")')) + exc_exec('call json_decode("{")')) eq('Vim(call):E474: Unexpected end of input: [', - exc_exec('call jsondecode("[")')) + exc_exec('call json_decode("[")')) end) it('fails to parse "}" and "]"', function() eq('Vim(call):E474: No container to close: ]', - exc_exec('call jsondecode("]")')) + exc_exec('call json_decode("]")')) eq('Vim(call):E474: No container to close: }', - exc_exec('call jsondecode("}")')) + exc_exec('call json_decode("}")')) end) it('fails to parse containers which are closed by different brackets', function() eq('Vim(call):E474: Closing dictionary with bracket: ]', - exc_exec('call jsondecode("{]")')) + exc_exec('call json_decode("{]")')) eq('Vim(call):E474: Closing list with figure brace: }', - exc_exec('call jsondecode("[}")')) + exc_exec('call json_decode("[}")')) end) it('fails to parse concat inside container', function() eq('Vim(call):E474: Expected comma before list item: []]', - exc_exec('call jsondecode("[[][]]")')) + exc_exec('call json_decode("[[][]]")')) eq('Vim(call):E474: Expected comma before list item: {}]', - exc_exec('call jsondecode("[{}{}]")')) + exc_exec('call json_decode("[{}{}]")')) eq('Vim(call):E474: Expected comma before list item: ]', - exc_exec('call jsondecode("[1 2]")')) + exc_exec('call json_decode("[1 2]")')) eq('Vim(call):E474: Expected comma before dictionary key: ": 4}', - exc_exec('call jsondecode("{\\"1\\": 2 \\"3\\": 4}")')) + exc_exec('call json_decode("{\\"1\\": 2 \\"3\\": 4}")')) eq('Vim(call):E474: Expected colon before dictionary value: , "3" 4}', - exc_exec('call jsondecode("{\\"1\\" 2, \\"3\\" 4}")')) + exc_exec('call json_decode("{\\"1\\" 2, \\"3\\" 4}")')) end) it('fails to parse containers with leading comma or colon', function() eq('Vim(call):E474: Leading comma: ,}', - exc_exec('call jsondecode("{,}")')) + exc_exec('call json_decode("{,}")')) eq('Vim(call):E474: Leading comma: ,]', - exc_exec('call jsondecode("[,]")')) + exc_exec('call json_decode("[,]")')) eq('Vim(call):E474: Using colon not in dictionary: :]', - exc_exec('call jsondecode("[:]")')) + exc_exec('call json_decode("[:]")')) eq('Vim(call):E474: Unexpected colon: :}', - exc_exec('call jsondecode("{:}")')) + exc_exec('call json_decode("{:}")')) end) it('fails to parse containers with trailing comma', function() eq('Vim(call):E474: Trailing comma: ]', - exc_exec('call jsondecode("[1,]")')) + exc_exec('call json_decode("[1,]")')) eq('Vim(call):E474: Trailing comma: }', - exc_exec('call jsondecode("{\\"1\\": 2,}")')) + exc_exec('call json_decode("{\\"1\\": 2,}")')) end) it('fails to parse dictionaries with missing value', function() eq('Vim(call):E474: Expected value after colon: }', - exc_exec('call jsondecode("{\\"1\\":}")')) + exc_exec('call json_decode("{\\"1\\":}")')) eq('Vim(call):E474: Expected value: }', - exc_exec('call jsondecode("{\\"1\\"}")')) + exc_exec('call json_decode("{\\"1\\"}")')) end) it('fails to parse containers with two commas or colons', function() eq('Vim(call):E474: Duplicate comma: , "2": 2}', - exc_exec('call jsondecode("{\\"1\\": 1,, \\"2\\": 2}")')) + exc_exec('call json_decode("{\\"1\\": 1,, \\"2\\": 2}")')) eq('Vim(call):E474: Duplicate comma: , "2", 2]', - exc_exec('call jsondecode("[\\"1\\", 1,, \\"2\\", 2]")')) + exc_exec('call json_decode("[\\"1\\", 1,, \\"2\\", 2]")')) eq('Vim(call):E474: Duplicate colon: : 2}', - exc_exec('call jsondecode("{\\"1\\": 1, \\"2\\":: 2}")')) + exc_exec('call json_decode("{\\"1\\": 1, \\"2\\":: 2}")')) eq('Vim(call):E474: Comma after colon: , 2}', - exc_exec('call jsondecode("{\\"1\\": 1, \\"2\\":, 2}")')) + exc_exec('call json_decode("{\\"1\\": 1, \\"2\\":, 2}")')) eq('Vim(call):E474: Unexpected colon: : "2": 2}', - exc_exec('call jsondecode("{\\"1\\": 1,: \\"2\\": 2}")')) + exc_exec('call json_decode("{\\"1\\": 1,: \\"2\\": 2}")')) eq('Vim(call):E474: Unexpected colon: :, "2": 2}', - exc_exec('call jsondecode("{\\"1\\": 1:, \\"2\\": 2}")')) + exc_exec('call json_decode("{\\"1\\": 1:, \\"2\\": 2}")')) end) it('fails to parse concat of two values', function() eq('Vim(call):E474: Trailing characters: []', - exc_exec('call jsondecode("{}[]")')) + exc_exec('call json_decode("{}[]")')) end) it('parses containers', function() - eq({1}, funcs.jsondecode('[1]')) - eq({nil, 1}, funcs.jsondecode('[null, 1]')) - eq({['1']=2}, funcs.jsondecode('{"1": 2}')) + eq({1}, funcs.json_decode('[1]')) + eq({nil, 1}, funcs.json_decode('[null, 1]')) + eq({['1']=2}, funcs.json_decode('{"1": 2}')) eq({['1']=2, ['3']={{['4']={['5']={{}, 1}}}}}, - funcs.jsondecode('{"1": 2, "3": [{"4": {"5": [[], 1]}}]}')) + funcs.json_decode('{"1": 2, "3": [{"4": {"5": [[], 1]}}]}')) end) it('fails to parse incomplete strings', function() eq('Vim(call):E474: Expected string end: \t"', - exc_exec('call jsondecode("\\t\\"")')) + exc_exec('call json_decode("\\t\\"")')) eq('Vim(call):E474: Expected string end: \t"abc', - exc_exec('call jsondecode("\\t\\"abc")')) + exc_exec('call json_decode("\\t\\"abc")')) eq('Vim(call):E474: Unfinished escape sequence: \t"abc\\', - exc_exec('call jsondecode("\\t\\"abc\\\\")')) + exc_exec('call json_decode("\\t\\"abc\\\\")')) eq('Vim(call):E474: Unfinished unicode escape sequence: \t"abc\\u', - exc_exec('call jsondecode("\\t\\"abc\\\\u")')) + exc_exec('call json_decode("\\t\\"abc\\\\u")')) eq('Vim(call):E474: Unfinished unicode escape sequence: \t"abc\\u0', - exc_exec('call jsondecode("\\t\\"abc\\\\u0")')) + exc_exec('call json_decode("\\t\\"abc\\\\u0")')) eq('Vim(call):E474: Unfinished unicode escape sequence: \t"abc\\u00', - exc_exec('call jsondecode("\\t\\"abc\\\\u00")')) + exc_exec('call json_decode("\\t\\"abc\\\\u00")')) eq('Vim(call):E474: Unfinished unicode escape sequence: \t"abc\\u000', - exc_exec('call jsondecode("\\t\\"abc\\\\u000")')) + exc_exec('call json_decode("\\t\\"abc\\\\u000")')) eq('Vim(call):E474: Expected four hex digits after \\u: \\u" ', - exc_exec('call jsondecode("\\t\\"abc\\\\u\\" ")')) + exc_exec('call json_decode("\\t\\"abc\\\\u\\" ")')) eq('Vim(call):E474: Expected four hex digits after \\u: \\u0" ', - exc_exec('call jsondecode("\\t\\"abc\\\\u0\\" ")')) + exc_exec('call json_decode("\\t\\"abc\\\\u0\\" ")')) eq('Vim(call):E474: Expected four hex digits after \\u: \\u00" ', - exc_exec('call jsondecode("\\t\\"abc\\\\u00\\" ")')) + exc_exec('call json_decode("\\t\\"abc\\\\u00\\" ")')) eq('Vim(call):E474: Expected four hex digits after \\u: \\u000" ', - exc_exec('call jsondecode("\\t\\"abc\\\\u000\\" ")')) + exc_exec('call json_decode("\\t\\"abc\\\\u000\\" ")')) eq('Vim(call):E474: Expected string end: \t"abc\\u0000', - exc_exec('call jsondecode("\\t\\"abc\\\\u0000")')) + exc_exec('call json_decode("\\t\\"abc\\\\u0000")')) end) it('fails to parse unknown escape sequnces', function() eq('Vim(call):E474: Unknown escape sequence: \\a"', - exc_exec('call jsondecode("\\t\\"\\\\a\\"")')) + exc_exec('call json_decode("\\t\\"\\\\a\\"")')) end) it('parses strings properly', function() - eq('\n', funcs.jsondecode('"\\n"')) - eq('', funcs.jsondecode('""')) - eq('\\/"\t\b\n\r\f', funcs.jsondecode([["\\\/\"\t\b\n\r\f"]])) - eq('/a', funcs.jsondecode([["\/a"]])) + eq('\n', funcs.json_decode('"\\n"')) + eq('', funcs.json_decode('""')) + eq('\\/"\t\b\n\r\f', funcs.json_decode([["\\\/\"\t\b\n\r\f"]])) + eq('/a', funcs.json_decode([["\/a"]])) -- Unicode characters: 2-byte, 3-byte, 4-byte eq({ '«', 'ફ', '\xF0\x90\x80\x80', - }, funcs.jsondecode({ + }, funcs.json_decode({ '[', '"«",', '"ફ",', @@ -323,74 +323,74 @@ describe('jsondecode() function', function() it('fails on strings with invalid bytes', function() eq('Vim(call):E474: Only UTF-8 strings allowed: \255"', - exc_exec('call jsondecode("\\t\\"\\xFF\\"")')) + exc_exec('call json_decode("\\t\\"\\xFF\\"")')) eq('Vim(call):E474: ASCII control characters cannot be present inside string: ', - exc_exec('call jsondecode(["\\"\\n\\""])')) + exc_exec('call json_decode(["\\"\\n\\""])')) -- 0xC2 starts 2-byte unicode character eq('Vim(call):E474: Only UTF-8 strings allowed: \194"', - exc_exec('call jsondecode("\\t\\"\\xC2\\"")')) + exc_exec('call json_decode("\\t\\"\\xC2\\"")')) -- 0xE0 0xAA starts 3-byte unicode character eq('Vim(call):E474: Only UTF-8 strings allowed: \224"', - exc_exec('call jsondecode("\\t\\"\\xE0\\"")')) + exc_exec('call json_decode("\\t\\"\\xE0\\"")')) eq('Vim(call):E474: Only UTF-8 strings allowed: \224\170"', - exc_exec('call jsondecode("\\t\\"\\xE0\\xAA\\"")')) + exc_exec('call json_decode("\\t\\"\\xE0\\xAA\\"")')) -- 0xF0 0x90 0x80 starts 4-byte unicode character eq('Vim(call):E474: Only UTF-8 strings allowed: \240"', - exc_exec('call jsondecode("\\t\\"\\xF0\\"")')) + exc_exec('call json_decode("\\t\\"\\xF0\\"")')) eq('Vim(call):E474: Only UTF-8 strings allowed: \240\144"', - exc_exec('call jsondecode("\\t\\"\\xF0\\x90\\"")')) + exc_exec('call json_decode("\\t\\"\\xF0\\x90\\"")')) eq('Vim(call):E474: Only UTF-8 strings allowed: \240\144\128"', - exc_exec('call jsondecode("\\t\\"\\xF0\\x90\\x80\\"")')) + exc_exec('call json_decode("\\t\\"\\xF0\\x90\\x80\\"")')) -- 0xF9 0x80 0x80 0x80 starts 5-byte unicode character eq('Vim(call):E474: Only UTF-8 strings allowed: \xF9"', - exc_exec('call jsondecode("\\t\\"\\xF9\\"")')) + exc_exec('call json_decode("\\t\\"\\xF9\\"")')) eq('Vim(call):E474: Only UTF-8 strings allowed: \xF9\x80"', - exc_exec('call jsondecode("\\t\\"\\xF9\\x80\\"")')) + exc_exec('call json_decode("\\t\\"\\xF9\\x80\\"")')) eq('Vim(call):E474: Only UTF-8 strings allowed: \xF9\x80\x80"', - exc_exec('call jsondecode("\\t\\"\\xF9\\x80\\x80\\"")')) + exc_exec('call json_decode("\\t\\"\\xF9\\x80\\x80\\"")')) eq('Vim(call):E474: Only UTF-8 strings allowed: \xF9\x80\x80\x80"', - exc_exec('call jsondecode("\\t\\"\\xF9\\x80\\x80\\x80\\"")')) + exc_exec('call json_decode("\\t\\"\\xF9\\x80\\x80\\x80\\"")')) -- 0xFC 0x90 0x80 0x80 0x80 starts 6-byte unicode character eq('Vim(call):E474: Only UTF-8 strings allowed: \xFC"', - exc_exec('call jsondecode("\\t\\"\\xFC\\"")')) + exc_exec('call json_decode("\\t\\"\\xFC\\"")')) eq('Vim(call):E474: Only UTF-8 strings allowed: \xFC\x90"', - exc_exec('call jsondecode("\\t\\"\\xFC\\x90\\"")')) + exc_exec('call json_decode("\\t\\"\\xFC\\x90\\"")')) eq('Vim(call):E474: Only UTF-8 strings allowed: \xFC\x90\x80"', - exc_exec('call jsondecode("\\t\\"\\xFC\\x90\\x80\\"")')) + exc_exec('call json_decode("\\t\\"\\xFC\\x90\\x80\\"")')) eq('Vim(call):E474: Only UTF-8 strings allowed: \xFC\x90\x80\x80"', - exc_exec('call jsondecode("\\t\\"\\xFC\\x90\\x80\\x80\\"")')) + exc_exec('call json_decode("\\t\\"\\xFC\\x90\\x80\\x80\\"")')) eq('Vim(call):E474: Only UTF-8 strings allowed: \xFC\x90\x80\x80\x80"', - exc_exec('call jsondecode("\\t\\"\\xFC\\x90\\x80\\x80\\x80\\"")')) + exc_exec('call json_decode("\\t\\"\\xFC\\x90\\x80\\x80\\x80\\"")')) -- Specification does not allow unquoted characters above 0x10FFFF eq('Vim(call):E474: Only UTF-8 code points up to U+10FFFF are allowed to appear unescaped: \xF9\x80\x80\x80\x80"', - exc_exec('call jsondecode("\\t\\"\\xF9\\x80\\x80\\x80\\x80\\"")')) + exc_exec('call json_decode("\\t\\"\\xF9\\x80\\x80\\x80\\x80\\"")')) eq('Vim(call):E474: Only UTF-8 code points up to U+10FFFF are allowed to appear unescaped: \xFC\x90\x80\x80\x80\x80"', - exc_exec('call jsondecode("\\t\\"\\xFC\\x90\\x80\\x80\\x80\\x80\\"")')) + exc_exec('call json_decode("\\t\\"\\xFC\\x90\\x80\\x80\\x80\\x80\\"")')) -- '"\xF9\x80\x80\x80\x80"', -- '"\xFC\x90\x80\x80\x80\x80"', end) it('parses surrogate pairs properly', function() - eq('\xF0\x90\x80\x80', funcs.jsondecode('"\\uD800\\uDC00"')) - eq('\xED\xA0\x80a\xED\xB0\x80', funcs.jsondecode('"\\uD800a\\uDC00"')) - eq('\xED\xA0\x80\t\xED\xB0\x80', funcs.jsondecode('"\\uD800\\t\\uDC00"')) + eq('\xF0\x90\x80\x80', funcs.json_decode('"\\uD800\\uDC00"')) + eq('\xED\xA0\x80a\xED\xB0\x80', funcs.json_decode('"\\uD800a\\uDC00"')) + eq('\xED\xA0\x80\t\xED\xB0\x80', funcs.json_decode('"\\uD800\\t\\uDC00"')) - eq('\xED\xA0\x80', funcs.jsondecode('"\\uD800"')) - eq('\xED\xA0\x80a', funcs.jsondecode('"\\uD800a"')) - eq('\xED\xA0\x80\t', funcs.jsondecode('"\\uD800\\t"')) + eq('\xED\xA0\x80', funcs.json_decode('"\\uD800"')) + eq('\xED\xA0\x80a', funcs.json_decode('"\\uD800a"')) + eq('\xED\xA0\x80\t', funcs.json_decode('"\\uD800\\t"')) - eq('\xED\xB0\x80', funcs.jsondecode('"\\uDC00"')) - eq('\xED\xB0\x80a', funcs.jsondecode('"\\uDC00a"')) - eq('\xED\xB0\x80\t', funcs.jsondecode('"\\uDC00\\t"')) + eq('\xED\xB0\x80', funcs.json_decode('"\\uDC00"')) + eq('\xED\xB0\x80a', funcs.json_decode('"\\uDC00a"')) + eq('\xED\xB0\x80\t', funcs.json_decode('"\\uDC00\\t"')) - eq('\xED\xB0\x80', funcs.jsondecode('"\\uDC00"')) - eq('a\xED\xB0\x80', funcs.jsondecode('"a\\uDC00"')) - eq('\t\xED\xB0\x80', funcs.jsondecode('"\\t\\uDC00"')) + eq('\xED\xB0\x80', funcs.json_decode('"\\uDC00"')) + eq('a\xED\xB0\x80', funcs.json_decode('"a\\uDC00"')) + eq('\t\xED\xB0\x80', funcs.json_decode('"\\t\\uDC00"')) end) local sp_decode_eq = function(expected, json) meths.set_var('__json', json) - speq(expected, 'jsondecode(g:__json)') + speq(expected, 'json_decode(g:__json)') execute('unlet! g:__json') end @@ -441,47 +441,47 @@ describe('jsondecode() function', function() it('converts strings to latin1 when &encoding is latin1', function() restart('set encoding=latin1') - eq('\xAB', funcs.jsondecode('"\\u00AB"')) + eq('\xAB', funcs.json_decode('"\\u00AB"')) sp_decode_eq({_TYPE='string', _VAL={'\n\xAB\n'}}, '"\\u0000\\u00AB\\u0000"') end) end) -describe('jsonencode() function', function() +describe('json_encode() function', function() before_each(clear) it('dumps strings', function() - eq('"Test"', funcs.jsonencode('Test')) - eq('""', funcs.jsonencode('')) - eq('"\\t"', funcs.jsonencode('\t')) - eq('"\\n"', funcs.jsonencode('\n')) - eq('"\\u001B"', funcs.jsonencode('\27')) - eq('"þÿþ"', funcs.jsonencode('þÿþ')) + eq('"Test"', funcs.json_encode('Test')) + eq('""', funcs.json_encode('')) + eq('"\\t"', funcs.json_encode('\t')) + eq('"\\n"', funcs.json_encode('\n')) + eq('"\\u001B"', funcs.json_encode('\27')) + eq('"þÿþ"', funcs.json_encode('þÿþ')) end) it('dumps numbers', function() - eq('0', funcs.jsonencode(0)) - eq('10', funcs.jsonencode(10)) - eq('-10', funcs.jsonencode(-10)) + eq('0', funcs.json_encode(0)) + eq('10', funcs.json_encode(10)) + eq('-10', funcs.json_encode(-10)) end) it('dumps floats', function() - eq('0.0', eval('jsonencode(0.0)')) - eq('10.5', funcs.jsonencode(10.5)) - eq('-10.5', funcs.jsonencode(-10.5)) - eq('-1.0e-5', funcs.jsonencode(-1e-5)) - eq('1.0e50', eval('jsonencode(1.0e50)')) + eq('0.0', eval('json_encode(0.0)')) + eq('10.5', funcs.json_encode(10.5)) + eq('-10.5', funcs.json_encode(-10.5)) + eq('-1.0e-5', funcs.json_encode(-1e-5)) + eq('1.0e50', eval('json_encode(1.0e50)')) end) it('dumps lists', function() - eq('[]', funcs.jsonencode({})) - eq('[[]]', funcs.jsonencode({{}})) - eq('[[], []]', funcs.jsonencode({{}, {}})) + eq('[]', funcs.json_encode({})) + eq('[[]]', funcs.json_encode({{}})) + eq('[[], []]', funcs.json_encode({{}, {}})) end) it('dumps dictionaries', function() - eq('{}', eval('jsonencode({})')) - eq('{"d": []}', funcs.jsonencode({d={}})) - eq('{"d": [], "e": []}', funcs.jsonencode({d={}, e={}})) + eq('{}', eval('json_encode({})')) + eq('{"d": []}', funcs.json_encode({d={}})) + eq('{"d": [], "e": []}', funcs.json_encode({d={}, e={}})) end) it('cannot dump generic mapping with generic mapping keys and values', @@ -490,148 +490,148 @@ describe('jsonencode() function', function() execute('let todumpv1 = {"_TYPE": v:msgpack_types.map, "_VAL": []}') execute('let todumpv2 = {"_TYPE": v:msgpack_types.map, "_VAL": []}') execute('call add(todump._VAL, [todumpv1, todumpv2])') - eq('Vim(call):E474: Invalid key in special dictionary', exc_exec('call jsonencode(todump)')) + eq('Vim(call):E474: Invalid key in special dictionary', exc_exec('call json_encode(todump)')) end) it('cannot dump generic mapping with ext key', function() execute('let todump = {"_TYPE": v:msgpack_types.ext, "_VAL": [5, ["",""]]}') execute('let todump = {"_TYPE": v:msgpack_types.map, "_VAL": [[todump, 1]]}') - eq('Vim(call):E474: Invalid key in special dictionary', exc_exec('call jsonencode(todump)')) + eq('Vim(call):E474: Invalid key in special dictionary', exc_exec('call json_encode(todump)')) end) it('cannot dump generic mapping with array key', function() execute('let todump = {"_TYPE": v:msgpack_types.array, "_VAL": [5, [""]]}') execute('let todump = {"_TYPE": v:msgpack_types.map, "_VAL": [[todump, 1]]}') - eq('Vim(call):E474: Invalid key in special dictionary', exc_exec('call jsonencode(todump)')) + eq('Vim(call):E474: Invalid key in special dictionary', exc_exec('call json_encode(todump)')) end) it('cannot dump generic mapping with UINT64_MAX key', function() execute('let todump = {"_TYPE": v:msgpack_types.integer}') execute('let todump._VAL = [1, 3, 0x7FFFFFFF, 0x7FFFFFFF]') execute('let todump = {"_TYPE": v:msgpack_types.map, "_VAL": [[todump, 1]]}') - eq('Vim(call):E474: Invalid key in special dictionary', exc_exec('call jsonencode(todump)')) + eq('Vim(call):E474: Invalid key in special dictionary', exc_exec('call json_encode(todump)')) end) it('cannot dump generic mapping with floating-point key', function() execute('let todump = {"_TYPE": v:msgpack_types.float, "_VAL": 0.125}') execute('let todump = {"_TYPE": v:msgpack_types.map, "_VAL": [[todump, 1]]}') - eq('Vim(call):E474: Invalid key in special dictionary', exc_exec('call jsonencode(todump)')) + eq('Vim(call):E474: Invalid key in special dictionary', exc_exec('call json_encode(todump)')) end) it('can dump generic mapping with STR special key and NUL', function() execute('let todump = {"_TYPE": v:msgpack_types.string, "_VAL": ["\\n"]}') execute('let todump = {"_TYPE": v:msgpack_types.map, "_VAL": [[todump, 1]]}') - eq('{"\\u0000": 1}', eval('jsonencode(todump)')) + eq('{"\\u0000": 1}', eval('json_encode(todump)')) end) it('can dump generic mapping with BIN special key and NUL', function() execute('let todump = {"_TYPE": v:msgpack_types.binary, "_VAL": ["\\n"]}') execute('let todump = {"_TYPE": v:msgpack_types.map, "_VAL": [[todump, 1]]}') - eq('{"\\u0000": 1}', eval('jsonencode(todump)')) + eq('{"\\u0000": 1}', eval('json_encode(todump)')) end) it('can dump STR special mapping with NUL and NL', function() execute('let todump = {"_TYPE": v:msgpack_types.string, "_VAL": ["\\n", ""]}') - eq('"\\u0000\\n"', eval('jsonencode(todump)')) + eq('"\\u0000\\n"', eval('json_encode(todump)')) end) it('can dump BIN special mapping with NUL and NL', function() execute('let todump = {"_TYPE": v:msgpack_types.binary, "_VAL": ["\\n", ""]}') - eq('"\\u0000\\n"', eval('jsonencode(todump)')) + eq('"\\u0000\\n"', eval('json_encode(todump)')) end) it('cannot dump special ext mapping', function() execute('let todump = {"_TYPE": v:msgpack_types.ext, "_VAL": [5, ["",""]]}') - eq('Vim(call):E474: Unable to convert EXT string to JSON', exc_exec('call jsonencode(todump)')) + eq('Vim(call):E474: Unable to convert EXT string to JSON', exc_exec('call json_encode(todump)')) end) it('can dump special array mapping', function() execute('let todump = {"_TYPE": v:msgpack_types.array, "_VAL": [5, [""]]}') - eq('[5, [""]]', eval('jsonencode(todump)')) + eq('[5, [""]]', eval('json_encode(todump)')) end) it('can dump special UINT64_MAX mapping', function() execute('let todump = {"_TYPE": v:msgpack_types.integer}') execute('let todump._VAL = [1, 3, 0x7FFFFFFF, 0x7FFFFFFF]') - eq('18446744073709551615', eval('jsonencode(todump)')) + eq('18446744073709551615', eval('json_encode(todump)')) end) it('can dump special INT64_MIN mapping', function() execute('let todump = {"_TYPE": v:msgpack_types.integer}') execute('let todump._VAL = [-1, 2, 0, 0]') - eq('-9223372036854775808', eval('jsonencode(todump)')) + eq('-9223372036854775808', eval('json_encode(todump)')) end) it('can dump special BOOLEAN true mapping', function() execute('let todump = {"_TYPE": v:msgpack_types.boolean, "_VAL": 1}') - eq('true', eval('jsonencode(todump)')) + eq('true', eval('json_encode(todump)')) end) it('can dump special BOOLEAN false mapping', function() execute('let todump = {"_TYPE": v:msgpack_types.boolean, "_VAL": 0}') - eq('false', eval('jsonencode(todump)')) + eq('false', eval('json_encode(todump)')) end) it('can dump special NIL mapping', function() execute('let todump = {"_TYPE": v:msgpack_types.nil, "_VAL": 0}') - eq('null', eval('jsonencode(todump)')) + eq('null', eval('json_encode(todump)')) end) it('fails to dump a function reference', function() eq('Vim(call):E474: Error while dumping encode_tv2json() argument, itself: attempt to dump function reference', - exc_exec('call jsonencode(function("tr"))')) + exc_exec('call json_encode(function("tr"))')) end) it('fails to dump a function reference in a list', function() eq('Vim(call):E474: Error while dumping encode_tv2json() argument, index 0: attempt to dump function reference', - exc_exec('call jsonencode([function("tr")])')) + exc_exec('call json_encode([function("tr")])')) end) it('fails to dump a recursive list', function() execute('let todump = [[[]]]') execute('call add(todump[0][0], todump)') eq('Vim(call):E724: unable to correctly dump variable with self-referencing container', - exc_exec('call jsonencode(todump)')) + exc_exec('call json_encode(todump)')) end) it('fails to dump a recursive dict', function() execute('let todump = {"d": {"d": {}}}') execute('call extend(todump.d.d, {"d": todump})') eq('Vim(call):E724: unable to correctly dump variable with self-referencing container', - exc_exec('call jsonencode([todump])')) + exc_exec('call json_encode([todump])')) end) it('can dump dict with two same dicts inside', function() execute('let inter = {}') execute('let todump = {"a": inter, "b": inter}') - eq('{"a": {}, "b": {}}', eval('jsonencode(todump)')) + eq('{"a": {}, "b": {}}', eval('json_encode(todump)')) end) it('can dump list with two same lists inside', function() execute('let inter = []') execute('let todump = [inter, inter]') - eq('[[], []]', eval('jsonencode(todump)')) + eq('[[], []]', eval('json_encode(todump)')) end) it('fails to dump a recursive list in a special dict', function() execute('let todump = {"_TYPE": v:msgpack_types.array, "_VAL": []}') execute('call add(todump._VAL, todump)') eq('Vim(call):E724: unable to correctly dump variable with self-referencing container', - exc_exec('call jsonencode(todump)')) + exc_exec('call json_encode(todump)')) end) it('fails to dump a recursive (val) map in a special dict', function() execute('let todump = {"_TYPE": v:msgpack_types.map, "_VAL": []}') execute('call add(todump._VAL, ["", todump])') eq('Vim(call):E724: unable to correctly dump variable with self-referencing container', - exc_exec('call jsonencode([todump])')) + exc_exec('call json_encode([todump])')) end) it('fails to dump a recursive (val) map in a special dict, _VAL reference', function() execute('let todump = {"_TYPE": v:msgpack_types.map, "_VAL": [["", []]]}') execute('call add(todump._VAL[0][1], todump._VAL)') eq('Vim(call):E724: unable to correctly dump variable with self-referencing container', - exc_exec('call jsonencode(todump)')) + exc_exec('call json_encode(todump)')) end) it('fails to dump a recursive (val) special list in a special dict', @@ -639,35 +639,35 @@ describe('jsonencode() function', function() execute('let todump = {"_TYPE": v:msgpack_types.array, "_VAL": []}') execute('call add(todump._VAL, ["", todump._VAL])') eq('Vim(call):E724: unable to correctly dump variable with self-referencing container', - exc_exec('call jsonencode(todump)')) + exc_exec('call json_encode(todump)')) end) it('fails when called with no arguments', function() - eq('Vim(call):E119: Not enough arguments for function: jsonencode', - exc_exec('call jsonencode()')) + eq('Vim(call):E119: Not enough arguments for function: json_encode', + exc_exec('call json_encode()')) end) it('fails when called with two arguments', function() - eq('Vim(call):E118: Too many arguments for function: jsonencode', - exc_exec('call jsonencode(["", ""], 1)')) + eq('Vim(call):E118: Too many arguments for function: json_encode', + exc_exec('call json_encode(["", ""], 1)')) end) it('converts strings from latin1 when &encoding is latin1', function() clear('set encoding=latin1') - eq('"\\u00AB"', funcs.jsonencode('\xAB')) - eq('"\\u0000\\u00AB\\u0000"', eval('jsonencode({"_TYPE": v:msgpack_types.string, "_VAL": ["\\n\xAB\\n"]})')) + eq('"\\u00AB"', funcs.json_encode('\xAB')) + eq('"\\u0000\\u00AB\\u0000"', eval('json_encode({"_TYPE": v:msgpack_types.string, "_VAL": ["\\n\xAB\\n"]})')) end) it('ignores improper values in &isprint', function() meths.set_option('isprint', '1') eq(1, eval('"\x01" =~# "\\\\p"')) - eq('"\\u0001"', funcs.jsonencode('\x01')) + eq('"\\u0001"', funcs.json_encode('\x01')) end) it('fails when using surrogate character in a UTF-8 string', function() eq('Vim(call):E474: UTF-8 string contains code point which belongs to a surrogate pair: \xED\xA0\x80', - exc_exec('call jsonencode("\xED\xA0\x80")')) + exc_exec('call json_encode("\xED\xA0\x80")')) eq('Vim(call):E474: UTF-8 string contains code point which belongs to a surrogate pair: \xED\xAF\xBF', - exc_exec('call jsonencode("\xED\xAF\xBF")')) + exc_exec('call json_encode("\xED\xAF\xBF")')) end) end) From 7cdd01983aeb452e0a3f3eb027e75fe02ce48718 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 21 Feb 2016 05:17:20 +0300 Subject: [PATCH 45/82] api/documentation: Add a warning that nil may mean v:null --- src/nvim/api/buffer.c | 5 ++++- src/nvim/api/tabpage.c | 5 ++++- src/nvim/api/vim.c | 5 ++++- src/nvim/api/window.c | 5 ++++- 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/src/nvim/api/buffer.c b/src/nvim/api/buffer.c index 075f101f61..a01188f98c 100644 --- a/src/nvim/api/buffer.c +++ b/src/nvim/api/buffer.c @@ -429,7 +429,10 @@ Object buffer_get_var(Buffer buffer, String name, Error *err) /// @param name The variable name /// @param value The variable value /// @param[out] err Details of an error that may have occurred -/// @return The old value +/// @return The old value or nil if there was no previous value. +/// +/// @warning It may return nil if there was no previous value +/// or if previous value was `v:null`. Object buffer_set_var(Buffer buffer, String name, Object value, Error *err) { buf_T *buf = find_buffer_by_handle(buffer, err); diff --git a/src/nvim/api/tabpage.c b/src/nvim/api/tabpage.c index 475b75b571..3148a4fab7 100644 --- a/src/nvim/api/tabpage.c +++ b/src/nvim/api/tabpage.c @@ -60,7 +60,10 @@ Object tabpage_get_var(Tabpage tabpage, String name, Error *err) /// @param name The variable name /// @param value The variable value /// @param[out] err Details of an error that may have occurred -/// @return The tab page handle +/// @return The old value or nil if there was no previous value. +/// +/// @warning It may return nil if there was no previous value +/// or if previous value was `v:null`. Object tabpage_set_var(Tabpage tabpage, String name, Object value, Error *err) { tabpage_T *tab = find_tab_by_handle(tabpage, err); diff --git a/src/nvim/api/vim.c b/src/nvim/api/vim.c index 09d5f1b6f8..dcae08d24c 100644 --- a/src/nvim/api/vim.c +++ b/src/nvim/api/vim.c @@ -336,7 +336,10 @@ Object vim_get_var(String name, Error *err) /// @param name The variable name /// @param value The variable value /// @param[out] err Details of an error that may have occurred -/// @return the old value if any +/// @return The old value or nil if there was no previous value. +/// +/// @warning It may return nil if there was no previous value +/// or if previous value was `v:null`. Object vim_set_var(String name, Object value, Error *err) { return dict_set_value(&globvardict, name, value, false, err); diff --git a/src/nvim/api/window.c b/src/nvim/api/window.c index 58218af09d..cca1e3bdd3 100644 --- a/src/nvim/api/window.c +++ b/src/nvim/api/window.c @@ -203,7 +203,10 @@ Object window_get_var(Window window, String name, Error *err) /// @param name The variable name /// @param value The variable value /// @param[out] err Details of an error that may have occurred -/// @return The old value +/// @return The old value or nil if there was no previous value. +/// +/// @warning It may return nil if there was no previous value +/// or if previous value was `v:null`. Object window_set_var(Window window, String name, Object value, Error *err) { win_T *win = find_window_by_handle(window, err); From 406562ac6d3863dfdaedbf40f9d4a23ca37c9ec5 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 21 Feb 2016 21:33:58 +0300 Subject: [PATCH 46/82] encode: Fail to dump NaN and infinity Thanks to vim/vim#654 --- src/nvim/eval/encode.c | 21 +++++++++++++++++--- test/functional/eval/json_functions_spec.lua | 9 +++++++++ 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/src/nvim/eval/encode.c b/src/nvim/eval/encode.c index a131f5c3c1..d21347cca6 100644 --- a/src/nvim/eval/encode.c +++ b/src/nvim/eval/encode.c @@ -7,6 +7,7 @@ #include #include #include +#include #include "nvim/eval/encode.h" #include "nvim/buffer_defs.h" // vimconv_T @@ -827,9 +828,23 @@ DEFINE_VIML_CONV_FUNCTIONS(, echo, garray_T *const, gap) #undef CONV_FLOAT #define CONV_FLOAT(flt) \ do { \ - char numbuf[NUMBUFLEN]; \ - vim_snprintf(numbuf, NUMBUFLEN - 1, "%g", (flt)); \ - ga_concat(gap, numbuf); \ + const float_T flt_ = (flt); \ + switch (fpclassify(flt_)) { \ + case FP_NAN: { \ + EMSG(_("E474: Unable to represent NaN value in JSON")); \ + return FAIL; \ + } \ + case FP_INFINITE: { \ + EMSG(_("E474: Unable to represent infinity in JSON")); \ + return FAIL; \ + } \ + default: { \ + char numbuf[NUMBUFLEN]; \ + vim_snprintf(numbuf, NUMBUFLEN - 1, "%g", flt_); \ + ga_concat(gap, (char_u *) numbuf); \ + break; \ + } \ + } \ } while (0) /// Last used p_enc value diff --git a/test/functional/eval/json_functions_spec.lua b/test/functional/eval/json_functions_spec.lua index 0b0403ce8e..33d177a2b6 100644 --- a/test/functional/eval/json_functions_spec.lua +++ b/test/functional/eval/json_functions_spec.lua @@ -472,6 +472,15 @@ describe('json_encode() function', function() eq('1.0e50', eval('json_encode(1.0e50)')) end) + it('fails to dump NaN and infinite values', function() + eq('Vim(call):E474: Unable to represent NaN value in JSON', + exc_exec('call json_encode(str2float("nan"))')) + eq('Vim(call):E474: Unable to represent infinity in JSON', + exc_exec('call json_encode(str2float("inf"))')) + eq('Vim(call):E474: Unable to represent infinity in JSON', + exc_exec('call json_encode(-str2float("inf"))')) + end) + it('dumps lists', function() eq('[]', funcs.json_encode({})) eq('[[]]', funcs.json_encode({{}})) From 942e0b338c9bff1dfdcb59e8308160449f1f38b4 Mon Sep 17 00:00:00 2001 From: ZyX Date: Thu, 25 Feb 2016 17:27:23 +0300 Subject: [PATCH 47/82] encode: Handle incomplete surrogates like `\uSURR\uOTHR` properly --- src/nvim/eval/decode.c | 18 +++++++++++------- test/functional/eval/json_functions_spec.lua | 2 ++ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/nvim/eval/decode.c b/src/nvim/eval/decode.c index c6706eb0dd..4955a4f5a4 100644 --- a/src/nvim/eval/decode.c +++ b/src/nvim/eval/decode.c @@ -459,12 +459,16 @@ json_decode_string_cycle_start: int fst_in_pair = 0; char *str_end = str; bool hasnul = false; +#define PUT_FST_IN_PAIR(fst_in_pair, str_end) \ + do { \ + if (fst_in_pair != 0) { \ + str_end += utf_char2bytes(fst_in_pair, (char_u *) str_end); \ + fst_in_pair = 0; \ + } \ + } while (0) for (const char *t = s; t < p; t++) { if (t[0] != '\\' || t[1] != 'u') { - if (fst_in_pair != 0) { - str_end += utf_char2bytes(fst_in_pair, (char_u *) str_end); - fst_in_pair = 0; - } + PUT_FST_IN_PAIR(fst_in_pair, str_end); } if (*t == '\\') { t++; @@ -489,6 +493,7 @@ json_decode_string_cycle_start: str_end += utf_char2bytes(full_char, (char_u *) str_end); fst_in_pair = 0; } else { + PUT_FST_IN_PAIR(fst_in_pair, str_end); str_end += utf_char2bytes((int) ch, (char_u *) str_end); } break; @@ -522,9 +527,8 @@ json_decode_string_cycle_start: *str_end++ = *t; } } - if (fst_in_pair != 0) { - str_end += utf_char2bytes((int) fst_in_pair, (char_u *) str_end); - } + PUT_FST_IN_PAIR(fst_in_pair, str_end); +#undef PUT_FST_IN_PAIR if (conv.vc_type != CONV_NONE) { size_t str_len = (size_t) (str_end - str); char *const new_str = (char *) string_convert(&conv, (char_u *) str, diff --git a/test/functional/eval/json_functions_spec.lua b/test/functional/eval/json_functions_spec.lua index 33d177a2b6..d6286611d2 100644 --- a/test/functional/eval/json_functions_spec.lua +++ b/test/functional/eval/json_functions_spec.lua @@ -386,6 +386,8 @@ describe('json_decode() function', function() eq('\xED\xB0\x80', funcs.json_decode('"\\uDC00"')) eq('a\xED\xB0\x80', funcs.json_decode('"a\\uDC00"')) eq('\t\xED\xB0\x80', funcs.json_decode('"\\t\\uDC00"')) + + eq('\xED\xA0\x80¬', funcs.json_decode('"\\uD800\\u00AC"')) end) local sp_decode_eq = function(expected, json) From 6a35f2ac8bd7a3a0e8c9b988b85237424bc10e73 Mon Sep 17 00:00:00 2001 From: ZyX Date: Tue, 1 Mar 2016 02:25:15 +0300 Subject: [PATCH 48/82] eval: Do not break when VimVarIndex and vimvars order mismatches Also makes sure that compiler will error out when new name is longer then vv_filler. --- src/nvim/assert.h | 101 +++++++++++++++++++--------- src/nvim/eval.c | 165 +++++++++++++++++++++++++--------------------- src/nvim/eval.h | 2 - 3 files changed, 161 insertions(+), 107 deletions(-) diff --git a/src/nvim/assert.h b/src/nvim/assert.h index 3a900aca65..0ce48e4766 100644 --- a/src/nvim/assert.h +++ b/src/nvim/assert.h @@ -8,17 +8,32 @@ // defined(__has_feature) && __has_feature(...). Therefore we define Clang's // __has_feature and __has_extension macro's before referring to them. #ifndef __has_feature - #define __has_feature(x) 0 +# define __has_feature(x) 0 #endif #ifndef __has_extension - #define __has_extension __has_feature +# define __has_extension __has_feature #endif -/// STATIC_ASSERT(condition, message) - assert at compile time if !cond +/// @def STATIC_ASSERT +/// @brief Assert at compile time if condition is not satisfied. /// -/// example: -/// STATIC_ASSERT(sizeof(void *) == 8, "need 64-bits mode"); +/// Should be put on its own line, followed by a semicolon. +/// +/// Example: +/// +/// STATIC_ASSERT(sizeof(void *) == 8, "Expected 64-bit mode"); +/// +/// @param[in] condition Condition to check, should be an integer constant +/// expression. +/// @param[in] message Message which will be given if check fails. + +/// @def STATIC_ASSERT_EXPR +/// @brief Like #STATIC_ASSERT, but can be used where expressions are used. +/// +/// STATIC_ASSERT_EXPR may be put in brace initializer lists. Error message +/// given in this case is not very nice with the current implementation though +/// and `message` argument is ignored. // define STATIC_ASSERT as C11's _Static_assert whenever either C11 mode is // detected or the compiler is known to support it. Note that Clang in C99 @@ -29,50 +44,74 @@ // clearer messages we get from _Static_assert, we suppress the warnings // temporarily. +#define STATIC_ASSERT_PRAGMA_START +#define STATIC_ASSERT_PRAGMA_END +#define STATIC_ASSERT(...) \ + do { \ + STATIC_ASSERT_PRAGMA_START \ + STATIC_ASSERT_STATEMENT(__VA_ARGS__); \ + STATIC_ASSERT_PRAGMA_END \ + } while (0) + // the easiest case, when the mode is C11 (generic compiler) or Clang // advertises explicit support for c_static_assert, meaning it won't warn. #if __STDC_VERSION__ >= 201112L || __has_feature(c_static_assert) - #define STATIC_ASSERT(cond, msg) _Static_assert(cond, msg) +# define STATIC_ASSERT_STATEMENT(cond, msg) _Static_assert(cond, msg) // if we're dealing with gcc >= 4.6 in C99 mode, we can still use // _Static_assert but we need to suppress warnings, this is pretty ugly. #elif (!defined(__clang__) && !defined(__INTEL_COMPILER)) && \ (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) - #define STATIC_ASSERT(cond, msg) \ - _Pragma("GCC diagnostic push") \ - _Pragma("GCC diagnostic ignored \"-pedantic\"") \ - _Static_assert(cond, msg); \ - _Pragma("GCC diagnostic pop") \ + +# define STATIC_ASSERT_STATEMENT(cond, msg) _Static_assert(cond, msg) + +# undef STATIC_ASSERT_PRAGMA_START +# define STATIC_ASSERT_PRAGMA_START \ + _Pragma("GCC diagnostic push") \ + _Pragma("GCC diagnostic ignored \"-pedantic\"") \ + +# undef STATIC_ASSERT_PRAGMA_END +# define STATIC_ASSERT_PRAGMA_END \ + _Pragma("GCC diagnostic pop") \ // the same goes for clang in C99 mode, but we suppress a different warning #elif defined(__clang__) && __has_extension(c_static_assert) - #define STATIC_ASSERT(cond, msg) \ - _Pragma("clang diagnostic push") \ - _Pragma("clang diagnostic ignored \"-Wc11-extensions\"") \ - _Static_assert(cond, msg); \ - _Pragma("clang diagnostic pop") \ + +# define STATIC_ASSERT_STATEMENT(cond, msg) _Static_assert(cond, msg) + +# undef STATIC_ASSERT_PRAGMA_START +# define STATIC_ASSERT_PRAGMA_START \ + _Pragma("clang diagnostic push") \ + _Pragma("clang diagnostic ignored \"-Wc11-extensions\"") \ + +# undef STATIC_ASSERT_PRAGMA_END +# define STATIC_ASSERT_PRAGMA_END \ + _Pragma("clang diagnostic pop") \ // TODO(aktau): verify that this works, don't have MSVC on hand. #elif _MSC_VER >= 1600 - #define STATIC_ASSERT(cond, msg) static_assert(cond, msg) + +# define STATIC_ASSERT_STATEMENT(cond, msg) static_assert(cond, msg) // fallback for compilers that don't support _Static_assert or static_assert // not as pretty but gets the job done. Credit goes to Pádraig Brady and // contributors. #else - #define ASSERT_CONCAT_(a, b) a##b - #define ASSERT_CONCAT(a, b) ASSERT_CONCAT_(a, b) - // These can't be used after statements in c89. - #ifdef __COUNTER__ - #define STATIC_ASSERT(e,m) \ - { enum { ASSERT_CONCAT(static_assert_, __COUNTER__) = 1/(!!(e)) }; } - #else - // This can't be used twice on the same line so ensure if using in headers - // that the headers are not included twice (by wrapping in #ifndef...#endif) - // Note it doesn't cause an issue when used on same line of separate modules - // compiled with gcc -combine -fwhole-program. - #define STATIC_ASSERT(e,m) \ - { enum { ASSERT_CONCAT(assert_line_, __LINE__) = 1/(!!(e)) }; } - #endif +# define STATIC_ASSERT_STATEMENT STATIC_ASSERT_EXPR +#endif + +#define ASSERT_CONCAT_(a, b) a##b +#define ASSERT_CONCAT(a, b) ASSERT_CONCAT_(a, b) +// These can't be used after statements in c89. +#ifdef __COUNTER__ +# define STATIC_ASSERT_EXPR(e, m) \ + ((enum { ASSERT_CONCAT(static_assert_, __COUNTER__) = 1/(!!(e)) }) 0) +#else +// This can't be used twice on the same line so ensure if using in headers +// that the headers are not included twice (by wrapping in #ifndef...#endif) +// Note it doesn't cause an issue when used on same line of separate modules +// compiled with gcc -combine -fwhole-program. +# define STATIC_ASSERT_EXPR(e, m) \ + ((enum { ASSERT_CONCAT(assert_line_, __LINE__) = 1/(!!(e)) }) 0) #endif #endif // NVIM_ASSERT_H diff --git a/src/nvim/eval.c b/src/nvim/eval.c index c23613aeab..8fcb570525 100644 --- a/src/nvim/eval.c +++ b/src/nvim/eval.c @@ -282,7 +282,21 @@ typedef enum { #define VV_RO 2 /* read-only */ #define VV_RO_SBX 4 /* read-only in the sandbox */ -#define VV_NAME(s, t) s, {{t, 0, {0}}, 0, {0}}, {0} +#define VV(idx, name, type, flags) \ + [idx] = { \ + .vv_name = name, \ + .vv_di = { \ + .di_tv = { .v_type = type }, \ + .di_flags = 0, \ + }, \ + .vv_filler = {( \ + STATIC_ASSERT_EXPR( \ + (sizeof(name) - 1 <= sizeof(vimvars[0].vv_filler)), \ + "Vim variable name is too long, adjust vv_filler size"), \ + 0 \ + )}, \ + .vv_flags = flags, \ + } // Array to hold the value of v: variables. // The value is in a dictitem, so that it can also be used in the v: scope. @@ -295,80 +309,83 @@ static struct vimvar { char vv_flags; ///< Flags: #VV_COMPAT, #VV_RO, #VV_RO_SBX. } vimvars[] = { - /* - * The order here must match the VV_ defines in eval.h! - * Initializing a union does not work, leave tv.vval empty to get zero's. - */ - { VV_NAME("count", VAR_NUMBER), VV_COMPAT+VV_RO }, - { VV_NAME("count1", VAR_NUMBER), VV_RO }, - { VV_NAME("prevcount", VAR_NUMBER), VV_RO }, - { VV_NAME("errmsg", VAR_STRING), VV_COMPAT }, - { VV_NAME("warningmsg", VAR_STRING), 0 }, - { VV_NAME("statusmsg", VAR_STRING), 0 }, - { VV_NAME("shell_error", VAR_NUMBER), VV_COMPAT+VV_RO }, - { VV_NAME("this_session", VAR_STRING), VV_COMPAT }, - { VV_NAME("version", VAR_NUMBER), VV_COMPAT+VV_RO }, - { VV_NAME("lnum", VAR_NUMBER), VV_RO_SBX }, - { VV_NAME("termresponse", VAR_STRING), VV_RO }, - { VV_NAME("fname", VAR_STRING), VV_RO }, - { VV_NAME("lang", VAR_STRING), VV_RO }, - { VV_NAME("lc_time", VAR_STRING), VV_RO }, - { VV_NAME("ctype", VAR_STRING), VV_RO }, - { VV_NAME("charconvert_from", VAR_STRING), VV_RO }, - { VV_NAME("charconvert_to", VAR_STRING), VV_RO }, - { VV_NAME("fname_in", VAR_STRING), VV_RO }, - { VV_NAME("fname_out", VAR_STRING), VV_RO }, - { VV_NAME("fname_new", VAR_STRING), VV_RO }, - { VV_NAME("fname_diff", VAR_STRING), VV_RO }, - { VV_NAME("cmdarg", VAR_STRING), VV_RO }, - { VV_NAME("foldstart", VAR_NUMBER), VV_RO_SBX }, - { VV_NAME("foldend", VAR_NUMBER), VV_RO_SBX }, - { VV_NAME("folddashes", VAR_STRING), VV_RO_SBX }, - { VV_NAME("foldlevel", VAR_NUMBER), VV_RO_SBX }, - { VV_NAME("progname", VAR_STRING), VV_RO }, - { VV_NAME("servername", VAR_STRING), VV_RO }, - { VV_NAME("dying", VAR_NUMBER), VV_RO }, - { VV_NAME("exception", VAR_STRING), VV_RO }, - { VV_NAME("throwpoint", VAR_STRING), VV_RO }, - { VV_NAME("register", VAR_STRING), VV_RO }, - { VV_NAME("cmdbang", VAR_NUMBER), VV_RO }, - { VV_NAME("insertmode", VAR_STRING), VV_RO }, - { VV_NAME("val", VAR_UNKNOWN), VV_RO }, - { VV_NAME("key", VAR_UNKNOWN), VV_RO }, - { VV_NAME("profiling", VAR_NUMBER), VV_RO }, - { VV_NAME("fcs_reason", VAR_STRING), VV_RO }, - { VV_NAME("fcs_choice", VAR_STRING), 0 }, - { VV_NAME("beval_bufnr", VAR_NUMBER), VV_RO }, - { VV_NAME("beval_winnr", VAR_NUMBER), VV_RO }, - { VV_NAME("beval_lnum", VAR_NUMBER), VV_RO }, - { VV_NAME("beval_col", VAR_NUMBER), VV_RO }, - { VV_NAME("beval_text", VAR_STRING), VV_RO }, - { VV_NAME("scrollstart", VAR_STRING), 0 }, - { VV_NAME("swapname", VAR_STRING), VV_RO }, - { VV_NAME("swapchoice", VAR_STRING), 0 }, - { VV_NAME("swapcommand", VAR_STRING), VV_RO }, - { VV_NAME("char", VAR_STRING), 0 }, - { VV_NAME("mouse_win", VAR_NUMBER), 0 }, - { VV_NAME("mouse_lnum", VAR_NUMBER), 0 }, - { VV_NAME("mouse_col", VAR_NUMBER), 0 }, - { VV_NAME("operator", VAR_STRING), VV_RO }, - { VV_NAME("searchforward", VAR_NUMBER), 0 }, - { VV_NAME("hlsearch", VAR_NUMBER), 0 }, - { VV_NAME("oldfiles", VAR_LIST), 0 }, - { VV_NAME("windowid", VAR_NUMBER), VV_RO }, - { VV_NAME("progpath", VAR_STRING), VV_RO }, - { VV_NAME("command_output", VAR_STRING), 0 }, - { VV_NAME("completed_item", VAR_DICT), VV_RO }, - { VV_NAME("option_new", VAR_STRING), VV_RO }, - { VV_NAME("option_old", VAR_STRING), VV_RO }, - { VV_NAME("option_type", VAR_STRING), VV_RO }, - { VV_NAME("errors", VAR_LIST), 0 }, - { VV_NAME("msgpack_types", VAR_DICT), VV_RO }, - { VV_NAME("event", VAR_DICT), VV_RO }, - { VV_NAME("false", VAR_SPECIAL), VV_RO }, - { VV_NAME("true", VAR_SPECIAL), VV_RO }, - { VV_NAME("null", VAR_SPECIAL), VV_RO }, + // VV_ tails differing from upcased string literals: + // VV_CC_FROM "charconvert_from" + // VV_CC_TO "charconvert_to" + // VV_SEND_SERVER "servername" + // VV_REG "register" + // VV_OP "operator" + VV(VV_COUNT, "count", VAR_NUMBER, VV_COMPAT+VV_RO), + VV(VV_COUNT1, "count1", VAR_NUMBER, VV_RO), + VV(VV_PREVCOUNT, "prevcount", VAR_NUMBER, VV_RO), + VV(VV_ERRMSG, "errmsg", VAR_STRING, VV_COMPAT), + VV(VV_WARNINGMSG, "warningmsg", VAR_STRING, 0), + VV(VV_STATUSMSG, "statusmsg", VAR_STRING, 0), + VV(VV_SHELL_ERROR, "shell_error", VAR_NUMBER, VV_COMPAT+VV_RO), + VV(VV_THIS_SESSION, "this_session", VAR_STRING, VV_COMPAT), + VV(VV_VERSION, "version", VAR_NUMBER, VV_COMPAT+VV_RO), + VV(VV_LNUM, "lnum", VAR_NUMBER, VV_RO_SBX), + VV(VV_TERMRESPONSE, "termresponse", VAR_STRING, VV_RO), + VV(VV_FNAME, "fname", VAR_STRING, VV_RO), + VV(VV_LANG, "lang", VAR_STRING, VV_RO), + VV(VV_LC_TIME, "lc_time", VAR_STRING, VV_RO), + VV(VV_CTYPE, "ctype", VAR_STRING, VV_RO), + VV(VV_CC_FROM, "charconvert_from", VAR_STRING, VV_RO), + VV(VV_CC_TO, "charconvert_to", VAR_STRING, VV_RO), + VV(VV_FNAME_IN, "fname_in", VAR_STRING, VV_RO), + VV(VV_FNAME_OUT, "fname_out", VAR_STRING, VV_RO), + VV(VV_FNAME_NEW, "fname_new", VAR_STRING, VV_RO), + VV(VV_FNAME_DIFF, "fname_diff", VAR_STRING, VV_RO), + VV(VV_CMDARG, "cmdarg", VAR_STRING, VV_RO), + VV(VV_FOLDSTART, "foldstart", VAR_NUMBER, VV_RO_SBX), + VV(VV_FOLDEND, "foldend", VAR_NUMBER, VV_RO_SBX), + VV(VV_FOLDDASHES, "folddashes", VAR_STRING, VV_RO_SBX), + VV(VV_FOLDLEVEL, "foldlevel", VAR_NUMBER, VV_RO_SBX), + VV(VV_PROGNAME, "progname", VAR_STRING, VV_RO), + VV(VV_SEND_SERVER, "servername", VAR_STRING, VV_RO), + VV(VV_DYING, "dying", VAR_NUMBER, VV_RO), + VV(VV_EXCEPTION, "exception", VAR_STRING, VV_RO), + VV(VV_THROWPOINT, "throwpoint", VAR_STRING, VV_RO), + VV(VV_REG, "register", VAR_STRING, VV_RO), + VV(VV_CMDBANG, "cmdbang", VAR_NUMBER, VV_RO), + VV(VV_INSERTMODE, "insertmode", VAR_STRING, VV_RO), + VV(VV_VAL, "val", VAR_UNKNOWN, VV_RO), + VV(VV_KEY, "key", VAR_UNKNOWN, VV_RO), + VV(VV_PROFILING, "profiling", VAR_NUMBER, VV_RO), + VV(VV_FCS_REASON, "fcs_reason", VAR_STRING, VV_RO), + VV(VV_FCS_CHOICE, "fcs_choice", VAR_STRING, 0), + VV(VV_BEVAL_BUFNR, "beval_bufnr", VAR_NUMBER, VV_RO), + VV(VV_BEVAL_WINNR, "beval_winnr", VAR_NUMBER, VV_RO), + VV(VV_BEVAL_LNUM, "beval_lnum", VAR_NUMBER, VV_RO), + VV(VV_BEVAL_COL, "beval_col", VAR_NUMBER, VV_RO), + VV(VV_BEVAL_TEXT, "beval_text", VAR_STRING, VV_RO), + VV(VV_SCROLLSTART, "scrollstart", VAR_STRING, 0), + VV(VV_SWAPNAME, "swapname", VAR_STRING, VV_RO), + VV(VV_SWAPCHOICE, "swapchoice", VAR_STRING, 0), + VV(VV_SWAPCOMMAND, "swapcommand", VAR_STRING, VV_RO), + VV(VV_CHAR, "char", VAR_STRING, 0), + VV(VV_MOUSE_WIN, "mouse_win", VAR_NUMBER, 0), + VV(VV_MOUSE_LNUM, "mouse_lnum", VAR_NUMBER, 0), + VV(VV_MOUSE_COL, "mouse_col", VAR_NUMBER, 0), + VV(VV_OP, "operator", VAR_STRING, VV_RO), + VV(VV_SEARCHFORWARD, "searchforward", VAR_NUMBER, 0), + VV(VV_HLSEARCH, "hlsearch", VAR_NUMBER, 0), + VV(VV_OLDFILES, "oldfiles", VAR_LIST, 0), + VV(VV_WINDOWID, "windowid", VAR_NUMBER, VV_RO), + VV(VV_PROGPATH, "progpath", VAR_STRING, VV_RO), + VV(VV_COMMAND_OUTPUT, "command_output", VAR_STRING, 0), + VV(VV_COMPLETED_ITEM, "completed_item", VAR_DICT, VV_RO), + VV(VV_OPTION_NEW, "option_new", VAR_STRING, VV_RO), + VV(VV_OPTION_OLD, "option_old", VAR_STRING, VV_RO), + VV(VV_OPTION_TYPE, "option_type", VAR_STRING, VV_RO), + VV(VV_ERRORS, "errors", VAR_LIST, 0), + VV(VV_MSGPACK_TYPES, "msgpack_types", VAR_DICT, VV_RO), + VV(VV_EVENT, "event", VAR_DICT, VV_RO), + VV(VV_FALSE, "false", VAR_SPECIAL, VV_RO), + VV(VV_TRUE, "true", VAR_SPECIAL, VV_RO), + VV(VV_NULL, "null", VAR_SPECIAL, VV_RO), }; +#undef VV /* shorthand */ #define vv_type vv_di.di_tv.v_type diff --git a/src/nvim/eval.h b/src/nvim/eval.h index 9d45b780a9..f81eb5d063 100644 --- a/src/nvim/eval.h +++ b/src/nvim/eval.h @@ -52,8 +52,6 @@ EXTERN ufunc_T dumuf; #define HI2UF(hi) HIKEY2UF((hi)->hi_key) /// Defines for Vim variables -/// -/// Order must match order in vimvars[] table in eval.c. typedef enum { VV_COUNT, VV_COUNT1, From 54cc6d8025f0a9e111b4e804e34dd2168b28c064 Mon Sep 17 00:00:00 2001 From: ZyX Date: Wed, 2 Mar 2016 13:30:39 +0300 Subject: [PATCH 49/82] eval: Remove assert expression GCC on travis thinks that 1. It is not constant. 2. Left-hand operand of comma has no effect (-Werror=unused-variable). --- src/nvim/eval.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/nvim/eval.c b/src/nvim/eval.c index 8fcb570525..9d1baec33f 100644 --- a/src/nvim/eval.c +++ b/src/nvim/eval.c @@ -289,12 +289,7 @@ typedef enum { .di_tv = { .v_type = type }, \ .di_flags = 0, \ }, \ - .vv_filler = {( \ - STATIC_ASSERT_EXPR( \ - (sizeof(name) - 1 <= sizeof(vimvars[0].vv_filler)), \ - "Vim variable name is too long, adjust vv_filler size"), \ - 0 \ - )}, \ + .vv_filler = { 0 }, \ .vv_flags = flags, \ } From 4ff5d6e41c3217bb3bb081743ac8b33667322137 Mon Sep 17 00:00:00 2001 From: ZyX Date: Mon, 7 Mar 2016 06:36:02 +0300 Subject: [PATCH 50/82] eval/decode: Also use VAR_UNLOCKED in old code --- src/nvim/eval/decode.c | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/nvim/eval/decode.c b/src/nvim/eval/decode.c index 4955a4f5a4..65c4359c49 100644 --- a/src/nvim/eval/decode.c +++ b/src/nvim/eval/decode.c @@ -54,7 +54,7 @@ static inline void create_special_dict(typval_T *const rettv, dict_T *const dict = dict_alloc(); dictitem_T *const type_di = dictitem_alloc((char_u *) "_TYPE"); type_di->di_tv.v_type = VAR_LIST; - type_di->di_tv.v_lock = 0; + type_di->di_tv.v_lock = VAR_UNLOCKED; type_di->di_tv.vval.v_list = (list_T *) eval_msgpack_type_lists[type]; type_di->di_tv.vval.v_list->lv_refcount++; dict_add(dict, type_di); @@ -548,7 +548,7 @@ json_decode_string_cycle_start: list->lv_refcount++; create_special_dict(&obj, kMPString, ((typval_T) { .v_type = VAR_LIST, - .v_lock = 0, + .v_lock = VAR_UNLOCKED, .vval = { .v_list = list }, })); if (encode_list_write((void *) list, str, (size_t) (str_end - str)) @@ -760,7 +760,7 @@ int msgpack_to_vim(const msgpack_object mobj, typval_T *const rettv) if (mobj.via.u64 <= VARNUMBER_MAX) { *rettv = (typval_T) { .v_type = VAR_NUMBER, - .v_lock = 0, + .v_lock = VAR_UNLOCKED, .vval = { .v_number = (varnumber_T) mobj.via.u64 }, }; } else { @@ -768,7 +768,7 @@ int msgpack_to_vim(const msgpack_object mobj, typval_T *const rettv) list->lv_refcount++; create_special_dict(rettv, kMPInteger, ((typval_T) { .v_type = VAR_LIST, - .v_lock = 0, + .v_lock = VAR_UNLOCKED, .vval = { .v_list = list }, })); uint64_t n = mobj.via.u64; @@ -783,7 +783,7 @@ int msgpack_to_vim(const msgpack_object mobj, typval_T *const rettv) if (mobj.via.i64 >= VARNUMBER_MIN) { *rettv = (typval_T) { .v_type = VAR_NUMBER, - .v_lock = 0, + .v_lock = VAR_UNLOCKED, .vval = { .v_number = (varnumber_T) mobj.via.i64 }, }; } else { @@ -791,7 +791,7 @@ int msgpack_to_vim(const msgpack_object mobj, typval_T *const rettv) list->lv_refcount++; create_special_dict(rettv, kMPInteger, ((typval_T) { .v_type = VAR_LIST, - .v_lock = 0, + .v_lock = VAR_UNLOCKED, .vval = { .v_list = list }, })); uint64_t n = -((uint64_t) mobj.via.i64); @@ -805,7 +805,7 @@ int msgpack_to_vim(const msgpack_object mobj, typval_T *const rettv) case MSGPACK_OBJECT_FLOAT: { *rettv = (typval_T) { .v_type = VAR_FLOAT, - .v_lock = 0, + .v_lock = VAR_UNLOCKED, .vval = { .v_float = mobj.via.f64 }, }; break; @@ -815,7 +815,7 @@ int msgpack_to_vim(const msgpack_object mobj, typval_T *const rettv) list->lv_refcount++; create_special_dict(rettv, kMPString, ((typval_T) { .v_type = VAR_LIST, - .v_lock = 0, + .v_lock = VAR_UNLOCKED, .vval = { .v_list = list }, })); if (encode_list_write((void *) list, mobj.via.str.ptr, mobj.via.str.size) @@ -828,7 +828,7 @@ int msgpack_to_vim(const msgpack_object mobj, typval_T *const rettv) if (memchr(mobj.via.bin.ptr, NUL, mobj.via.bin.size) == NULL) { *rettv = (typval_T) { .v_type = VAR_STRING, - .v_lock = 0, + .v_lock = VAR_UNLOCKED, .vval = { .v_string = xmemdupz(mobj.via.bin.ptr, mobj.via.bin.size) }, }; break; @@ -837,7 +837,7 @@ int msgpack_to_vim(const msgpack_object mobj, typval_T *const rettv) list->lv_refcount++; create_special_dict(rettv, kMPBinary, ((typval_T) { .v_type = VAR_LIST, - .v_lock = 0, + .v_lock = VAR_UNLOCKED, .vval = { .v_list = list }, })); if (encode_list_write((void *) list, mobj.via.bin.ptr, mobj.via.bin.size) @@ -851,7 +851,7 @@ int msgpack_to_vim(const msgpack_object mobj, typval_T *const rettv) list->lv_refcount++; *rettv = (typval_T) { .v_type = VAR_LIST, - .v_lock = 0, + .v_lock = VAR_UNLOCKED, .vval = { .v_list = list }, }; for (size_t i = 0; i < mobj.via.array.size; i++) { @@ -877,7 +877,7 @@ int msgpack_to_vim(const msgpack_object mobj, typval_T *const rettv) dict->dv_refcount++; *rettv = (typval_T) { .v_type = VAR_DICT, - .v_lock = 0, + .v_lock = VAR_UNLOCKED, .vval = { .v_dict = dict }, }; for (size_t i = 0; i < mobj.via.map.size; i++) { @@ -902,7 +902,7 @@ msgpack_to_vim_generic_map: {} list->lv_refcount++; create_special_dict(rettv, kMPMap, ((typval_T) { .v_type = VAR_LIST, - .v_lock = 0, + .v_lock = VAR_UNLOCKED, .vval = { .v_list = list }, })); for (size_t i = 0; i < mobj.via.map.size; i++) { @@ -931,7 +931,7 @@ msgpack_to_vim_generic_map: {} list_append_list(list, ext_val_list); create_special_dict(rettv, kMPExt, ((typval_T) { .v_type = VAR_LIST, - .v_lock = 0, + .v_lock = VAR_UNLOCKED, .vval = { .v_list = list }, })); if (encode_list_write((void *) ext_val_list, mobj.via.ext.ptr, From 1fc84ae2cd07f10f769ef966dd92b18ca8552748 Mon Sep 17 00:00:00 2001 From: ZyX Date: Mon, 7 Mar 2016 06:39:12 +0300 Subject: [PATCH 51/82] eval/decode: Record that `obj` may be freed --- src/nvim/eval/decode.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/nvim/eval/decode.c b/src/nvim/eval/decode.c index 65c4359c49..d6426ee643 100644 --- a/src/nvim/eval/decode.c +++ b/src/nvim/eval/decode.c @@ -73,7 +73,8 @@ static inline void create_special_dict(typval_T *const rettv, /// Helper function used for working with stack vectors used by JSON decoder /// -/// @param[in] obj New object. +/// @param[in,out] obj New object. Will either be put into the stack (and, +/// probably, also inside container) or freed. /// @param[out] stack Object stack. /// @param[out] container_stack Container objects stack. /// @param[in,out] pp Position in string which is currently being parsed. Used From 4a29995fe74ed95c641ef40c68d8a4223e90cccf Mon Sep 17 00:00:00 2001 From: ZyX Date: Mon, 7 Mar 2016 06:41:00 +0300 Subject: [PATCH 52/82] eval/decode: Rename brackets in error messages U+007D is officially RIGHT CURLY BRACKET. U+005D is officially RIGHT SQUARE BRACKET. --- src/nvim/eval/decode.c | 4 ++-- test/functional/eval/json_functions_spec.lua | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/nvim/eval/decode.c b/src/nvim/eval/decode.c index d6426ee643..fc6e912c20 100644 --- a/src/nvim/eval/decode.c +++ b/src/nvim/eval/decode.c @@ -249,10 +249,10 @@ json_decode_string_cycle_start: } ContainerStackItem last_container = kv_last(container_stack); if (*p == '}' && last_container.container.v_type != VAR_DICT) { - EMSG2(_("E474: Closing list with figure brace: %s"), p); + EMSG2(_("E474: Closing list with curly bracket: %s"), p); goto json_decode_string_fail; } else if (*p == ']' && last_container.container.v_type != VAR_LIST) { - EMSG2(_("E474: Closing dictionary with bracket: %s"), p); + EMSG2(_("E474: Closing dictionary with square bracket: %s"), p); goto json_decode_string_fail; } else if (didcomma) { EMSG2(_("E474: Trailing comma: %s"), p); diff --git a/test/functional/eval/json_functions_spec.lua b/test/functional/eval/json_functions_spec.lua index d6286611d2..131b177622 100644 --- a/test/functional/eval/json_functions_spec.lua +++ b/test/functional/eval/json_functions_spec.lua @@ -198,9 +198,9 @@ describe('json_decode() function', function() it('fails to parse containers which are closed by different brackets', function() - eq('Vim(call):E474: Closing dictionary with bracket: ]', + eq('Vim(call):E474: Closing dictionary with square bracket: ]', exc_exec('call json_decode("{]")')) - eq('Vim(call):E474: Closing list with figure brace: }', + eq('Vim(call):E474: Closing list with curly bracket: }', exc_exec('call json_decode("[}")')) end) From 4eb5d05f018bc568580c85f17ddb304fcec364ca Mon Sep 17 00:00:00 2001 From: ZyX Date: Mon, 7 Mar 2016 07:10:38 +0300 Subject: [PATCH 53/82] eval/decode: Avoid overflow when parsing incomplete null/true/false Note: second test does not crash or produce asan errors, even though it should. --- src/nvim/eval/decode.c | 6 ++-- test/unit/eval/decode_spec.lua | 65 ++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 3 deletions(-) create mode 100644 test/unit/eval/decode_spec.lua diff --git a/src/nvim/eval/decode.c b/src/nvim/eval/decode.c index fc6e912c20..35e8421716 100644 --- a/src/nvim/eval/decode.c +++ b/src/nvim/eval/decode.c @@ -334,7 +334,7 @@ json_decode_string_cycle_start: continue; } case 'n': { - if (strncmp(p + 1, "ull", 3) != 0) { + if ((p + 3) >= e || strncmp(p + 1, "ull", 3) != 0) { EMSG2(_("E474: Expected null: %s"), p); goto json_decode_string_fail; } @@ -347,7 +347,7 @@ json_decode_string_cycle_start: break; } case 't': { - if (strncmp(p + 1, "rue", 3) != 0) { + if ((p + 3) >= e || strncmp(p + 1, "rue", 3) != 0) { EMSG2(_("E474: Expected true: %s"), p); goto json_decode_string_fail; } @@ -360,7 +360,7 @@ json_decode_string_cycle_start: break; } case 'f': { - if (strncmp(p + 1, "alse", 4) != 0) { + if ((p + 4) >= e || strncmp(p + 1, "alse", 4) != 0) { EMSG2(_("E474: Expected false: %s"), p); goto json_decode_string_fail; } diff --git a/test/unit/eval/decode_spec.lua b/test/unit/eval/decode_spec.lua new file mode 100644 index 0000000000..6ae000a00a --- /dev/null +++ b/test/unit/eval/decode_spec.lua @@ -0,0 +1,65 @@ +local ffi = require('ffi') +local helpers = require('test.unit.helpers') +local eval_helpers = require('test.unit.eval.helpers') + +local cimport = helpers.cimport +local to_cstr = helpers.to_cstr +local eq = helpers.eq + +local list = eval_helpers.list +local lst2tbl = eval_helpers.lst2tbl +local type_key = eval_helpers.type_key +local list_type = eval_helpers.list_type +local null_string = eval_helpers.null_string + +local decode = cimport('./src/nvim/eval/decode.h', './src/nvim/eval_defs.h', + './src/nvim/globals.h', './src/nvim/memory.h') + +describe('json_decode_string()', function() + after_each(function() + decode.emsg_silent = 0 + end) + + it('does not overflow when running with `n…`, `t…`, `f…`', function() + local rettv = ffi.new('typval_T') + decode.emsg_silent = 1 + rettv.v_type = decode.VAR_UNKNOWN + -- This will not crash, but if `len` argument will be ignored it will parse + -- `null` as `null` and if not it will parse `null` as `n`. + eq(0, decode.json_decode_string('null', 1, rettv)) + eq(decode.VAR_UNKNOWN, rettv.v_type) + eq(0, decode.json_decode_string('true', 1, rettv)) + eq(decode.VAR_UNKNOWN, rettv.v_type) + eq(0, decode.json_decode_string('false', 1, rettv)) + eq(decode.VAR_UNKNOWN, rettv.v_type) + eq(0, decode.json_decode_string('null', 2, rettv)) + eq(decode.VAR_UNKNOWN, rettv.v_type) + eq(0, decode.json_decode_string('true', 2, rettv)) + eq(decode.VAR_UNKNOWN, rettv.v_type) + eq(0, decode.json_decode_string('false', 2, rettv)) + eq(decode.VAR_UNKNOWN, rettv.v_type) + eq(0, decode.json_decode_string('null', 3, rettv)) + eq(decode.VAR_UNKNOWN, rettv.v_type) + eq(0, decode.json_decode_string('true', 3, rettv)) + eq(decode.VAR_UNKNOWN, rettv.v_type) + eq(0, decode.json_decode_string('false', 3, rettv)) + eq(decode.VAR_UNKNOWN, rettv.v_type) + eq(0, decode.json_decode_string('false', 4, rettv)) + eq(decode.VAR_UNKNOWN, rettv.v_type) + end) + + it('does not overflow and crash when running with `n`, `t`, `f`', function() + local rettv = ffi.new('typval_T') + decode.emsg_silent = 1 + rettv.v_type = decode.VAR_UNKNOWN + local char = function(c) + return ffi.gc(decode.xmemdup(c, 1), decode.xfree) + end + eq(0, decode.json_decode_string(char('n'), 1, rettv)) + eq(decode.VAR_UNKNOWN, rettv.v_type) + eq(0, decode.json_decode_string(char('t'), 1, rettv)) + eq(decode.VAR_UNKNOWN, rettv.v_type) + eq(0, decode.json_decode_string(char('f'), 1, rettv)) + eq(decode.VAR_UNKNOWN, rettv.v_type) + end) +end) From b725f6b4287d800f22bce32f32022ad07aa2610e Mon Sep 17 00:00:00 2001 From: ZyX Date: Mon, 7 Mar 2016 07:18:29 +0300 Subject: [PATCH 54/82] functests: Make sure that json functions are tested with C messages --- test/functional/eval/json_functions_spec.lua | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/functional/eval/json_functions_spec.lua b/test/functional/eval/json_functions_spec.lua index 131b177622..59dbb804e4 100644 --- a/test/functional/eval/json_functions_spec.lua +++ b/test/functional/eval/json_functions_spec.lua @@ -10,6 +10,7 @@ local exc_exec = helpers.exc_exec describe('json_decode() function', function() local restart = function(cmd) clear(cmd) + execute('language C') execute([[ function Eq(exp, act) let act = a:act @@ -449,7 +450,10 @@ describe('json_decode() function', function() end) describe('json_encode() function', function() - before_each(clear) + before_each(function() + clear() + execute('language C') + end) it('dumps strings', function() eq('"Test"', funcs.json_encode('Test')) From 394830631f130ad646f23358bf7863e7a37c6d78 Mon Sep 17 00:00:00 2001 From: ZyX Date: Mon, 7 Mar 2016 07:27:14 +0300 Subject: [PATCH 55/82] eval/decode: Make sure that U+00C3 is parsed correctly --- src/nvim/eval/decode.c | 5 ++++- test/functional/eval/json_functions_spec.lua | 4 ++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/nvim/eval/decode.c b/src/nvim/eval/decode.c index 35e8421716..ce2723147d 100644 --- a/src/nvim/eval/decode.c +++ b/src/nvim/eval/decode.c @@ -431,7 +431,10 @@ json_decode_string_cycle_start: // and thus cannot possibly be equal to *p. But utf_ptr2char({0xFF, // 0}) will return 0xFF, even though 0xFF cannot start any UTF-8 // code point at all. - if (ch >= 0x80 && p_byte == ch) { + // + // The only exception is U+00C3 which is represented as 0xC3 0x83. + if (ch >= 0x80 && p_byte == ch && !( + ch == 0xC3 && p + 1 < e && (uint8_t) p[1] == 0x83)) { EMSG2(_("E474: Only UTF-8 strings allowed: %s"), p); goto json_decode_string_fail; } else if (ch > 0x10FFFF) { diff --git a/test/functional/eval/json_functions_spec.lua b/test/functional/eval/json_functions_spec.lua index 59dbb804e4..7ec3882a58 100644 --- a/test/functional/eval/json_functions_spec.lua +++ b/test/functional/eval/json_functions_spec.lua @@ -447,6 +447,10 @@ describe('json_decode() function', function() eq('\xAB', funcs.json_decode('"\\u00AB"')) sp_decode_eq({_TYPE='string', _VAL={'\n\xAB\n'}}, '"\\u0000\\u00AB\\u0000"') end) + + it('parses U+00C3 correctly', function() + eq('\xC3\x83', funcs.json_decode('"\xC3\x83"')) + end) end) describe('json_encode() function', function() From 224d7df6309319cfa1f98aad3aa93c5b63ee4145 Mon Sep 17 00:00:00 2001 From: ZyX Date: Mon, 7 Mar 2016 07:37:21 +0300 Subject: [PATCH 56/82] eval/decode: Make sure that blank input does not crash Neovim --- src/nvim/eval/decode.c | 11 +++++++++-- test/functional/eval/json_functions_spec.lua | 13 +++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/src/nvim/eval/decode.c b/src/nvim/eval/decode.c index ce2723147d..4ce47a5e19 100644 --- a/src/nvim/eval/decode.c +++ b/src/nvim/eval/decode.c @@ -223,6 +223,15 @@ int json_decode_string(const char *const buf, const size_t len, typval_T *const rettv) FUNC_ATTR_NONNULL_ALL FUNC_ATTR_WARN_UNUSED_RESULT { + const char *p = buf; + const char *const e = buf + len; + while (p < e && (*p == ' ' || *p == '\t' || *p == '\n')) { + p++; + } + if (p == e) { + EMSG(_("E474: Attempt to decode a blank string")); + return FAIL; + } vimconv_T conv = { .vc_type = CONV_NONE }; convert_setup(&conv, (char_u *) "utf-8", p_enc); conv.vc_fail = true; @@ -232,11 +241,9 @@ int json_decode_string(const char *const buf, const size_t len, ContainerStack container_stack; kv_init(container_stack); rettv->v_type = VAR_UNKNOWN; - const char *const e = buf + len; bool didcomma = false; bool didcolon = false; bool next_map_special = false; - const char *p = buf; for (; p < e; p++) { json_decode_string_cycle_start: assert(*p == '{' || next_map_special == false); diff --git a/test/functional/eval/json_functions_spec.lua b/test/functional/eval/json_functions_spec.lua index 7ec3882a58..aa36b62757 100644 --- a/test/functional/eval/json_functions_spec.lua +++ b/test/functional/eval/json_functions_spec.lua @@ -451,6 +451,19 @@ describe('json_decode() function', function() it('parses U+00C3 correctly', function() eq('\xC3\x83', funcs.json_decode('"\xC3\x83"')) end) + + it('fails to parse empty string', function() + eq('Vim(call):E474: Attempt to decode a blank string', + exc_exec('call json_decode("")')) + eq('Vim(call):E474: Attempt to decode a blank string', + exc_exec('call json_decode(" ")')) + eq('Vim(call):E474: Attempt to decode a blank string', + exc_exec('call json_decode("\\t")')) + eq('Vim(call):E474: Attempt to decode a blank string', + exc_exec('call json_decode("\\n")')) + eq('Vim(call):E474: Attempt to decode a blank string', + exc_exec('call json_decode(" \\t\\n \\n\\t\\t \\n\\t\\n \\n \\t\\n\\t ")')) + end) end) describe('json_encode() function', function() From 52c6cc21899d0d5bf0dffc2cee849063e176e931 Mon Sep 17 00:00:00 2001 From: ZyX Date: Mon, 7 Mar 2016 07:50:17 +0300 Subject: [PATCH 57/82] eval/decode: Make sure that parsing strings does not overflow --- src/nvim/eval/decode.c | 2 +- test/unit/eval/decode_spec.lua | 29 ++++++++++++++++++++++------- 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/src/nvim/eval/decode.c b/src/nvim/eval/decode.c index 4ce47a5e19..8bd7f5d940 100644 --- a/src/nvim/eval/decode.c +++ b/src/nvim/eval/decode.c @@ -455,7 +455,7 @@ json_decode_string_cycle_start: p += ch_len; } } - if (*p != '"') { + if (p == e || *p != '"') { EMSG2(_("E474: Expected string end: %s"), buf); goto json_decode_string_fail; } diff --git a/test/unit/eval/decode_spec.lua b/test/unit/eval/decode_spec.lua index 6ae000a00a..2bf7f0b987 100644 --- a/test/unit/eval/decode_spec.lua +++ b/test/unit/eval/decode_spec.lua @@ -18,12 +18,16 @@ local decode = cimport('./src/nvim/eval/decode.h', './src/nvim/eval_defs.h', describe('json_decode_string()', function() after_each(function() decode.emsg_silent = 0 + decode.trylevel = 0 end) + local char = function(c) + return ffi.gc(decode.xmemdup(c, 1), decode.xfree) + end + it('does not overflow when running with `n…`, `t…`, `f…`', function() - local rettv = ffi.new('typval_T') + local rettv = ffi.new('typval_T', {v_type=decode.VAR_UNKNOWN}) decode.emsg_silent = 1 - rettv.v_type = decode.VAR_UNKNOWN -- This will not crash, but if `len` argument will be ignored it will parse -- `null` as `null` and if not it will parse `null` as `n`. eq(0, decode.json_decode_string('null', 1, rettv)) @@ -49,12 +53,8 @@ describe('json_decode_string()', function() end) it('does not overflow and crash when running with `n`, `t`, `f`', function() - local rettv = ffi.new('typval_T') + local rettv = ffi.new('typval_T', {v_type=decode.VAR_UNKNOWN}) decode.emsg_silent = 1 - rettv.v_type = decode.VAR_UNKNOWN - local char = function(c) - return ffi.gc(decode.xmemdup(c, 1), decode.xfree) - end eq(0, decode.json_decode_string(char('n'), 1, rettv)) eq(decode.VAR_UNKNOWN, rettv.v_type) eq(0, decode.json_decode_string(char('t'), 1, rettv)) @@ -62,4 +62,19 @@ describe('json_decode_string()', function() eq(0, decode.json_decode_string(char('f'), 1, rettv)) eq(decode.VAR_UNKNOWN, rettv.v_type) end) + + it('does not overflow when running with `"…`', function() + local rettv = ffi.new('typval_T', {v_type=decode.VAR_UNKNOWN}) + decode.emsg_silent = 1 + eq(0, decode.json_decode_string('"t"', 2, rettv)) + eq(decode.VAR_UNKNOWN, rettv.v_type) + eq(0, decode.json_decode_string('""', 1, rettv)) + eq(decode.VAR_UNKNOWN, rettv.v_type) + end) + + it('does not overflow and crash when running with `"`', function() + local rettv = ffi.new('typval_T', {v_type=decode.VAR_UNKNOWN}) + eq(0, decode.json_decode_string(char('"'), 1, rettv)) + eq(decode.VAR_UNKNOWN, rettv.v_type) + end) end) From eb806c96205ff776d9cd5df82da72c14e030f6d6 Mon Sep 17 00:00:00 2001 From: ZyX Date: Mon, 7 Mar 2016 08:54:39 +0300 Subject: [PATCH 58/82] eval/decode: Make sure that error messages do not cause overflow --- src/nvim/eval/decode.c | 86 +++++++++++--------- src/nvim/message.c | 10 +-- src/nvim/message.h | 12 +++ test/functional/eval/json_functions_spec.lua | 6 ++ test/unit/eval/decode_spec.lua | 78 +++++++++++++++--- 5 files changed, 134 insertions(+), 58 deletions(-) diff --git a/src/nvim/eval/decode.c b/src/nvim/eval/decode.c index 8bd7f5d940..266da86b74 100644 --- a/src/nvim/eval/decode.c +++ b/src/nvim/eval/decode.c @@ -215,16 +215,18 @@ static inline int json_decoder_pop(ValuesStackItem obj, /// Convert JSON string into VimL object /// /// @param[in] buf String to convert. UTF-8 encoding is assumed. -/// @param[in] len Length of the string. +/// @param[in] buf_len Length of the string. /// @param[out] rettv Location where to save results. /// /// @return OK in case of success, FAIL otherwise. -int json_decode_string(const char *const buf, const size_t len, +int json_decode_string(const char *const buf, const size_t buf_len, typval_T *const rettv) FUNC_ATTR_NONNULL_ALL FUNC_ATTR_WARN_UNUSED_RESULT { +#define LENP(p, e) \ + ((int) ((e) - (p))), (p) const char *p = buf; - const char *const e = buf + len; + const char *const e = buf + buf_len; while (p < e && (*p == ' ' || *p == '\t' || *p == '\n')) { p++; } @@ -251,25 +253,26 @@ json_decode_string_cycle_start: case '}': case ']': { if (kv_size(container_stack) == 0) { - EMSG2(_("E474: No container to close: %s"), p); + emsgf(_("E474: No container to close: %.*s"), LENP(p, e)); goto json_decode_string_fail; } ContainerStackItem last_container = kv_last(container_stack); if (*p == '}' && last_container.container.v_type != VAR_DICT) { - EMSG2(_("E474: Closing list with curly bracket: %s"), p); + emsgf(_("E474: Closing list with curly bracket: %.*s"), LENP(p, e)); goto json_decode_string_fail; } else if (*p == ']' && last_container.container.v_type != VAR_LIST) { - EMSG2(_("E474: Closing dictionary with square bracket: %s"), p); + emsgf(_("E474: Closing dictionary with square bracket: %.*s"), + LENP(p, e)); goto json_decode_string_fail; } else if (didcomma) { - EMSG2(_("E474: Trailing comma: %s"), p); + emsgf(_("E474: Trailing comma: %.*s"), LENP(p, e)); goto json_decode_string_fail; } else if (didcolon) { - EMSG2(_("E474: Expected value after colon: %s"), p); + emsgf(_("E474: Expected value after colon: %.*s"), LENP(p, e)); goto json_decode_string_fail; } else if (last_container.stack_index != kv_size(stack) - 1) { assert(last_container.stack_index < kv_size(stack) - 1); - EMSG2(_("E474: Expected value: %s"), p); + emsgf(_("E474: Expected value: %.*s"), LENP(p, e)); goto json_decode_string_fail; } if (kv_size(stack) == 1) { @@ -288,26 +291,26 @@ json_decode_string_cycle_start: } case ',': { if (kv_size(container_stack) == 0) { - EMSG2(_("E474: Comma not inside container: %s"), p); + emsgf(_("E474: Comma not inside container: %.*s"), LENP(p, e)); goto json_decode_string_fail; } ContainerStackItem last_container = kv_last(container_stack); if (didcomma) { - EMSG2(_("E474: Duplicate comma: %s"), p); + emsgf(_("E474: Duplicate comma: %.*s"), LENP(p, e)); goto json_decode_string_fail; } else if (didcolon) { - EMSG2(_("E474: Comma after colon: %s"), p); + emsgf(_("E474: Comma after colon: %.*s"), LENP(p, e)); goto json_decode_string_fail; } else if (last_container.container.v_type == VAR_DICT && last_container.stack_index != kv_size(stack) - 1) { - EMSG2(_("E474: Using comma in place of colon: %s"), p); + emsgf(_("E474: Using comma in place of colon: %.*s"), LENP(p, e)); goto json_decode_string_fail; } else if (last_container.special_val == NULL ? (last_container.container.v_type == VAR_DICT ? (DICT_LEN(last_container.container.vval.v_dict) == 0) : (last_container.container.vval.v_list->lv_len == 0)) : (last_container.special_val->lv_len == 0)) { - EMSG2(_("E474: Leading comma: %s"), p); + emsgf(_("E474: Leading comma: %.*s"), LENP(p, e)); goto json_decode_string_fail; } didcomma = true; @@ -315,21 +318,21 @@ json_decode_string_cycle_start: } case ':': { if (kv_size(container_stack) == 0) { - EMSG2(_("E474: Colon not inside container: %s"), p); + emsgf(_("E474: Colon not inside container: %.*s"), LENP(p, e)); goto json_decode_string_fail; } ContainerStackItem last_container = kv_last(container_stack); if (last_container.container.v_type != VAR_DICT) { - EMSG2(_("E474: Using colon not in dictionary: %s"), p); + emsgf(_("E474: Using colon not in dictionary: %.*s"), LENP(p, e)); goto json_decode_string_fail; } else if (last_container.stack_index != kv_size(stack) - 2) { - EMSG2(_("E474: Unexpected colon: %s"), p); + emsgf(_("E474: Unexpected colon: %.*s"), LENP(p, e)); goto json_decode_string_fail; } else if (didcomma) { - EMSG2(_("E474: Colon after comma: %s"), p); + emsgf(_("E474: Colon after comma: %.*s"), LENP(p, e)); goto json_decode_string_fail; } else if (didcolon) { - EMSG2(_("E474: Duplicate colon: %s"), p); + emsgf(_("E474: Duplicate colon: %.*s"), LENP(p, e)); goto json_decode_string_fail; } didcolon = true; @@ -342,7 +345,7 @@ json_decode_string_cycle_start: } case 'n': { if ((p + 3) >= e || strncmp(p + 1, "ull", 3) != 0) { - EMSG2(_("E474: Expected null: %s"), p); + emsgf(_("E474: Expected null: %.*s"), LENP(p, e)); goto json_decode_string_fail; } p += 3; @@ -355,7 +358,7 @@ json_decode_string_cycle_start: } case 't': { if ((p + 3) >= e || strncmp(p + 1, "rue", 3) != 0) { - EMSG2(_("E474: Expected true: %s"), p); + emsgf(_("E474: Expected true: %.*s"), LENP(p, e)); goto json_decode_string_fail; } p += 3; @@ -368,7 +371,7 @@ json_decode_string_cycle_start: } case 'f': { if ((p + 4) >= e || strncmp(p + 1, "alse", 4) != 0) { - EMSG2(_("E474: Expected false: %s"), p); + emsgf(_("E474: Expected false: %.*s"), LENP(p, e)); goto json_decode_string_fail; } p += 4; @@ -386,20 +389,22 @@ json_decode_string_cycle_start: if (*p == '\\') { p++; if (p == e) { - EMSG2(_("E474: Unfinished escape sequence: %s"), buf); + emsgf(_("E474: Unfinished escape sequence: %.*s"), + (int) buf_len, buf); goto json_decode_string_fail; } switch (*p) { case 'u': { if (p + 4 >= e) { - EMSG2(_("E474: Unfinished unicode escape sequence: %s"), buf); + emsgf(_("E474: Unfinished unicode escape sequence: %.*s"), + (int) buf_len, buf); goto json_decode_string_fail; } else if (!ascii_isxdigit(p[1]) || !ascii_isxdigit(p[2]) || !ascii_isxdigit(p[3]) || !ascii_isxdigit(p[4])) { - EMSG2(_("E474: Expected four hex digits after \\u: %s"), - p - 1); + emsgf(_("E474: Expected four hex digits after \\u: %.*s"), + LENP(p - 1, e)); goto json_decode_string_fail; } // One UTF-8 character below U+10000 can take up to 3 bytes, @@ -421,7 +426,7 @@ json_decode_string_cycle_start: break; } default: { - EMSG2(_("E474: Unknown escape sequence: %s"), p - 1); + emsgf(_("E474: Unknown escape sequence: %.*s"), LENP(p - 1, e)); goto json_decode_string_fail; } } @@ -429,8 +434,8 @@ json_decode_string_cycle_start: uint8_t p_byte = (uint8_t) *p; // unescaped = %x20-21 / %x23-5B / %x5D-10FFFF if (p_byte < 0x20) { - EMSG2(_("E474: ASCII control characters cannot be present " - "inside string: %s"), p); + emsgf(_("E474: ASCII control characters cannot be present " + "inside string: %.*s"), LENP(p, e)); goto json_decode_string_fail; } const int ch = utf_ptr2char((char_u *) p); @@ -442,11 +447,11 @@ json_decode_string_cycle_start: // The only exception is U+00C3 which is represented as 0xC3 0x83. if (ch >= 0x80 && p_byte == ch && !( ch == 0xC3 && p + 1 < e && (uint8_t) p[1] == 0x83)) { - EMSG2(_("E474: Only UTF-8 strings allowed: %s"), p); + emsgf(_("E474: Only UTF-8 strings allowed: %.*s"), LENP(p, e)); goto json_decode_string_fail; } else if (ch > 0x10FFFF) { - EMSG2(_("E474: Only UTF-8 code points up to U+10FFFF " - "are allowed to appear unescaped: %s"), p); + emsgf(_("E474: Only UTF-8 code points up to U+10FFFF " + "are allowed to appear unescaped: %.*s"), LENP(p, e)); goto json_decode_string_fail; } const size_t ch_len = (size_t) utf_char2len(ch); @@ -456,7 +461,7 @@ json_decode_string_cycle_start: } } if (p == e || *p != '"') { - EMSG2(_("E474: Expected string end: %s"), buf); + emsgf(_("E474: Expected string end: %.*s"), (int) buf_len, buf); goto json_decode_string_fail; } if (len == 0) { @@ -545,7 +550,8 @@ json_decode_string_cycle_start: char *const new_str = (char *) string_convert(&conv, (char_u *) str, &str_len); if (new_str == NULL) { - EMSG2(_("E474: Failed to convert string \"%s\" from UTF-8"), str); + emsgf(_("E474: Failed to convert string \"%.*s\" from UTF-8"), + (int) str_len, str); xfree(str); goto json_decode_string_fail; } @@ -619,13 +625,13 @@ json_decode_string_cycle_start: } } if (p == ints) { - EMSG2(_("E474: Missing number after minus sign: %s"), s); + emsgf(_("E474: Missing number after minus sign: %.*s"), LENP(s, e)); goto json_decode_string_fail; } else if (p == fracs) { - EMSG2(_("E474: Missing number after decimal dot: %s"), s); + emsgf(_("E474: Missing number after decimal dot: %.*s"), LENP(s, e)); goto json_decode_string_fail; } else if (p == exps) { - EMSG2(_("E474: Missing exponent: %s"), s); + emsgf(_("E474: Missing exponent: %.*s"), LENP(s, e)); goto json_decode_string_fail; } typval_T tv = { @@ -694,7 +700,7 @@ json_decode_string_cycle_start: break; } default: { - EMSG2(_("E474: Unidentified byte: %s"), p); + emsgf(_("E474: Unidentified byte: %.*s"), LENP(p, e)); goto json_decode_string_fail; } } @@ -714,13 +720,13 @@ json_decode_string_after_cycle: break; } default: { - EMSG2(_("E474: Trailing characters: %s"), p); + emsgf(_("E474: Trailing characters: %.*s"), LENP(p, e)); goto json_decode_string_fail; } } } if (kv_size(stack) > 1 || kv_size(container_stack)) { - EMSG2(_("E474: Unexpected end of input: %s"), buf); + emsgf(_("E474: Unexpected end of input: %.*s"), (int) buf_len, buf); goto json_decode_string_fail; } goto json_decode_string_ret; diff --git a/src/nvim/message.c b/src/nvim/message.c index c4207fbe9e..21fd97ed21 100644 --- a/src/nvim/message.c +++ b/src/nvim/message.c @@ -61,14 +61,8 @@ static int confirm_msg_used = FALSE; /* displaying confirm_msg */ static char_u *confirm_msg = NULL; /* ":confirm" message */ static char_u *confirm_msg_tail; /* tail of confirm_msg */ -struct msg_hist { - struct msg_hist *next; - char_u *msg; - int attr; -}; - -static struct msg_hist *first_msg_hist = NULL; -static struct msg_hist *last_msg_hist = NULL; +MessageHistoryEntry *first_msg_hist = NULL; +MessageHistoryEntry *last_msg_hist = NULL; static int msg_hist_len = 0; static FILE *verbose_fd = NULL; diff --git a/src/nvim/message.h b/src/nvim/message.h index 0cd0ede2c4..b69b7264b4 100644 --- a/src/nvim/message.h +++ b/src/nvim/message.h @@ -64,6 +64,18 @@ /// Like #MSG_PUTS_ATTR, but if middle part of long messages will be replaced #define MSG_PUTS_LONG_ATTR(s, a) msg_puts_long_attr((char_u *)(s), (a)) +/// Message history for `:messages` +typedef struct msg_hist { + struct msg_hist *next; ///< Next message. + char_u *msg; ///< Message text. + int attr; ///< Message highlighting. +} MessageHistoryEntry; + +/// First message +extern MessageHistoryEntry *first_msg_hist; +/// Last message +extern MessageHistoryEntry *last_msg_hist; + #ifdef INCLUDE_GENERATED_DECLARATIONS # include "message.h.generated.h" #endif diff --git a/test/functional/eval/json_functions_spec.lua b/test/functional/eval/json_functions_spec.lua index aa36b62757..58030fca72 100644 --- a/test/functional/eval/json_functions_spec.lua +++ b/test/functional/eval/json_functions_spec.lua @@ -448,6 +448,12 @@ describe('json_decode() function', function() sp_decode_eq({_TYPE='string', _VAL={'\n\xAB\n'}}, '"\\u0000\\u00AB\\u0000"') end) + it('fails to convert string to latin1 if it is impossible', function() + restart('set encoding=latin1') + eq('Vim(call):E474: Failed to convert string "ꯍ" from UTF-8', + exc_exec('call json_decode(\'"\\uABCD"\')')) + end) + it('parses U+00C3 correctly', function() eq('\xC3\x83', funcs.json_decode('"\xC3\x83"')) end) diff --git a/test/unit/eval/decode_spec.lua b/test/unit/eval/decode_spec.lua index 2bf7f0b987..57fc5f7b94 100644 --- a/test/unit/eval/decode_spec.lua +++ b/test/unit/eval/decode_spec.lua @@ -1,24 +1,28 @@ -local ffi = require('ffi') local helpers = require('test.unit.helpers') -local eval_helpers = require('test.unit.eval.helpers') local cimport = helpers.cimport local to_cstr = helpers.to_cstr local eq = helpers.eq - -local list = eval_helpers.list -local lst2tbl = eval_helpers.lst2tbl -local type_key = eval_helpers.type_key -local list_type = eval_helpers.list_type -local null_string = eval_helpers.null_string +local neq = helpers.neq +local ffi = helpers.ffi local decode = cimport('./src/nvim/eval/decode.h', './src/nvim/eval_defs.h', - './src/nvim/globals.h', './src/nvim/memory.h') + './src/nvim/globals.h', './src/nvim/memory.h', + './src/nvim/message.h') describe('json_decode_string()', function() + local saved_p_enc = nil + + before_each(function() + saved_p_enc = decode.p_enc + end) + after_each(function() decode.emsg_silent = 0 - decode.trylevel = 0 + decode.p_enc = saved_p_enc + while decode.delete_first_msg() == 1 do + -- Delete all messages + end end) local char = function(c) @@ -72,8 +76,62 @@ describe('json_decode_string()', function() eq(decode.VAR_UNKNOWN, rettv.v_type) end) + it('does not overflow in error messages', function() + local check_failure = function(s, len, msg) + local rettv = ffi.new('typval_T', {v_type=decode.VAR_UNKNOWN}) + eq(0, decode.json_decode_string(s, len, rettv)) + eq(decode.VAR_UNKNOWN, rettv.v_type) + neq(nil, decode.last_msg_hist) + eq(msg, ffi.string(decode.last_msg_hist.msg)) + end + local rettv = ffi.new('typval_T', {v_type=decode.VAR_UNKNOWN}) + check_failure(']test', 1, 'E474: No container to close: ]') + check_failure('[}test', 2, 'E474: Closing list with curly bracket: }') + check_failure('{]test', 2, + 'E474: Closing dictionary with square bracket: ]') + check_failure('[1,]test', 4, 'E474: Trailing comma: ]') + check_failure('{"1":}test', 6, 'E474: Expected value after colon: }') + check_failure('{"1"}test', 5, 'E474: Expected value: }') + check_failure(',test', 1, 'E474: Comma not inside container: ,') + check_failure('[1,,1]test', 6, 'E474: Duplicate comma: ,1]') + check_failure('{"1":,}test', 7, 'E474: Comma after colon: ,}') + check_failure('{"1",}test', 6, 'E474: Using comma in place of colon: ,}') + check_failure('{,}test', 3, 'E474: Leading comma: ,}') + check_failure('[,]test', 3, 'E474: Leading comma: ,]') + check_failure(':test', 1, 'E474: Colon not inside container: :') + check_failure('[:]test', 3, 'E474: Using colon not in dictionary: :]') + check_failure('{:}test', 3, 'E474: Unexpected colon: :}') + check_failure('{"1"::1}test', 8, 'E474: Duplicate colon: :1}') + check_failure('ntest', 1, 'E474: Expected null: n') + check_failure('ttest', 1, 'E474: Expected true: t') + check_failure('ftest', 1, 'E474: Expected false: f') + check_failure('"\\test', 2, 'E474: Unfinished escape sequence: "\\') + check_failure('"\\u"test', 4, + 'E474: Unfinished unicode escape sequence: "\\u"') + check_failure('"\\uXXXX"est', 8, + 'E474: Expected four hex digits after \\u: \\uXXXX"') + check_failure('"\\?"test', 4, 'E474: Unknown escape sequence: \\?"') + check_failure( + '"\t"test', 3, + 'E474: ASCII control characters cannot be present inside string: \t"') + check_failure('"\194"test', 3, 'E474: Only UTF-8 strings allowed: \194"') + check_failure('"\xFC\x90\x80\x80\x80\x80"test', 8, 'E474: Only UTF-8 code points up to U+10FFFF are allowed to appear unescaped: \xFC\x90\x80\x80\x80\x80"') + check_failure('"test', 1, 'E474: Expected string end: "') + decode.p_enc = to_cstr('latin1') + check_failure('"\\uABCD"test', 8, + 'E474: Failed to convert string "ꯍ" from UTF-8') + decode.p_enc = saved_p_enc + check_failure('-test', 1, 'E474: Missing number after minus sign: -') + check_failure('-1.test', 3, 'E474: Missing number after decimal dot: -1.') + check_failure('-1.0etest', 5, 'E474: Missing exponent: -1.0e') + check_failure('?test', 1, 'E474: Unidentified byte: ?') + check_failure('1?test', 2, 'E474: Trailing characters: ?') + check_failure('[1test', 2, 'E474: Unexpected end of input: [1') + end) + it('does not overflow and crash when running with `"`', function() local rettv = ffi.new('typval_T', {v_type=decode.VAR_UNKNOWN}) + decode.emsg_silent = 1 eq(0, decode.json_decode_string(char('"'), 1, rettv)) eq(decode.VAR_UNKNOWN, rettv.v_type) end) From 032ac502ff1378757d9ba56e5760d362570e48e4 Mon Sep 17 00:00:00 2001 From: ZyX Date: Mon, 7 Mar 2016 08:59:03 +0300 Subject: [PATCH 59/82] eval/decode: Do not loose high surrogates followed by high surrogates --- src/nvim/eval/decode.c | 1 + test/functional/eval/json_functions_spec.lua | 2 ++ 2 files changed, 3 insertions(+) diff --git a/src/nvim/eval/decode.c b/src/nvim/eval/decode.c index 266da86b74..604b758344 100644 --- a/src/nvim/eval/decode.c +++ b/src/nvim/eval/decode.c @@ -499,6 +499,7 @@ json_decode_string_cycle_start: hasnul = true; } if (SURROGATE_HI_START <= ch && ch <= SURROGATE_HI_END) { + PUT_FST_IN_PAIR(fst_in_pair, str_end); fst_in_pair = (int) ch; } else if (SURROGATE_LO_START <= ch && ch <= SURROGATE_LO_END && fst_in_pair != 0) { diff --git a/test/functional/eval/json_functions_spec.lua b/test/functional/eval/json_functions_spec.lua index 58030fca72..6379e7fed8 100644 --- a/test/functional/eval/json_functions_spec.lua +++ b/test/functional/eval/json_functions_spec.lua @@ -389,6 +389,8 @@ describe('json_decode() function', function() eq('\t\xED\xB0\x80', funcs.json_decode('"\\t\\uDC00"')) eq('\xED\xA0\x80¬', funcs.json_decode('"\\uD800\\u00AC"')) + + eq('\xED\xA0\x80\xED\xA0\x80', funcs.json_decode('"\\uD800\\uD800"')) end) local sp_decode_eq = function(expected, json) From 9c543f2e246469adec1daddf156f4bcabe30931a Mon Sep 17 00:00:00 2001 From: ZyX Date: Mon, 7 Mar 2016 09:09:39 +0300 Subject: [PATCH 60/82] eval/decode: Reject more numbers, accept 1e5 --- src/nvim/eval/decode.c | 30 ++++++++++---- test/functional/eval/json_functions_spec.lua | 43 +++++++++++--------- 2 files changed, 46 insertions(+), 27 deletions(-) diff --git a/src/nvim/eval/decode.c b/src/nvim/eval/decode.c index 604b758344..75c88e308b 100644 --- a/src/nvim/eval/decode.c +++ b/src/nvim/eval/decode.c @@ -608,11 +608,13 @@ json_decode_string_cycle_start: while (p < e && ascii_isdigit(*p)) { p++; } - if (p < e && *p == '.') { - p++; - fracs = p; - while (p < e && ascii_isdigit(*p)) { + if (p < e && p != ints && (*p == '.' || *p == 'e' || *p == 'E')) { + if (*p == '.') { p++; + fracs = p; + while (p < e && ascii_isdigit(*p)) { + p++; + } } if (p < e && (*p == 'e' || *p == 'E')) { p++; @@ -628,7 +630,7 @@ json_decode_string_cycle_start: if (p == ints) { emsgf(_("E474: Missing number after minus sign: %.*s"), LENP(s, e)); goto json_decode_string_fail; - } else if (p == fracs) { + } else if (p == fracs || exps == fracs + 1) { emsgf(_("E474: Missing number after decimal dot: %.*s"), LENP(s, e)); goto json_decode_string_fail; } else if (p == exps) { @@ -639,14 +641,26 @@ json_decode_string_cycle_start: .v_type = VAR_NUMBER, .v_lock = VAR_UNLOCKED, }; - if (fracs) { + const size_t exp_num_len = (size_t) (p - s); + if (fracs || exps) { // Convert floating-point number - (void) string2float(s, &tv.vval.v_float); + const size_t num_len = string2float(s, &tv.vval.v_float); + if (exp_num_len != num_len) { + emsgf(_("E685: internal error: while converting number \"%.*s\" " + "to float string2float consumed %zu bytes in place of %zu"), + (int) exp_num_len, s, num_len, exp_num_len); + } tv.v_type = VAR_FLOAT; } else { // Convert integer long nr; - vim_str2nr((char_u *) s, NULL, NULL, 0, &nr, NULL, (int) (p - s)); + int num_len; + vim_str2nr((char_u *) s, NULL, &num_len, 0, &nr, NULL, (int) (p - s)); + if ((int) exp_num_len != num_len) { + emsgf(_("E685: internal error: while converting number \"%.*s\" " + "to float vim_str2nr consumed %i bytes in place of %zu"), + (int) exp_num_len, s, num_len, exp_num_len); + } tv.vval.v_number = (varnumber_T) nr; } POP(tv, false); diff --git a/test/functional/eval/json_functions_spec.lua b/test/functional/eval/json_functions_spec.lua index 6379e7fed8..0140ad79aa 100644 --- a/test/functional/eval/json_functions_spec.lua +++ b/test/functional/eval/json_functions_spec.lua @@ -121,19 +121,32 @@ describe('json_decode() function', function() eq(-100000, funcs.json_decode(' -100000 ')) end) - it('fails to parse +numbers', function() + it('fails to parse +numbers and .number', function() eq('Vim(call):E474: Unidentified byte: +1000', exc_exec('call json_decode("+1000")')) + eq('Vim(call):E474: Unidentified byte: .1000', + exc_exec('call json_decode(".1000")')) end) - it('fails to parse negative numbers with space after -', function() - eq('Vim(call):E474: Missing number after minus sign: - 1000', - exc_exec('call json_decode("- 1000")')) - end) - - it('fails to parse -', function() + it('fails to parse incomplete numbers', function() + eq('Vim(call):E474: Missing number after minus sign: -.1', + exc_exec('call json_decode("-.1")')) eq('Vim(call):E474: Missing number after minus sign: -', exc_exec('call json_decode("-")')) + eq('Vim(call):E474: Missing number after decimal dot: -1.', + exc_exec('call json_decode("-1.")')) + eq('Vim(call):E474: Missing number after decimal dot: 0.', + exc_exec('call json_decode("0.")')) + eq('Vim(call):E474: Missing exponent: 0.0e', + exc_exec('call json_decode("0.0e")')) + eq('Vim(call):E474: Missing exponent: 0.0e+', + exc_exec('call json_decode("0.0e+")')) + eq('Vim(call):E474: Missing exponent: 0.0e-', + exc_exec('call json_decode("0.0e-")')) + eq('Vim(call):E474: Missing exponent: 0.0e-', + exc_exec('call json_decode("0.0e-")')) + eq('Vim(call):E474: Missing number after decimal dot: 1.e5', + exc_exec('call json_decode("1.e5")')) end) it('parses floating-point numbers', function() @@ -145,20 +158,12 @@ describe('json_decode() function', function() eq(100000.5e50, funcs.json_decode('100000.5e+50')) eq(-100000.5e-50, funcs.json_decode('-100000.5e-50')) eq(100000.5e-50, funcs.json_decode('100000.5e-50')) + eq(100000e-50, funcs.json_decode('100000e-50')) end) - it('fails to parse incomplete floating-point numbers', function() - eq('Vim(call):E474: Missing number after decimal dot: 0.', - exc_exec('call json_decode("0.")')) - eq('Vim(call):E474: Missing exponent: 0.0e', - exc_exec('call json_decode("0.0e")')) - eq('Vim(call):E474: Missing exponent: 0.0e+', - exc_exec('call json_decode("0.0e+")')) - eq('Vim(call):E474: Missing exponent: 0.0e-', - exc_exec('call json_decode("0.0e-")')) - end) - - it('fails to parse floating-point numbers with spaces inside', function() + it('fails to parse numbers with spaces inside', function() + eq('Vim(call):E474: Missing number after minus sign: - 1000', + exc_exec('call json_decode("- 1000")')) eq('Vim(call):E474: Missing number after decimal dot: 0. ', exc_exec('call json_decode("0. ")')) eq('Vim(call):E474: Missing number after decimal dot: 0. 0', From 9a56fcb2e8e97dec1e4ebce1d1287e7ab8a6ee79 Mon Sep 17 00:00:00 2001 From: ZyX Date: Mon, 7 Mar 2016 09:11:09 +0300 Subject: [PATCH 61/82] eval/decode: Rewrite json_decode_string end as suggested by oni-link --- src/nvim/eval/decode.c | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/nvim/eval/decode.c b/src/nvim/eval/decode.c index 75c88e308b..1e45336ed9 100644 --- a/src/nvim/eval/decode.c +++ b/src/nvim/eval/decode.c @@ -740,21 +740,17 @@ json_decode_string_after_cycle: } } } - if (kv_size(stack) > 1 || kv_size(container_stack)) { - emsgf(_("E474: Unexpected end of input: %.*s"), (int) buf_len, buf); - goto json_decode_string_fail; + if (kv_size(stack) == 1 && kv_size(container_stack) == 0) { + *rettv = kv_pop(stack).val; + goto json_decode_string_ret; } - goto json_decode_string_ret; + emsgf(_("E474: Unexpected end of input: %.*s"), (int) buf_len, buf); json_decode_string_fail: ret = FAIL; while (kv_size(stack)) { clear_tv(&(kv_pop(stack).val)); } json_decode_string_ret: - if (ret != FAIL) { - assert(kv_size(stack) == 1); - *rettv = kv_pop(stack).val; - } kv_destroy(stack); kv_destroy(container_stack); return ret; From 82f249f8297a26ef3fcb9675acebd76476bb40e4 Mon Sep 17 00:00:00 2001 From: ZyX Date: Mon, 7 Mar 2016 09:29:38 +0300 Subject: [PATCH 62/82] message: Remove useless emsg* functions, leaving only emsgf --- src/nvim/message.c | 41 ----------------- src/nvim/message.h | 9 ++-- src/nvim/shada.c | 109 ++++++++++++++++++++++----------------------- src/nvim/vim.h | 5 +-- 4 files changed, 59 insertions(+), 105 deletions(-) diff --git a/src/nvim/message.c b/src/nvim/message.c index 21fd97ed21..e0806d8234 100644 --- a/src/nvim/message.c +++ b/src/nvim/message.c @@ -557,52 +557,11 @@ int emsg(char_u *s) return msg_attr(s, attr); } -/* - * Print an error message with one "%s" and one string argument. - */ -int emsg2(char_u *s, char_u *a1) -{ - return emsg3(s, a1, NULL); -} - void emsg_invreg(int name) { EMSG2(_("E354: Invalid register name: '%s'"), transchar(name)); } -/// Print an error message with one or two "%s" and one or two string arguments. -int emsg3(char_u *s, char_u *a1, char_u *a2) -{ - if (emsg_not_now()) { - return TRUE; // no error messages at the moment - } - - vim_snprintf((char *)IObuff, IOSIZE, (char *)s, a1, a2); - return emsg(IObuff); -} - -/// Print an error message with one "%" PRId64 and one (int64_t) argument. -int emsgn(char_u *s, int64_t n) -{ - if (emsg_not_now()) { - return TRUE; // no error messages at the moment - } - - vim_snprintf((char *)IObuff, IOSIZE, (char *)s, n); - return emsg(IObuff); -} - -/// Print an error message with one "%" PRIu64 and one (uint64_t) argument. -int emsgu(char_u *s, uint64_t n) -{ - if (emsg_not_now()) { - return TRUE; // no error messages at the moment - } - - vim_snprintf((char *)IObuff, IOSIZE, (char *)s, n); - return emsg(IObuff); -} - /// Print an error message with unknown number of arguments bool emsgf(const char *const fmt, ...) { diff --git a/src/nvim/message.h b/src/nvim/message.h index b69b7264b4..d3a16fff93 100644 --- a/src/nvim/message.h +++ b/src/nvim/message.h @@ -37,17 +37,16 @@ #define EMSG(s) emsg((char_u *)(s)) /// Like #EMSG, but for messages with one "%s" inside -#define EMSG2(s, p) emsg2((char_u *)(s), (char_u *)(p)) +#define EMSG2(s, p) emsgf((const char *) (s), (p)) /// Like #EMSG, but for messages with two "%s" inside -#define EMSG3(s, p, q) emsg3((char_u *)(s), (char_u *)(p), \ - (char_u *)(q)) +#define EMSG3(s, p, q) emsgf((const char *) (s), (p), (q)) /// Like #EMSG, but for messages with one "%" PRId64 inside -#define EMSGN(s, n) emsgn((char_u *)(s), (int64_t)(n)) +#define EMSGN(s, n) emsgf((const char *) (s), (int64_t)(n)) /// Like #EMSG, but for messages with one "%" PRIu64 inside -#define EMSGU(s, n) emsgu((char_u *)(s), (uint64_t)(n)) +#define EMSGU(s, n) emsgf((const char *) (s), (uint64_t)(n)) /// Display message at the recorded position #define MSG_PUTS(s) msg_puts((char_u *)(s)) diff --git a/src/nvim/shada.c b/src/nvim/shada.c index 84880d1a99..32a02b0fb7 100644 --- a/src/nvim/shada.c +++ b/src/nvim/shada.c @@ -66,9 +66,6 @@ KHASH_SET_INIT_STR(strset) ((char *) copy_option_part((char_u **) src, (char_u *) dest, __VA_ARGS__)) #define find_shada_parameter(...) \ ((const char *) find_shada_parameter(__VA_ARGS__)) -#define emsg2(a, b) emsg2((char_u *) a, (char_u *) b) -#define emsg3(a, b, c) emsg3((char_u *) a, (char_u *) b, (char_u *) c) -#define emsgu(a, ...) emsgu((char_u *) a, __VA_ARGS__) #define home_replace_save(a, b) \ ((char *)home_replace_save(a, (char_u *)b)) #define home_replace(a, b, c, d, e) \ @@ -762,7 +759,7 @@ static void close_sd_writer(ShaDaWriteDef *const sd_writer) { const int fd = (int)(intptr_t) sd_writer->cookie; if (os_fsync(fd) < 0) { - emsg2(_(SERR "System error while synchronizing ShaDa file: %s"), + emsgf(_(SERR "System error while synchronizing ShaDa file: %s"), os_strerror(errno)); errno = 0; } @@ -812,11 +809,11 @@ static ShaDaReadResult sd_reader_skip(ShaDaReadDef *const sd_reader, { if (sd_reader->skip(sd_reader, offset) != OK) { if (sd_reader->error != NULL) { - emsg2(_(SERR "System error while skipping in ShaDa file: %s"), + emsgf(_(SERR "System error while skipping in ShaDa file: %s"), sd_reader->error); return kSDReadStatusReadError; } else if (sd_reader->eof) { - emsgu(_(RCERR "Error while reading ShaDa file: " + emsgf(_(RCERR "Error while reading ShaDa file: " "last entry specified that it occupies %" PRIu64 " bytes, " "but file ended earlier"), (uint64_t) offset); @@ -850,7 +847,7 @@ open_file_start: goto open_file_start; } if (fd != UV_EEXIST) { - emsg3(_(SERR "System error while opening ShaDa file %s: %s"), + emsgf(_(SERR "System error while opening ShaDa file %s: %s"), fname, os_strerror(fd)); } return fd; @@ -898,7 +895,7 @@ close_file_start: errno = 0; goto close_file_start; } else { - emsg2(_(SERR "System error while closing ShaDa file: %s"), + emsgf(_(SERR "System error while closing ShaDa file: %s"), strerror(errno)); errno = 0; } @@ -935,7 +932,7 @@ static int msgpack_sd_writer_write(void *data, const char *buf, size_t len) ShaDaWriteDef *const sd_writer = (ShaDaWriteDef *) data; ptrdiff_t written_bytes = sd_writer->write(sd_writer, buf, len); if (written_bytes == -1) { - emsg2(_(SERR "System error while writing ShaDa file: %s"), + emsgf(_(SERR "System error while writing ShaDa file: %s"), sd_writer->error); return -1; } @@ -982,7 +979,7 @@ static int shada_read_file(const char *const file, const int flags) if (of_ret != 0) { if (of_ret == UV_ENOENT && (flags & kShaDaMissingError)) { - emsg3(_(SERR "System error while opening ShaDa file %s for reading: %s"), + emsgf(_(SERR "System error while opening ShaDa file %s for reading: %s"), fname, os_strerror(of_ret)); } xfree(fname); @@ -2162,7 +2159,7 @@ shada_parse_msgpack_read_next: {} break; } case MSGPACK_UNPACK_PARSE_ERROR: { - emsgu(_(RCERR "Failed to parse ShaDa file due to a msgpack parser error " + emsgf(_(RCERR "Failed to parse ShaDa file due to a msgpack parser error " "at position %" PRIu64), (uint64_t) initial_fpos); ret = kSDReadStatusNotShaDa; @@ -2179,7 +2176,7 @@ shada_parse_msgpack_read_next: {} break; } case MSGPACK_UNPACK_CONTINUE: { - emsgu(_(RCERR "Failed to parse ShaDa file: incomplete msgpack string " + emsgf(_(RCERR "Failed to parse ShaDa file: incomplete msgpack string " "at position %" PRIu64), (uint64_t) initial_fpos); ret = kSDReadStatusNotShaDa; @@ -2187,7 +2184,7 @@ shada_parse_msgpack_read_next: {} } case MSGPACK_UNPACK_EXTRA_BYTES: { shada_parse_msgpack_extra_bytes: - emsgu(_(RCERR "Failed to parse ShaDa file: extra bytes in msgpack string " + emsgf(_(RCERR "Failed to parse ShaDa file: extra bytes in msgpack string " "at position %" PRIu64), (uint64_t) initial_fpos); ret = kSDReadStatusNotShaDa; @@ -3268,11 +3265,11 @@ static ShaDaReadResult fread_len(ShaDaReadDef *const sd_reader, (void) read_bytes; if (sd_reader->error != NULL) { - emsg2(_(SERR "System error while reading ShaDa file: %s"), + emsgf(_(SERR "System error while reading ShaDa file: %s"), sd_reader->error); return kSDReadStatusReadError; } else if (sd_reader->eof) { - emsgu(_(RCERR "Error while reading ShaDa file: " + emsgf(_(RCERR "Error while reading ShaDa file: " "last entry specified that it occupies %" PRIu64 " bytes, " "but file ended earlier"), (uint64_t) length); @@ -3307,11 +3304,11 @@ static ShaDaReadResult msgpack_read_uint64(ShaDaReadDef *const sd_reader, if (first_char == EOF) { if (sd_reader->error) { - emsg2(_(SERR "System error while reading integer from ShaDa file: %s"), + emsgf(_(SERR "System error while reading integer from ShaDa file: %s"), sd_reader->error); return kSDReadStatusReadError; } else if (sd_reader->eof) { - emsgu(_(RCERR "Error while reading ShaDa file: " + emsgf(_(RCERR "Error while reading ShaDa file: " "expected positive integer at position %" PRIu64 ", but got nothing"), (uint64_t) fpos); @@ -3342,7 +3339,7 @@ static ShaDaReadResult msgpack_read_uint64(ShaDaReadDef *const sd_reader, break; } default: { - emsgu(_(RCERR "Error while reading ShaDa file: " + emsgf(_(RCERR "Error while reading ShaDa file: " "expected positive integer at position %" PRIu64), (uint64_t) fpos); return kSDReadStatusNotShaDa; @@ -3406,18 +3403,18 @@ static inline char *get_converted_string(const vimconv_T *const sd_conv, proc) \ do { \ if (!(condition)) { \ - emsgu(_(READERR(entry_name, error_desc)), initial_fpos); \ + emsgf(_(READERR(entry_name, error_desc)), initial_fpos); \ CLEAR_GA_AND_ERROR_OUT(ad_ga); \ } \ tgt = proc(obj.via.attr); \ } while (0) #define CHECK_KEY_IS_STR(entry_name) \ if (unpacked.data.via.map.ptr[i].key.type != MSGPACK_OBJECT_STR) { \ - emsgu(_(READERR(entry_name, "has key which is not a string")), \ + emsgf(_(READERR(entry_name, "has key which is not a string")), \ initial_fpos); \ CLEAR_GA_AND_ERROR_OUT(ad_ga); \ } else if (unpacked.data.via.map.ptr[i].key.via.str.size == 0) { \ - emsgu(_(READERR(entry_name, "has empty key")), initial_fpos); \ + emsgf(_(READERR(entry_name, "has empty key")), initial_fpos); \ CLEAR_GA_AND_ERROR_OUT(ad_ga); \ } #define CHECKED_KEY(entry_name, name, error_desc, tgt, condition, attr, proc) \ @@ -3480,7 +3477,7 @@ static inline char *get_converted_string(const vimconv_T *const sd_conv, typval_T adtv; \ if (msgpack_to_vim(obj, &adtv) == FAIL \ || adtv.v_type != VAR_DICT) { \ - emsgu(_(READERR(name, \ + emsgf(_(READERR(name, \ "cannot be converted to a VimL dictionary")), \ initial_fpos); \ ga_clear(&ad_ga); \ @@ -3505,7 +3502,7 @@ static inline char *get_converted_string(const vimconv_T *const sd_conv, }; \ typval_T aetv; \ if (msgpack_to_vim(obj, &aetv) == FAIL) { \ - emsgu(_(READERR(name, "cannot be converted to a VimL list")), \ + emsgf(_(READERR(name, "cannot be converted to a VimL list")), \ initial_fpos); \ clear_tv(&aetv); \ goto shada_read_next_item_error; \ @@ -3573,7 +3570,7 @@ shada_read_next_item_start: // kSDItemUnknown cannot possibly pass that far because it is -1 and that // will fail in msgpack_read_uint64. But kSDItemMissing may and it will // otherwise be skipped because (1 << 0) will never appear in flags. - emsgu(_(RCERR "Error while reading ShaDa file: " + emsgf(_(RCERR "Error while reading ShaDa file: " "there is an item at position %" PRIu64 " " "that must not be there: Missing items are " "for internal uses only"), @@ -3643,14 +3640,14 @@ shada_read_next_item_start: switch ((ShadaEntryType) type_u64) { case kSDItemHeader: { if (!msgpack_rpc_to_dictionary(&(unpacked.data), &(entry->data.header))) { - emsgu(_(READERR("header", "is not a dictionary")), initial_fpos); + emsgf(_(READERR("header", "is not a dictionary")), initial_fpos); goto shada_read_next_item_error; } break; } case kSDItemSearchPattern: { if (unpacked.data.type != MSGPACK_OBJECT_MAP) { - emsgu(_(READERR("search pattern", "is not a dictionary")), + emsgf(_(READERR("search pattern", "is not a dictionary")), initial_fpos); goto shada_read_next_item_error; } @@ -3681,7 +3678,7 @@ shada_read_next_item_start: ADDITIONAL_KEY } if (entry->data.search_pattern.pat == NULL) { - emsgu(_(READERR("search pattern", "has no pattern")), initial_fpos); + emsgf(_(READERR("search pattern", "has no pattern")), initial_fpos); CLEAR_GA_AND_ERROR_OUT(ad_ga); } SET_ADDITIONAL_DATA(entry->data.search_pattern.additional_data, @@ -3693,7 +3690,7 @@ shada_read_next_item_start: case kSDItemGlobalMark: case kSDItemLocalMark: { if (unpacked.data.type != MSGPACK_OBJECT_MAP) { - emsgu(_(READERR("mark", "is not a dictionary")), initial_fpos); + emsgf(_(READERR("mark", "is not a dictionary")), initial_fpos); goto shada_read_next_item_error; } garray_T ad_ga; @@ -3702,7 +3699,7 @@ shada_read_next_item_start: CHECK_KEY_IS_STR("mark") if (CHECK_KEY(unpacked.data.via.map.ptr[i].key, KEY_NAME_CHAR)) { if (type_u64 == kSDItemJump || type_u64 == kSDItemChange) { - emsgu(_(READERR("mark", "has n key which is only valid for " + emsgf(_(READERR("mark", "has n key which is only valid for " "local and global mark entries")), initial_fpos); CLEAR_GA_AND_ERROR_OUT(ad_ga); } @@ -3719,15 +3716,15 @@ shada_read_next_item_start: ADDITIONAL_KEY } if (entry->data.filemark.fname == NULL) { - emsgu(_(READERR("mark", "is missing file name")), initial_fpos); + emsgf(_(READERR("mark", "is missing file name")), initial_fpos); CLEAR_GA_AND_ERROR_OUT(ad_ga); } if (entry->data.filemark.mark.lnum <= 0) { - emsgu(_(READERR("mark", "has invalid line number")), initial_fpos); + emsgf(_(READERR("mark", "has invalid line number")), initial_fpos); CLEAR_GA_AND_ERROR_OUT(ad_ga); } if (entry->data.filemark.mark.col < 0) { - emsgu(_(READERR("mark", "has invalid column number")), initial_fpos); + emsgf(_(READERR("mark", "has invalid column number")), initial_fpos); CLEAR_GA_AND_ERROR_OUT(ad_ga); } SET_ADDITIONAL_DATA(entry->data.filemark.additional_data, "mark"); @@ -3735,7 +3732,7 @@ shada_read_next_item_start: } case kSDItemRegister: { if (unpacked.data.type != MSGPACK_OBJECT_MAP) { - emsgu(_(READERR("register", "is not a dictionary")), initial_fpos); + emsgf(_(READERR("register", "is not a dictionary")), initial_fpos); goto shada_read_next_item_error; } garray_T ad_ga; @@ -3745,14 +3742,14 @@ shada_read_next_item_start: if (CHECK_KEY(unpacked.data.via.map.ptr[i].key, REG_KEY_CONTENTS)) { if (unpacked.data.via.map.ptr[i].val.type != MSGPACK_OBJECT_ARRAY) { - emsgu(_(READERR("register", + emsgf(_(READERR("register", "has " REG_KEY_CONTENTS " key with non-array value")), initial_fpos); CLEAR_GA_AND_ERROR_OUT(ad_ga); } if (unpacked.data.via.map.ptr[i].val.via.array.size == 0) { - emsgu(_(READERR("register", + emsgf(_(READERR("register", "has " REG_KEY_CONTENTS " key with empty array")), initial_fpos); CLEAR_GA_AND_ERROR_OUT(ad_ga); @@ -3761,7 +3758,7 @@ shada_read_next_item_start: unpacked.data.via.map.ptr[i].val.via.array; for (size_t i = 0; i < arr.size; i++) { if (arr.ptr[i].type != MSGPACK_OBJECT_BIN) { - emsgu(_(READERR("register", "has " REG_KEY_CONTENTS " array " + emsgf(_(READERR("register", "has " REG_KEY_CONTENTS " array " "with non-binary value")), initial_fpos); CLEAR_GA_AND_ERROR_OUT(ad_ga); } @@ -3781,7 +3778,7 @@ shada_read_next_item_start: ADDITIONAL_KEY } if (entry->data.reg.contents == NULL) { - emsgu(_(READERR("register", "has missing " REG_KEY_CONTENTS " array")), + emsgf(_(READERR("register", "has missing " REG_KEY_CONTENTS " array")), initial_fpos); CLEAR_GA_AND_ERROR_OUT(ad_ga); } @@ -3790,29 +3787,29 @@ shada_read_next_item_start: } case kSDItemHistoryEntry: { if (unpacked.data.type != MSGPACK_OBJECT_ARRAY) { - emsgu(_(READERR("history", "is not an array")), initial_fpos); + emsgf(_(READERR("history", "is not an array")), initial_fpos); goto shada_read_next_item_error; } if (unpacked.data.via.array.size < 2) { - emsgu(_(READERR("history", "does not have enough elements")), + emsgf(_(READERR("history", "does not have enough elements")), initial_fpos); goto shada_read_next_item_error; } if (unpacked.data.via.array.ptr[0].type != MSGPACK_OBJECT_POSITIVE_INTEGER) { - emsgu(_(READERR("history", "has wrong history type type")), + emsgf(_(READERR("history", "has wrong history type type")), initial_fpos); goto shada_read_next_item_error; } if (unpacked.data.via.array.ptr[1].type != MSGPACK_OBJECT_BIN) { - emsgu(_(READERR("history", "has wrong history string type")), + emsgf(_(READERR("history", "has wrong history string type")), initial_fpos); goto shada_read_next_item_error; } if (memchr(unpacked.data.via.array.ptr[1].via.bin.ptr, 0, unpacked.data.via.array.ptr[1].via.bin.size) != NULL) { - emsgu(_(READERR("history", "contains string with zero byte inside")), + emsgf(_(READERR("history", "contains string with zero byte inside")), initial_fpos); goto shada_read_next_item_error; } @@ -3822,13 +3819,13 @@ shada_read_next_item_start: entry->data.history_item.histtype == HIST_SEARCH; if (is_hist_search) { if (unpacked.data.via.array.size < 3) { - emsgu(_(READERR("search history", + emsgf(_(READERR("search history", "does not have separator character")), initial_fpos); goto shada_read_next_item_error; } if (unpacked.data.via.array.ptr[2].type != MSGPACK_OBJECT_POSITIVE_INTEGER) { - emsgu(_(READERR("search history", + emsgf(_(READERR("search history", "has wrong history separator type")), initial_fpos); goto shada_read_next_item_error; } @@ -3870,16 +3867,16 @@ shada_read_next_item_hist_no_conv: } case kSDItemVariable: { if (unpacked.data.type != MSGPACK_OBJECT_ARRAY) { - emsgu(_(READERR("variable", "is not an array")), initial_fpos); + emsgf(_(READERR("variable", "is not an array")), initial_fpos); goto shada_read_next_item_error; } if (unpacked.data.via.array.size < 2) { - emsgu(_(READERR("variable", "does not have enough elements")), + emsgf(_(READERR("variable", "does not have enough elements")), initial_fpos); goto shada_read_next_item_error; } if (unpacked.data.via.array.ptr[0].type != MSGPACK_OBJECT_BIN) { - emsgu(_(READERR("variable", "has wrong variable name type")), + emsgf(_(READERR("variable", "has wrong variable name type")), initial_fpos); goto shada_read_next_item_error; } @@ -3888,7 +3885,7 @@ shada_read_next_item_hist_no_conv: unpacked.data.via.array.ptr[0].via.bin.size); if (msgpack_to_vim(unpacked.data.via.array.ptr[1], &(entry->data.global_var.value)) == FAIL) { - emsgu(_(READERR("variable", "has value that cannot " + emsgf(_(READERR("variable", "has value that cannot " "be converted to the VimL value")), initial_fpos); goto shada_read_next_item_error; } @@ -3909,16 +3906,16 @@ shada_read_next_item_hist_no_conv: } case kSDItemSubString: { if (unpacked.data.type != MSGPACK_OBJECT_ARRAY) { - emsgu(_(READERR("sub string", "is not an array")), initial_fpos); + emsgf(_(READERR("sub string", "is not an array")), initial_fpos); goto shada_read_next_item_error; } if (unpacked.data.via.array.size < 1) { - emsgu(_(READERR("sub string", "does not have enough elements")), + emsgf(_(READERR("sub string", "does not have enough elements")), initial_fpos); goto shada_read_next_item_error; } if (unpacked.data.via.array.ptr[0].type != MSGPACK_OBJECT_BIN) { - emsgu(_(READERR("sub string", "has wrong sub string type")), + emsgf(_(READERR("sub string", "has wrong sub string type")), initial_fpos); goto shada_read_next_item_error; } @@ -3931,7 +3928,7 @@ shada_read_next_item_hist_no_conv: } case kSDItemBufferList: { if (unpacked.data.type != MSGPACK_OBJECT_ARRAY) { - emsgu(_(READERR("buffer list", "is not an array")), initial_fpos); + emsgf(_(READERR("buffer list", "is not an array")), initial_fpos); goto shada_read_next_item_error; } if (unpacked.data.via.array.size == 0) { @@ -3948,7 +3945,7 @@ shada_read_next_item_hist_no_conv: { msgpack_unpacked unpacked = unpacked_2; if (unpacked.data.type != MSGPACK_OBJECT_MAP) { - emsgu(_(RERR "Error while reading ShaDa file: " + emsgf(_(RERR "Error while reading ShaDa file: " "buffer list at position %" PRIu64 " " "contains entry that is not a dictionary"), initial_fpos); @@ -3973,21 +3970,21 @@ shada_read_next_item_hist_no_conv: } } if (entry->data.buffer_list.buffers[i].pos.lnum <= 0) { - emsgu(_(RERR "Error while reading ShaDa file: " + emsgf(_(RERR "Error while reading ShaDa file: " "buffer list at position %" PRIu64 " " "contains entry with invalid line number"), initial_fpos); CLEAR_GA_AND_ERROR_OUT(ad_ga); } if (entry->data.buffer_list.buffers[i].pos.col < 0) { - emsgu(_(RERR "Error while reading ShaDa file: " + emsgf(_(RERR "Error while reading ShaDa file: " "buffer list at position %" PRIu64 " " "contains entry with invalid column number"), initial_fpos); CLEAR_GA_AND_ERROR_OUT(ad_ga); } if (entry->data.buffer_list.buffers[i].fname == NULL) { - emsgu(_(RERR "Error while reading ShaDa file: " + emsgf(_(RERR "Error while reading ShaDa file: " "buffer list at position %" PRIu64 " " "contains entry that does not have a file name"), initial_fpos); diff --git a/src/nvim/vim.h b/src/nvim/vim.h index a94b2e12f2..e01b54e21c 100644 --- a/src/nvim/vim.h +++ b/src/nvim/vim.h @@ -273,10 +273,9 @@ enum { #include "nvim/message.h" -/* Prefer using emsg3(), because perror() may send the output to the wrong +/* Prefer using emsgf(), because perror() may send the output to the wrong * destination and mess up the screen. */ -#define PERROR(msg) \ - (void) emsg3((char_u *) "%s: %s", (char_u *)msg, (char_u *)strerror(errno)) +#define PERROR(msg) (void) emsgf("%s: %s", msg, strerror(errno)) #define SHOWCMD_COLS 10 /* columns needed by shown command */ #define STL_MAX_ITEM 80 /* max nr of % in statusline */ From 1bada1fde0a1b7251690ecfe0c7fc0c8052e2a8e Mon Sep 17 00:00:00 2001 From: ZyX Date: Mon, 7 Mar 2016 09:35:44 +0300 Subject: [PATCH 63/82] documentation: Update type() documentation --- runtime/doc/eval.txt | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/runtime/doc/eval.txt b/runtime/doc/eval.txt index fea9e669e0..d171dacad1 100644 --- a/runtime/doc/eval.txt +++ b/runtime/doc/eval.txt @@ -6783,12 +6783,14 @@ trunc({expr}) *trunc()* type({expr}) *type()* The result is a Number, depending on the type of {expr}: - Number: 0 - String: 1 + Number: 0 + String: 1 Funcref: 2 - List: 3 + List: 3 Dictionary: 4 - Float: 5 + Float: 5 + Boolean: 6 (|v:true| and |v:false|) + Null: 7 (|v:null|) To avoid the magic numbers it should be used this way: > :if type(myvar) == type(0) :if type(myvar) == type("") @@ -6796,6 +6798,10 @@ type({expr}) *type()* :if type(myvar) == type([]) :if type(myvar) == type({}) :if type(myvar) == type(0.0) + :if type(myvar) == type(v:true) +< In place of checking for |v:null| type it is better to check + for |v:null| directly as it is the only value of this type: > + :if myvar is v:null undofile({name}) *undofile()* Return the name of the undo file that would be used for a file From 69ce17878eb6a95e40b6e5c36c62a5ffdf2df62d Mon Sep 17 00:00:00 2001 From: ZyX Date: Mon, 7 Mar 2016 10:06:16 +0300 Subject: [PATCH 64/82] *: Fix linter errors --- src/nvim/eval/decode.c | 663 ++++++++++++++++++++++++----------------- src/nvim/vim.h | 4 +- 2 files changed, 394 insertions(+), 273 deletions(-) diff --git a/src/nvim/eval/decode.c b/src/nvim/eval/decode.c index 1e45336ed9..ec3be2cfb6 100644 --- a/src/nvim/eval/decode.c +++ b/src/nvim/eval/decode.c @@ -193,16 +193,388 @@ static inline int json_decoder_pop(ValuesStackItem obj, return OK; } -#define OBJ(obj_tv, is_sp_string) \ +#define LENP(p, e) \ + ((int) ((e) - (p))), (p) +#define OBJ(obj_tv, is_sp_string, didcomma_, didcolon_) \ ((ValuesStackItem) { \ .is_special_string = (is_sp_string), \ .val = (obj_tv), \ - .didcomma = didcomma, \ - .didcolon = didcolon, \ + .didcomma = (didcomma_), \ + .didcolon = (didcolon_), \ }) + #define POP(obj_tv, is_sp_string) \ do { \ - if (json_decoder_pop(OBJ(obj_tv, is_sp_string), &stack, &container_stack, \ + if (json_decoder_pop(OBJ(obj_tv, is_sp_string, *didcomma, *didcolon), \ + stack, container_stack, \ + &p, next_map_special, didcomma, didcolon) \ + == FAIL) { \ + goto parse_json_string_fail; \ + } \ + if (*next_map_special) { \ + goto parse_json_string_ret; \ + } \ + } while (0) + +/// Parse JSON double-quoted string +/// +/// @param[in] conv Defines conversion necessary to convert UTF-8 string to +/// &encoding. +/// @param[in] buf Buffer being converted. +/// @param[in] buf_len Length of the buffer. +/// @param[in,out] pp Pointer to the start of the string. Must point to '"'. +/// Is advanced to the closing '"'. +/// @param[out] stack Object stack. +/// @param[out] container_stack Container objects stack. +/// @param[out] next_map_special Is set to true when dictionary is converted +/// to a special map, otherwise not touched. +/// @param[out] didcomma True if previous token was comma. Is set to recorded +/// value when decoder is restarted, otherwise unused. +/// @param[out] didcolon True if previous token was colon. Is set to recorded +/// value when decoder is restarted, otherwise unused. +/// +/// @return OK in case of success, FAIL in case of error. +static inline int parse_json_string(vimconv_T *const conv, + const char *const buf, const size_t buf_len, + const char **const pp, + ValuesStack *const stack, + ContainerStack *const container_stack, + bool *const next_map_special, + bool *const didcomma, + bool *const didcolon) + FUNC_ATTR_NONNULL_ALL FUNC_ATTR_WARN_UNUSED_RESULT FUNC_ATTR_ALWAYS_INLINE +{ + const char *const e = buf + buf_len; + const char *p = *pp; + size_t len = 0; + const char *const s = ++p; + int ret = OK; + while (p < e && *p != '"') { + if (*p == '\\') { + p++; + if (p == e) { + emsgf(_("E474: Unfinished escape sequence: %.*s"), + (int) buf_len, buf); + goto parse_json_string_fail; + } + switch (*p) { + case 'u': { + if (p + 4 >= e) { + emsgf(_("E474: Unfinished unicode escape sequence: %.*s"), + (int) buf_len, buf); + goto parse_json_string_fail; + } else if (!ascii_isxdigit(p[1]) + || !ascii_isxdigit(p[2]) + || !ascii_isxdigit(p[3]) + || !ascii_isxdigit(p[4])) { + emsgf(_("E474: Expected four hex digits after \\u: %.*s"), + LENP(p - 1, e)); + goto parse_json_string_fail; + } + // One UTF-8 character below U+10000 can take up to 3 bytes, + // above up to 6, but they are encoded using two \u escapes. + len += 3; + p += 5; + break; + } + case '\\': + case '/': + case '"': + case 't': + case 'b': + case 'n': + case 'r': + case 'f': { + len++; + p++; + break; + } + default: { + emsgf(_("E474: Unknown escape sequence: %.*s"), LENP(p - 1, e)); + goto parse_json_string_fail; + } + } + } else { + uint8_t p_byte = (uint8_t) *p; + // unescaped = %x20-21 / %x23-5B / %x5D-10FFFF + if (p_byte < 0x20) { + emsgf(_("E474: ASCII control characters cannot be present " + "inside string: %.*s"), LENP(p, e)); + goto parse_json_string_fail; + } + const int ch = utf_ptr2char((char_u *) p); + // All characters above U+007F are encoded using two or more bytes + // and thus cannot possibly be equal to *p. But utf_ptr2char({0xFF, + // 0}) will return 0xFF, even though 0xFF cannot start any UTF-8 + // code point at all. + // + // The only exception is U+00C3 which is represented as 0xC3 0x83. + if (ch >= 0x80 && p_byte == ch + && !(ch == 0xC3 && p + 1 < e && (uint8_t) p[1] == 0x83)) { + emsgf(_("E474: Only UTF-8 strings allowed: %.*s"), LENP(p, e)); + goto parse_json_string_fail; + } else if (ch > 0x10FFFF) { + emsgf(_("E474: Only UTF-8 code points up to U+10FFFF " + "are allowed to appear unescaped: %.*s"), LENP(p, e)); + goto parse_json_string_fail; + } + const size_t ch_len = (size_t) utf_char2len(ch); + assert(ch_len == (size_t) (ch ? utf_ptr2len((char_u *) p) : 1)); + len += ch_len; + p += ch_len; + } + } + if (p == e || *p != '"') { + emsgf(_("E474: Expected string end: %.*s"), (int) buf_len, buf); + goto parse_json_string_fail; + } + if (len == 0) { + POP(((typval_T) { + .v_type = VAR_STRING, + .vval = { .v_string = NULL }, + }), false); + goto parse_json_string_ret; + } + char *str = xmalloc(len + 1); + int fst_in_pair = 0; + char *str_end = str; + bool hasnul = false; +#define PUT_FST_IN_PAIR(fst_in_pair, str_end) \ + do { \ + if (fst_in_pair != 0) { \ + str_end += utf_char2bytes(fst_in_pair, (char_u *) str_end); \ + fst_in_pair = 0; \ + } \ + } while (0) + for (const char *t = s; t < p; t++) { + if (t[0] != '\\' || t[1] != 'u') { + PUT_FST_IN_PAIR(fst_in_pair, str_end); + } + if (*t == '\\') { + t++; + switch (*t) { + case 'u': { + const char ubuf[] = { t[1], t[2], t[3], t[4] }; + t += 4; + unsigned long ch; + vim_str2nr((char_u *) ubuf, NULL, NULL, + STR2NR_HEX | STR2NR_FORCE, NULL, &ch, 4); + if (ch == 0) { + hasnul = true; + } + if (SURROGATE_HI_START <= ch && ch <= SURROGATE_HI_END) { + PUT_FST_IN_PAIR(fst_in_pair, str_end); + fst_in_pair = (int) ch; + } else if (SURROGATE_LO_START <= ch && ch <= SURROGATE_LO_END + && fst_in_pair != 0) { + const int full_char = ( + (int) (ch - SURROGATE_LO_START) + + ((fst_in_pair - SURROGATE_HI_START) << 10) + + SURROGATE_FIRST_CHAR); + str_end += utf_char2bytes(full_char, (char_u *) str_end); + fst_in_pair = 0; + } else { + PUT_FST_IN_PAIR(fst_in_pair, str_end); + str_end += utf_char2bytes((int) ch, (char_u *) str_end); + } + break; + } + case '\\': + case '/': + case '"': + case 't': + case 'b': + case 'n': + case 'r': + case 'f': { + static const char escapes[] = { + ['\\'] = '\\', + ['/'] = '/', + ['"'] = '"', + ['t'] = TAB, + ['b'] = BS, + ['n'] = NL, + ['r'] = CAR, + ['f'] = FF, + }; + *str_end++ = escapes[(int) *t]; + break; + } + default: { + assert(false); + } + } + } else { + *str_end++ = *t; + } + } + PUT_FST_IN_PAIR(fst_in_pair, str_end); +#undef PUT_FST_IN_PAIR + if (conv->vc_type != CONV_NONE) { + size_t str_len = (size_t) (str_end - str); + char *const new_str = (char *) string_convert(conv, (char_u *) str, + &str_len); + if (new_str == NULL) { + emsgf(_("E474: Failed to convert string \"%.*s\" from UTF-8"), + (int) str_len, str); + xfree(str); + goto parse_json_string_fail; + } + xfree(str); + str = new_str; + str_end = new_str + str_len; + } + if (hasnul) { + typval_T obj; + list_T *const list = list_alloc(); + list->lv_refcount++; + create_special_dict(&obj, kMPString, ((typval_T) { + .v_type = VAR_LIST, + .v_lock = VAR_UNLOCKED, + .vval = { .v_list = list }, + })); + if (encode_list_write((void *) list, str, (size_t) (str_end - str)) + == -1) { + clear_tv(&obj); + goto parse_json_string_fail; + } + xfree(str); + POP(obj, true); + } else { + *str_end = NUL; + POP(((typval_T) { + .v_type = VAR_STRING, + .vval = { .v_string = (char_u *) str }, + }), false); + } + goto parse_json_string_ret; +parse_json_string_fail: + ret = FAIL; +parse_json_string_ret: + *pp = p; + return ret; +} + +#undef POP + +/// Parse JSON number: both floating-point and integer +/// +/// Number format: `-?\d+(?:.\d+)?(?:[eE][+-]?\d+)?`. +/// +/// @param[in] buf Buffer being converted. +/// @param[in] buf_len Length of the buffer. +/// @param[in,out] pp Pointer to the start of the number. Must point to +/// a digit or a minus sign. Is advanced to the last +/// character of the number. +/// @param[out] stack Object stack. +/// @param[out] container_stack Container objects stack. +/// @param[out] next_map_special Is set to true when dictionary is converted +/// to a special map, otherwise not touched. +/// @param[out] didcomma True if previous token was comma. Is set to recorded +/// value when decoder is restarted, otherwise unused. +/// @param[out] didcolon True if previous token was colon. Is set to recorded +/// value when decoder is restarted, otherwise unused. +/// +/// @return OK in case of success, FAIL in case of error. +static inline int parse_json_number(const char *const buf, const size_t buf_len, + const char **const pp, + ValuesStack *const stack, + ContainerStack *const container_stack, + bool *const next_map_special, + bool *const didcomma, + bool *const didcolon) + FUNC_ATTR_NONNULL_ALL FUNC_ATTR_WARN_UNUSED_RESULT FUNC_ATTR_ALWAYS_INLINE +{ + const char *const e = buf + buf_len; + const char *p = *pp; + int ret = OK; + const char *const s = p; + const char *ints = NULL; + const char *fracs = NULL; + const char *exps = NULL; + if (*p == '-') { + p++; + } + ints = p; + while (p < e && ascii_isdigit(*p)) { + p++; + } + if (p < e && p != ints && (*p == '.' || *p == 'e' || *p == 'E')) { + if (*p == '.') { + p++; + fracs = p; + while (p < e && ascii_isdigit(*p)) { + p++; + } + } + if (p < e && (*p == 'e' || *p == 'E')) { + p++; + if (p < e && (*p == '-' || *p == '+')) { + p++; + } + exps = p; + while (p < e && ascii_isdigit(*p)) { + p++; + } + } + } + if (p == ints) { + emsgf(_("E474: Missing number after minus sign: %.*s"), LENP(s, e)); + goto parse_json_number_fail; + } else if (p == fracs || exps == fracs + 1) { + emsgf(_("E474: Missing number after decimal dot: %.*s"), LENP(s, e)); + goto parse_json_number_fail; + } else if (p == exps) { + emsgf(_("E474: Missing exponent: %.*s"), LENP(s, e)); + goto parse_json_number_fail; + } + typval_T tv = { + .v_type = VAR_NUMBER, + .v_lock = VAR_UNLOCKED, + }; + const size_t exp_num_len = (size_t) (p - s); + if (fracs || exps) { + // Convert floating-point number + const size_t num_len = string2float(s, &tv.vval.v_float); + if (exp_num_len != num_len) { + emsgf(_("E685: internal error: while converting number \"%.*s\" " + "to float string2float consumed %zu bytes in place of %zu"), + (int) exp_num_len, s, num_len, exp_num_len); + } + tv.v_type = VAR_FLOAT; + } else { + // Convert integer + long nr; + int num_len; + vim_str2nr((char_u *) s, NULL, &num_len, 0, &nr, NULL, (int) (p - s)); + if ((int) exp_num_len != num_len) { + emsgf(_("E685: internal error: while converting number \"%.*s\" " + "to float vim_str2nr consumed %i bytes in place of %zu"), + (int) exp_num_len, s, num_len, exp_num_len); + } + tv.vval.v_number = (varnumber_T) nr; + } + if (json_decoder_pop(OBJ(tv, false, *didcomma, *didcolon), + stack, container_stack, + &p, next_map_special, didcomma, didcolon) == FAIL) { + goto parse_json_number_fail; + } + if (*next_map_special) { + goto parse_json_number_ret; + } + p--; + goto parse_json_number_ret; +parse_json_number_fail: + ret = FAIL; +parse_json_number_ret: + *pp = p; + return ret; +} + +#define POP(obj_tv, is_sp_string) \ + do { \ + if (json_decoder_pop(OBJ(obj_tv, is_sp_string, didcomma, didcolon), \ + &stack, &container_stack, \ &p, &next_map_special, &didcomma, &didcolon) \ == FAIL) { \ goto json_decode_string_fail; \ @@ -223,8 +595,6 @@ int json_decode_string(const char *const buf, const size_t buf_len, typval_T *const rettv) FUNC_ATTR_NONNULL_ALL FUNC_ATTR_WARN_UNUSED_RESULT { -#define LENP(p, e) \ - ((int) ((e) - (p))), (p) const char *p = buf; const char *const e = buf + buf_len; while (p < e && (*p == ' ' || *p == '\t' || *p == '\n')) { @@ -383,205 +753,14 @@ json_decode_string_cycle_start: break; } case '"': { - size_t len = 0; - const char *const s = ++p; - while (p < e && *p != '"') { - if (*p == '\\') { - p++; - if (p == e) { - emsgf(_("E474: Unfinished escape sequence: %.*s"), - (int) buf_len, buf); - goto json_decode_string_fail; - } - switch (*p) { - case 'u': { - if (p + 4 >= e) { - emsgf(_("E474: Unfinished unicode escape sequence: %.*s"), - (int) buf_len, buf); - goto json_decode_string_fail; - } else if (!ascii_isxdigit(p[1]) - || !ascii_isxdigit(p[2]) - || !ascii_isxdigit(p[3]) - || !ascii_isxdigit(p[4])) { - emsgf(_("E474: Expected four hex digits after \\u: %.*s"), - LENP(p - 1, e)); - goto json_decode_string_fail; - } - // One UTF-8 character below U+10000 can take up to 3 bytes, - // above up to 6, but they are encoded using two \u escapes. - len += 3; - p += 5; - break; - } - case '\\': - case '/': - case '"': - case 't': - case 'b': - case 'n': - case 'r': - case 'f': { - len++; - p++; - break; - } - default: { - emsgf(_("E474: Unknown escape sequence: %.*s"), LENP(p - 1, e)); - goto json_decode_string_fail; - } - } - } else { - uint8_t p_byte = (uint8_t) *p; - // unescaped = %x20-21 / %x23-5B / %x5D-10FFFF - if (p_byte < 0x20) { - emsgf(_("E474: ASCII control characters cannot be present " - "inside string: %.*s"), LENP(p, e)); - goto json_decode_string_fail; - } - const int ch = utf_ptr2char((char_u *) p); - // All characters above U+007F are encoded using two or more bytes - // and thus cannot possibly be equal to *p. But utf_ptr2char({0xFF, - // 0}) will return 0xFF, even though 0xFF cannot start any UTF-8 - // code point at all. - // - // The only exception is U+00C3 which is represented as 0xC3 0x83. - if (ch >= 0x80 && p_byte == ch && !( - ch == 0xC3 && p + 1 < e && (uint8_t) p[1] == 0x83)) { - emsgf(_("E474: Only UTF-8 strings allowed: %.*s"), LENP(p, e)); - goto json_decode_string_fail; - } else if (ch > 0x10FFFF) { - emsgf(_("E474: Only UTF-8 code points up to U+10FFFF " - "are allowed to appear unescaped: %.*s"), LENP(p, e)); - goto json_decode_string_fail; - } - const size_t ch_len = (size_t) utf_char2len(ch); - assert(ch_len == (size_t) (ch ? utf_ptr2len((char_u *) p) : 1)); - len += ch_len; - p += ch_len; - } - } - if (p == e || *p != '"') { - emsgf(_("E474: Expected string end: %.*s"), (int) buf_len, buf); + if (parse_json_string(&conv, buf, buf_len, &p, &stack, &container_stack, + &next_map_special, &didcomma, &didcolon) + == FAIL) { + // Error message was already given goto json_decode_string_fail; } - if (len == 0) { - POP(((typval_T) { - .v_type = VAR_STRING, - .vval = { .v_string = NULL }, - }), false); - break; - } - char *str = xmalloc(len + 1); - int fst_in_pair = 0; - char *str_end = str; - bool hasnul = false; -#define PUT_FST_IN_PAIR(fst_in_pair, str_end) \ - do { \ - if (fst_in_pair != 0) { \ - str_end += utf_char2bytes(fst_in_pair, (char_u *) str_end); \ - fst_in_pair = 0; \ - } \ - } while (0) - for (const char *t = s; t < p; t++) { - if (t[0] != '\\' || t[1] != 'u') { - PUT_FST_IN_PAIR(fst_in_pair, str_end); - } - if (*t == '\\') { - t++; - switch (*t) { - case 'u': { - const char ubuf[] = { t[1], t[2], t[3], t[4] }; - t += 4; - unsigned long ch; - vim_str2nr((char_u *) ubuf, NULL, NULL, - STR2NR_HEX | STR2NR_FORCE, NULL, &ch, 4); - if (ch == 0) { - hasnul = true; - } - if (SURROGATE_HI_START <= ch && ch <= SURROGATE_HI_END) { - PUT_FST_IN_PAIR(fst_in_pair, str_end); - fst_in_pair = (int) ch; - } else if (SURROGATE_LO_START <= ch && ch <= SURROGATE_LO_END - && fst_in_pair != 0) { - const int full_char = ( - (int) (ch - SURROGATE_LO_START) - + ((fst_in_pair - SURROGATE_HI_START) << 10) - + SURROGATE_FIRST_CHAR); - str_end += utf_char2bytes(full_char, (char_u *) str_end); - fst_in_pair = 0; - } else { - PUT_FST_IN_PAIR(fst_in_pair, str_end); - str_end += utf_char2bytes((int) ch, (char_u *) str_end); - } - break; - } - case '\\': - case '/': - case '"': - case 't': - case 'b': - case 'n': - case 'r': - case 'f': { - static const char escapes[] = { - ['\\'] = '\\', - ['/'] = '/', - ['"'] = '"', - ['t'] = TAB, - ['b'] = BS, - ['n'] = NL, - ['r'] = CAR, - ['f'] = FF, - }; - *str_end++ = escapes[(int) *t]; - break; - } - default: { - assert(false); - } - } - } else { - *str_end++ = *t; - } - } - PUT_FST_IN_PAIR(fst_in_pair, str_end); -#undef PUT_FST_IN_PAIR - if (conv.vc_type != CONV_NONE) { - size_t str_len = (size_t) (str_end - str); - char *const new_str = (char *) string_convert(&conv, (char_u *) str, - &str_len); - if (new_str == NULL) { - emsgf(_("E474: Failed to convert string \"%.*s\" from UTF-8"), - (int) str_len, str); - xfree(str); - goto json_decode_string_fail; - } - xfree(str); - str = new_str; - str_end = new_str + str_len; - } - if (hasnul) { - typval_T obj; - list_T *const list = list_alloc(); - list->lv_refcount++; - create_special_dict(&obj, kMPString, ((typval_T) { - .v_type = VAR_LIST, - .v_lock = VAR_UNLOCKED, - .vval = { .v_list = list }, - })); - if (encode_list_write((void *) list, str, (size_t) (str_end - str)) - == -1) { - clear_tv(&obj); - goto json_decode_string_fail; - } - xfree(str); - POP(obj, true); - } else { - *str_end = NUL; - POP(((typval_T) { - .v_type = VAR_STRING, - .vval = { .v_string = (char_u *) str }, - }), false); + if (next_map_special) { + goto json_decode_string_cycle_start; } break; } @@ -596,75 +775,15 @@ json_decode_string_cycle_start: case '7': case '8': case '9': { - // a.bE[+-]exp - const char *const s = p; - const char *ints = NULL; - const char *fracs = NULL; - const char *exps = NULL; - if (*p == '-') { - p++; - } - ints = p; - while (p < e && ascii_isdigit(*p)) { - p++; - } - if (p < e && p != ints && (*p == '.' || *p == 'e' || *p == 'E')) { - if (*p == '.') { - p++; - fracs = p; - while (p < e && ascii_isdigit(*p)) { - p++; - } - } - if (p < e && (*p == 'e' || *p == 'E')) { - p++; - if (p < e && (*p == '-' || *p == '+')) { - p++; - } - exps = p; - while (p < e && ascii_isdigit(*p)) { - p++; - } - } - } - if (p == ints) { - emsgf(_("E474: Missing number after minus sign: %.*s"), LENP(s, e)); - goto json_decode_string_fail; - } else if (p == fracs || exps == fracs + 1) { - emsgf(_("E474: Missing number after decimal dot: %.*s"), LENP(s, e)); - goto json_decode_string_fail; - } else if (p == exps) { - emsgf(_("E474: Missing exponent: %.*s"), LENP(s, e)); + if (parse_json_number(buf, buf_len, &p, &stack, &container_stack, + &next_map_special, &didcomma, &didcolon) + == FAIL) { + // Error message was already given goto json_decode_string_fail; } - typval_T tv = { - .v_type = VAR_NUMBER, - .v_lock = VAR_UNLOCKED, - }; - const size_t exp_num_len = (size_t) (p - s); - if (fracs || exps) { - // Convert floating-point number - const size_t num_len = string2float(s, &tv.vval.v_float); - if (exp_num_len != num_len) { - emsgf(_("E685: internal error: while converting number \"%.*s\" " - "to float string2float consumed %zu bytes in place of %zu"), - (int) exp_num_len, s, num_len, exp_num_len); - } - tv.v_type = VAR_FLOAT; - } else { - // Convert integer - long nr; - int num_len; - vim_str2nr((char_u *) s, NULL, &num_len, 0, &nr, NULL, (int) (p - s)); - if ((int) exp_num_len != num_len) { - emsgf(_("E685: internal error: while converting number \"%.*s\" " - "to float vim_str2nr consumed %i bytes in place of %zu"), - (int) exp_num_len, s, num_len, exp_num_len); - } - tv.vval.v_number = (varnumber_T) nr; + if (next_map_special) { + goto json_decode_string_cycle_start; } - POP(tv, false); - p--; break; } case '[': { @@ -681,7 +800,7 @@ json_decode_string_cycle_start: .container = tv, .special_val = NULL, })); - kv_push(ValuesStackItem, stack, OBJ(tv, false)); + kv_push(ValuesStackItem, stack, OBJ(tv, false, didcomma, didcolon)); break; } case '{': { @@ -711,7 +830,7 @@ json_decode_string_cycle_start: .container = tv, .special_val = val_list, })); - kv_push(ValuesStackItem, stack, OBJ(tv, false)); + kv_push(ValuesStackItem, stack, OBJ(tv, false, didcomma, didcolon)); break; } default: { @@ -756,7 +875,9 @@ json_decode_string_ret: return ret; } +#undef LENP #undef POP + #undef OBJ #undef DICT_LEN diff --git a/src/nvim/vim.h b/src/nvim/vim.h index e01b54e21c..623ea19e36 100644 --- a/src/nvim/vim.h +++ b/src/nvim/vim.h @@ -273,8 +273,8 @@ enum { #include "nvim/message.h" -/* Prefer using emsgf(), because perror() may send the output to the wrong - * destination and mess up the screen. */ +// Prefer using emsgf(), because perror() may send the output to the wrong +// destination and mess up the screen. #define PERROR(msg) (void) emsgf("%s: %s", msg, strerror(errno)) #define SHOWCMD_COLS 10 /* columns needed by shown command */ From 515fea1ef09e3debee9e226f34d3e62e47e8a08d Mon Sep 17 00:00:00 2001 From: ZyX Date: Wed, 9 Mar 2016 02:08:53 +0300 Subject: [PATCH 65/82] eval/decode: Reject even more numbers Rejects leading zeroes and numbers like 1.e+5 (decimal dot with missing number with signed exponent). --- src/nvim/eval/decode.c | 8 ++++- test/functional/eval/json_functions_spec.lua | 38 ++++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/src/nvim/eval/decode.c b/src/nvim/eval/decode.c index ec3be2cfb6..f74f2b3150 100644 --- a/src/nvim/eval/decode.c +++ b/src/nvim/eval/decode.c @@ -492,6 +492,7 @@ static inline int parse_json_number(const char *const buf, const size_t buf_len, const char *ints = NULL; const char *fracs = NULL; const char *exps = NULL; + const char *exps_s = NULL; if (*p == '-') { p++; } @@ -499,6 +500,10 @@ static inline int parse_json_number(const char *const buf, const size_t buf_len, while (p < e && ascii_isdigit(*p)) { p++; } + if (p != ints + 1 && *ints == '0') { + emsgf(_("E474: Leading zeroes are not allowed: %.*s"), LENP(s, e)); + goto parse_json_number_fail; + } if (p < e && p != ints && (*p == '.' || *p == 'e' || *p == 'E')) { if (*p == '.') { p++; @@ -509,6 +514,7 @@ static inline int parse_json_number(const char *const buf, const size_t buf_len, } if (p < e && (*p == 'e' || *p == 'E')) { p++; + exps_s = p; if (p < e && (*p == '-' || *p == '+')) { p++; } @@ -521,7 +527,7 @@ static inline int parse_json_number(const char *const buf, const size_t buf_len, if (p == ints) { emsgf(_("E474: Missing number after minus sign: %.*s"), LENP(s, e)); goto parse_json_number_fail; - } else if (p == fracs || exps == fracs + 1) { + } else if (p == fracs || exps_s == fracs + 1) { emsgf(_("E474: Missing number after decimal dot: %.*s"), LENP(s, e)); goto parse_json_number_fail; } else if (p == exps) { diff --git a/test/functional/eval/json_functions_spec.lua b/test/functional/eval/json_functions_spec.lua index 0140ad79aa..8438620109 100644 --- a/test/functional/eval/json_functions_spec.lua +++ b/test/functional/eval/json_functions_spec.lua @@ -119,6 +119,8 @@ describe('json_decode() function', function() eq(-100000, funcs.json_decode('-100000')) eq(100000, funcs.json_decode(' 100000 ')) eq(-100000, funcs.json_decode(' -100000 ')) + eq(0, funcs.json_decode('0')) + eq(0, funcs.json_decode('-0')) end) it('fails to parse +numbers and .number', function() @@ -128,6 +130,17 @@ describe('json_decode() function', function() exc_exec('call json_decode(".1000")')) end) + it('fails to parse numbers with leading zeroes', function() + eq('Vim(call):E474: Leading zeroes are not allowed: 00.1', + exc_exec('call json_decode("00.1")')) + eq('Vim(call):E474: Leading zeroes are not allowed: 01', + exc_exec('call json_decode("01")')) + eq('Vim(call):E474: Leading zeroes are not allowed: -01', + exc_exec('call json_decode("-01")')) + eq('Vim(call):E474: Leading zeroes are not allowed: -001.0', + exc_exec('call json_decode("-001.0")')) + end) + it('fails to parse incomplete numbers', function() eq('Vim(call):E474: Missing number after minus sign: -.1', exc_exec('call json_decode("-.1")')) @@ -147,6 +160,10 @@ describe('json_decode() function', function() exc_exec('call json_decode("0.0e-")')) eq('Vim(call):E474: Missing number after decimal dot: 1.e5', exc_exec('call json_decode("1.e5")')) + eq('Vim(call):E474: Missing number after decimal dot: 1.e+5', + exc_exec('call json_decode("1.e+5")')) + eq('Vim(call):E474: Missing number after decimal dot: 1.e+', + exc_exec('call json_decode("1.e+")')) end) it('parses floating-point numbers', function() @@ -159,6 +176,27 @@ describe('json_decode() function', function() eq(-100000.5e-50, funcs.json_decode('-100000.5e-50')) eq(100000.5e-50, funcs.json_decode('100000.5e-50')) eq(100000e-50, funcs.json_decode('100000e-50')) + eq(0.5, funcs.json_decode('0.5')) + eq(0.005, funcs.json_decode('0.005')) + eq(0.005, funcs.json_decode('0.00500')) + eq(0.5, funcs.json_decode('0.00500e+002')) + eq(0.00005, funcs.json_decode('0.00500e-002')) + + eq(-0.0, funcs.json_decode('-0.0')) + eq(-0.0, funcs.json_decode('-0.0e0')) + eq(-0.0, funcs.json_decode('-0.0e+0')) + eq(-0.0, funcs.json_decode('-0.0e-0')) + eq(-0.0, funcs.json_decode('-0e-0')) + eq(-0.0, funcs.json_decode('-0e-2')) + eq(-0.0, funcs.json_decode('-0e+2')) + + eq(0.0, funcs.json_decode('0.0')) + eq(0.0, funcs.json_decode('0.0e0')) + eq(0.0, funcs.json_decode('0.0e+0')) + eq(0.0, funcs.json_decode('0.0e-0')) + eq(0.0, funcs.json_decode('0e-0')) + eq(0.0, funcs.json_decode('0e-2')) + eq(0.0, funcs.json_decode('0e+2')) end) it('fails to parse numbers with spaces inside', function() From 0c598774d8f6358f9cdf86a56cbe1355b503907f Mon Sep 17 00:00:00 2001 From: ZyX Date: Wed, 9 Mar 2016 02:10:53 +0300 Subject: [PATCH 66/82] eval/decode: Fix typo in internal error message --- src/nvim/eval/decode.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nvim/eval/decode.c b/src/nvim/eval/decode.c index f74f2b3150..b31b21b4da 100644 --- a/src/nvim/eval/decode.c +++ b/src/nvim/eval/decode.c @@ -555,7 +555,7 @@ static inline int parse_json_number(const char *const buf, const size_t buf_len, vim_str2nr((char_u *) s, NULL, &num_len, 0, &nr, NULL, (int) (p - s)); if ((int) exp_num_len != num_len) { emsgf(_("E685: internal error: while converting number \"%.*s\" " - "to float vim_str2nr consumed %i bytes in place of %zu"), + "to integer vim_str2nr consumed %i bytes in place of %zu"), (int) exp_num_len, s, num_len, exp_num_len); } tv.vval.v_number = (varnumber_T) nr; From 2b0d46195be0792791171aa23d04ee7ba31c54c9 Mon Sep 17 00:00:00 2001 From: ZyX Date: Wed, 9 Mar 2016 02:28:12 +0300 Subject: [PATCH 67/82] eval/decode: Clarify meaning of some pointer arguments --- src/nvim/eval/decode.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/nvim/eval/decode.c b/src/nvim/eval/decode.c index b31b21b4da..1303e288c3 100644 --- a/src/nvim/eval/decode.c +++ b/src/nvim/eval/decode.c @@ -81,8 +81,10 @@ static inline void create_special_dict(typval_T *const rettv, /// for error reporting and is also set when decoding is /// restarted due to the necessity of converting regular /// dictionary to a special map. -/// @param[out] next_map_special Is set to true when dictionary is converted -/// to a special map, otherwise not touched. +/// @param[out] next_map_special Is set to true when dictionary needs to be +/// converted to a special map, otherwise not +/// touched. Indicates that decoding has been +/// restarted. /// @param[out] didcomma True if previous token was comma. Is set to recorded /// value when decoder is restarted, otherwise unused. /// @param[out] didcolon True if previous token was colon. Is set to recorded @@ -223,7 +225,9 @@ static inline int json_decoder_pop(ValuesStackItem obj, /// @param[in] buf Buffer being converted. /// @param[in] buf_len Length of the buffer. /// @param[in,out] pp Pointer to the start of the string. Must point to '"'. -/// Is advanced to the closing '"'. +/// Is advanced to the closing '"'. Also see +/// json_decoder_pop(), it may set pp to another location +/// and alter next_map_special, didcomma and didcolon. /// @param[out] stack Object stack. /// @param[out] container_stack Container objects stack. /// @param[out] next_map_special Is set to true when dictionary is converted @@ -465,7 +469,9 @@ parse_json_string_ret: /// @param[in] buf_len Length of the buffer. /// @param[in,out] pp Pointer to the start of the number. Must point to /// a digit or a minus sign. Is advanced to the last -/// character of the number. +/// character of the number. Also see json_decoder_pop(), it +/// may set pp to another location and alter +/// next_map_special, didcomma and didcolon. /// @param[out] stack Object stack. /// @param[out] container_stack Container objects stack. /// @param[out] next_map_special Is set to true when dictionary is converted From d06c2a1b1846a96a45625ad5472a235b2d249933 Mon Sep 17 00:00:00 2001 From: ZyX Date: Thu, 10 Mar 2016 01:06:43 +0300 Subject: [PATCH 68/82] eval/decode: Do not overflow when parsing `-` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Also makes if’s less nested. --- src/nvim/eval/decode.c | 40 ++++++++++++++++++++-------------- test/unit/eval/decode_spec.lua | 20 ++++++++++------- 2 files changed, 36 insertions(+), 24 deletions(-) diff --git a/src/nvim/eval/decode.c b/src/nvim/eval/decode.c index 1303e288c3..2e9bf8fbac 100644 --- a/src/nvim/eval/decode.c +++ b/src/nvim/eval/decode.c @@ -503,6 +503,9 @@ static inline int parse_json_number(const char *const buf, const size_t buf_len, p++; } ints = p; + if (p >= e) { + goto parse_json_number_check; + } while (p < e && ascii_isdigit(*p)) { p++; } @@ -510,26 +513,31 @@ static inline int parse_json_number(const char *const buf, const size_t buf_len, emsgf(_("E474: Leading zeroes are not allowed: %.*s"), LENP(s, e)); goto parse_json_number_fail; } - if (p < e && p != ints && (*p == '.' || *p == 'e' || *p == 'E')) { - if (*p == '.') { + if (p >= e || p == ints) { + goto parse_json_number_check; + } + if (*p == '.') { + p++; + fracs = p; + while (p < e && ascii_isdigit(*p)) { p++; - fracs = p; - while (p < e && ascii_isdigit(*p)) { - p++; - } } - if (p < e && (*p == 'e' || *p == 'E')) { - p++; - exps_s = p; - if (p < e && (*p == '-' || *p == '+')) { - p++; - } - exps = p; - while (p < e && ascii_isdigit(*p)) { - p++; - } + if (p >= e || p == fracs) { + goto parse_json_number_check; } } + if (*p == 'e' || *p == 'E') { + p++; + exps_s = p; + if (p < e && (*p == '-' || *p == '+')) { + p++; + } + exps = p; + while (p < e && ascii_isdigit(*p)) { + p++; + } + } +parse_json_number_check: if (p == ints) { emsgf(_("E474: Missing number after minus sign: %.*s"), LENP(s, e)); goto parse_json_number_fail; diff --git a/test/unit/eval/decode_spec.lua b/test/unit/eval/decode_spec.lua index 57fc5f7b94..44471c1877 100644 --- a/test/unit/eval/decode_spec.lua +++ b/test/unit/eval/decode_spec.lua @@ -76,15 +76,15 @@ describe('json_decode_string()', function() eq(decode.VAR_UNKNOWN, rettv.v_type) end) - it('does not overflow in error messages', function() - local check_failure = function(s, len, msg) - local rettv = ffi.new('typval_T', {v_type=decode.VAR_UNKNOWN}) - eq(0, decode.json_decode_string(s, len, rettv)) - eq(decode.VAR_UNKNOWN, rettv.v_type) - neq(nil, decode.last_msg_hist) - eq(msg, ffi.string(decode.last_msg_hist.msg)) - end + local check_failure = function(s, len, msg) local rettv = ffi.new('typval_T', {v_type=decode.VAR_UNKNOWN}) + eq(0, decode.json_decode_string(s, len, rettv)) + eq(decode.VAR_UNKNOWN, rettv.v_type) + neq(nil, decode.last_msg_hist) + eq(msg, ffi.string(decode.last_msg_hist.msg)) + end + + it('does not overflow in error messages', function() check_failure(']test', 1, 'E474: No container to close: ]') check_failure('[}test', 2, 'E474: Closing list with curly bracket: }') check_failure('{]test', 2, @@ -129,6 +129,10 @@ describe('json_decode_string()', function() check_failure('[1test', 2, 'E474: Unexpected end of input: [1') end) + it('does not overflow with `-`', function() + check_failure('-0', 1, 'E474: Missing number after minus sign: -') + end) + it('does not overflow and crash when running with `"`', function() local rettv = ffi.new('typval_T', {v_type=decode.VAR_UNKNOWN}) decode.emsg_silent = 1 From c129f6cfafc77d3f6e22b2ac11b5c8f2cec033d3 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sat, 12 Mar 2016 13:47:34 +0300 Subject: [PATCH 69/82] eval/decode: Accept `\r` as space character --- src/nvim/eval/decode.c | 8 +++++--- test/functional/eval/json_functions_spec.lua | 6 ++++++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/nvim/eval/decode.c b/src/nvim/eval/decode.c index 2e9bf8fbac..10dd36c137 100644 --- a/src/nvim/eval/decode.c +++ b/src/nvim/eval/decode.c @@ -617,7 +617,7 @@ int json_decode_string(const char *const buf, const size_t buf_len, { const char *p = buf; const char *const e = buf + buf_len; - while (p < e && (*p == ' ' || *p == '\t' || *p == '\n')) { + while (p < e && (*p == ' ' || *p == TAB || *p == NL || *p == CAR)) { p++; } if (p == e) { @@ -730,7 +730,8 @@ json_decode_string_cycle_start: } case ' ': case TAB: - case NL: { + case NL: + case CAR: { continue; } case 'n': { @@ -870,7 +871,8 @@ json_decode_string_after_cycle: switch (*p) { case NL: case ' ': - case TAB: { + case TAB: + case CAR: { break; } default: { diff --git a/test/functional/eval/json_functions_spec.lua b/test/functional/eval/json_functions_spec.lua index 8438620109..8483152dbf 100644 --- a/test/functional/eval/json_functions_spec.lua +++ b/test/functional/eval/json_functions_spec.lua @@ -515,6 +515,12 @@ describe('json_decode() function', function() eq('Vim(call):E474: Attempt to decode a blank string', exc_exec('call json_decode(" \\t\\n \\n\\t\\t \\n\\t\\n \\n \\t\\n\\t ")')) end) + + it('accepts all spaces in every position where space may be put', function() + local s = ' \t\n\r \t\r\n \n\t\r \n\r\t \r\t\n \r\n\t\t \n\r\t \r\n\t\n \r\t\n\r \t\r \n\t\r\n \n \t\r\n \r\t\n\t \r\n\t\r \n\r \t\n\r\t \r \t\n\r \n\t\r\t \n\r\t\n \r\n \t\r\n\t' + local str = ('%s{%s"key"%s:%s[%s"val"%s,%s"val2"%s]%s,%s"key2"%s:%s1%s}%s'):gsub('%%s', s) + eq({key={'val', 'val2'}, key2=1}, funcs.json_decode(str)) + end) end) describe('json_encode() function', function() From 4f8b6864350c3aac5427103e27c856d1782b1be1 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sat, 12 Mar 2016 20:54:19 +0300 Subject: [PATCH 70/82] documentation,functests: State that UTF-8-only support is intentional --- runtime/doc/eval.txt | 13 +++++++++++++ test/functional/eval/json_functions_spec.lua | 8 ++++++++ 2 files changed, 21 insertions(+) diff --git a/runtime/doc/eval.txt b/runtime/doc/eval.txt index d171dacad1..b1485f1195 100644 --- a/runtime/doc/eval.txt +++ b/runtime/doc/eval.txt @@ -4327,6 +4327,13 @@ json_decode({expr}) *json_decode()* dictionary and for string will be emitted in case string with NUL byte was a dictionary key. + Note: function treats its input as UTF-8 always regardless of + 'encoding' value. This is needed because JSON source is + supposed to be external (e.g. |readfile()|) and JSON standard + allows only a few encodings, of which UTF-8 is recommended and + the only one required to be supported. Non-UTF-8 characters + are an error. + json_encode({expr}) *json_encode()* Convert {expr} into a JSON string. Accepts |msgpack-special-dict| as the input. Converts from 'encoding' @@ -4341,6 +4348,12 @@ json_encode({expr}) *json_encode()* Non-printable characters are converted into "\u1234" escapes or special escapes like "\t", other are dumped as-is. + Note: all characters above U+0079 are considered non-printable + when 'encoding' is not UTF-8. This function always outputs + UTF-8 strings as required by the standard thus when 'encoding' + is not unicode resulting string will look incorrect if + "\u1234" notation is not used. + keys({dict}) *keys()* Return a |List| with all the keys of {dict}. The |List| is in arbitrary order. diff --git a/test/functional/eval/json_functions_spec.lua b/test/functional/eval/json_functions_spec.lua index 8483152dbf..bed9d668fa 100644 --- a/test/functional/eval/json_functions_spec.lua +++ b/test/functional/eval/json_functions_spec.lua @@ -521,6 +521,14 @@ describe('json_decode() function', function() local str = ('%s{%s"key"%s:%s[%s"val"%s,%s"val2"%s]%s,%s"key2"%s:%s1%s}%s'):gsub('%%s', s) eq({key={'val', 'val2'}, key2=1}, funcs.json_decode(str)) end) + + it('always treats input as UTF-8', function() + -- When &encoding is latin1 string "«" is U+00C2 U+00AB U+00C2: «Â. So if + -- '"«"' was parsed as latin1 json_decode would return three characters, and + -- only one U+00AB when this string is parsed as latin1. + restart('set encoding=latin1') + eq(('%c'):format(0xAB), funcs.json_decode('"«"')) + end) end) describe('json_encode() function', function() From 25bb08dad930a975010798c592b5277d1b5a18b0 Mon Sep 17 00:00:00 2001 From: ZyX Date: Thu, 17 Mar 2016 23:31:31 +0300 Subject: [PATCH 71/82] api: Add warnings about the returned value to \*del_var functions --- src/nvim/api/buffer.c | 5 ++++- src/nvim/api/private/defs.h | 1 - src/nvim/api/tabpage.c | 5 ++++- src/nvim/api/vim.c | 5 ++++- src/nvim/api/window.c | 5 ++++- 5 files changed, 16 insertions(+), 5 deletions(-) diff --git a/src/nvim/api/buffer.c b/src/nvim/api/buffer.c index a01188f98c..55b535c78c 100644 --- a/src/nvim/api/buffer.c +++ b/src/nvim/api/buffer.c @@ -449,7 +449,10 @@ Object buffer_set_var(Buffer buffer, String name, Object value, Error *err) /// @param buffer The buffer handle /// @param name The variable name /// @param[out] err Details of an error that may have occurred -/// @return The old value +/// @return The old value or nil if there was no previous value. +/// +/// @warning It may return nil if there was no previous value +/// or if previous value was `v:null`. Object buffer_del_var(Buffer buffer, String name, Error *err) { buf_T *buf = find_buffer_by_handle(buffer, err); diff --git a/src/nvim/api/private/defs.h b/src/nvim/api/private/defs.h index 6c8e324649..fbfa87d5ae 100644 --- a/src/nvim/api/private/defs.h +++ b/src/nvim/api/private/defs.h @@ -99,4 +99,3 @@ struct key_value_pair { #endif // NVIM_API_PRIVATE_DEFS_H - diff --git a/src/nvim/api/tabpage.c b/src/nvim/api/tabpage.c index 3148a4fab7..c8311b0aa0 100644 --- a/src/nvim/api/tabpage.c +++ b/src/nvim/api/tabpage.c @@ -80,7 +80,10 @@ Object tabpage_set_var(Tabpage tabpage, String name, Object value, Error *err) /// @param tabpage handle /// @param name The variable name /// @param[out] err Details of an error that may have occurred -/// @return The tab page handle +/// @return The old value or nil if there was no previous value. +/// +/// @warning It may return nil if there was no previous value +/// or if previous value was `v:null`. Object tabpage_del_var(Tabpage tabpage, String name, Error *err) { tabpage_T *tab = find_tab_by_handle(tabpage, err); diff --git a/src/nvim/api/vim.c b/src/nvim/api/vim.c index dcae08d24c..10110b0f62 100644 --- a/src/nvim/api/vim.c +++ b/src/nvim/api/vim.c @@ -349,7 +349,10 @@ Object vim_set_var(String name, Object value, Error *err) /// /// @param name The variable name /// @param[out] err Details of an error that may have occurred -/// @return the old value if any +/// @return The old value or nil if there was no previous value. +/// +/// @warning It may return nil if there was no previous value +/// or if previous value was `v:null`. Object vim_del_var(String name, Error *err) { return dict_set_value(&globvardict, name, NIL, true, err); diff --git a/src/nvim/api/window.c b/src/nvim/api/window.c index cca1e3bdd3..a52f53a3e6 100644 --- a/src/nvim/api/window.c +++ b/src/nvim/api/window.c @@ -223,7 +223,10 @@ Object window_set_var(Window window, String name, Object value, Error *err) /// @param window The window handle /// @param name The variable name /// @param[out] err Details of an error that may have occurred -/// @return The old value +/// @return The old value or nil if there was no previous value. +/// +/// @warning It may return nil if there was no previous value +/// or if previous value was `v:null`. Object window_del_var(Window window, String name, Error *err) { win_T *win = find_window_by_handle(window, err); From 9709cf2cdbd98403aede9edbc8bbe435aeefb463 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 20 Mar 2016 19:08:42 +0300 Subject: [PATCH 72/82] documentation: Update assert_{false,true}() and empty() documentation Also removes a note regarding the performance of `empty(long_list)` vs `len(long_list) == 0` because this has nothing to do with the actual state: first checks that list is not NULL and pointer to its first element is also not NULL, second gets length by comparing list with NULL and falls back to `tv->vval.v_list->lv_len` if not. `len(long_list)` *may* still be *slightly* slower, but the slow down has nothing to do with the length of the list, is hardly noticeable and depends on how good compiler is at inlining and what exactly have author of the plugin written (I mean `len(long_list) == 0` vs `empty(long_list)` vs `!len(long_list)`). --- runtime/doc/eval.txt | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/runtime/doc/eval.txt b/runtime/doc/eval.txt index b1485f1195..da65708ac0 100644 --- a/runtime/doc/eval.txt +++ b/runtime/doc/eval.txt @@ -2245,17 +2245,17 @@ assert_equal({expected}, {actual}, [, {msg}]) assert_false({actual}, [, {msg}]) *assert_false()* When {actual} is not false an error message is added to - |v:errors|, like with |assert_equal()|.. - A value is false when it is zero. When "{actual}" is not a - number the assert fails. + |v:errors|, like with |assert_equal()|. + A value is false when it is zero or |v:false|. When "{actual}" + is not a number or |v:false| the assert fails. When {msg} is omitted an error in the form "Expected False but got {actual}" is produced. assert_true({actual}, [, {msg}]) *assert_true()* When {actual} is not true an error message is added to - |v:errors|, like with |assert_equal()|.. - A value is true when it is a non-zeron number. When {actual} - is not a number the assert fails. + |v:errors|, like with |assert_equal()|. + A value is true when it is a non-zero number or |v:true|. + When {actual} is not a number or |v:true| the assert fails. When {msg} is omitted an error in the form "Expected True but got {actual}" is produced. @@ -2849,9 +2849,8 @@ diff_hlID({lnum}, {col}) *diff_hlID()* empty({expr}) *empty()* Return the Number 1 if {expr} is empty, zero otherwise. A |List| or |Dictionary| is empty when it does not have any - items. A Number is empty when its value is zero. - For a long |List| this is much faster than comparing the - length with zero. + items. A Number is empty when its value is zero. Special + variable is empty when it is |v:false| or |v:null|. escape({string}, {chars}) *escape()* Escape the characters in {chars} that occur in {string} with a From af7ff808c73110f04aadaaab72eac6307dae0cc2 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 20 Mar 2016 19:55:59 +0300 Subject: [PATCH 73/82] eval: Fix overflow in error message in f_json_decode --- src/nvim/eval.c | 2 +- test/functional/eval/json_functions_spec.lua | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/nvim/eval.c b/src/nvim/eval.c index 9d1baec33f..bec234bb22 100644 --- a/src/nvim/eval.c +++ b/src/nvim/eval.c @@ -11618,7 +11618,7 @@ static void f_json_decode(typval_T *argvars, typval_T *rettv) return; } if (json_decode_string(s, len, rettv) == FAIL) { - EMSG2(_("E474: Failed to parse %s"), s); + emsgf(_("E474: Failed to parse %.*s"), (int) len, s); rettv->v_type = VAR_NUMBER; rettv->vval.v_number = 0; } diff --git a/test/functional/eval/json_functions_spec.lua b/test/functional/eval/json_functions_spec.lua index bed9d668fa..091b6f5457 100644 --- a/test/functional/eval/json_functions_spec.lua +++ b/test/functional/eval/json_functions_spec.lua @@ -6,6 +6,7 @@ local eq = helpers.eq local eval = helpers.eval local execute = helpers.execute local exc_exec = helpers.exc_exec +local redir_exec = helpers.redir_exec describe('json_decode() function', function() local restart = function(cmd) @@ -529,6 +530,13 @@ describe('json_decode() function', function() restart('set encoding=latin1') eq(('%c'):format(0xAB), funcs.json_decode('"«"')) end) + + it('does not overflow when writing error message about decoding ["", ""]', + function() + eq('\nE474: Attempt to decode a blank string' + .. '\nE474: Failed to parse \n', + redir_exec('call json_decode(["", ""])')) + end) end) describe('json_encode() function', function() From 9af400f97916851ecd86975118faea2a8c68598f Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 20 Mar 2016 20:03:12 +0300 Subject: [PATCH 74/82] eval: Treat [] and [""] as any other empty string --- src/nvim/eval.c | 9 ++++++--- test/functional/eval/json_functions_spec.lua | 4 ++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/nvim/eval.c b/src/nvim/eval.c index bec234bb22..e23702ac16 100644 --- a/src/nvim/eval.c +++ b/src/nvim/eval.c @@ -11608,15 +11608,18 @@ static void f_json_decode(typval_T *argvars, typval_T *rettv) return; } tofree = s; + if (s == NULL) { + assert(len == 0); + s = ""; + } } else { s = (char *) get_tv_string_buf_chk(&argvars[0], (char_u *) numbuf); if (s) { len = strlen(s); + } else { + return; } } - if (s == NULL) { - return; - } if (json_decode_string(s, len, rettv) == FAIL) { emsgf(_("E474: Failed to parse %.*s"), (int) len, s); rettv->v_type = VAR_NUMBER; diff --git a/test/functional/eval/json_functions_spec.lua b/test/functional/eval/json_functions_spec.lua index 091b6f5457..0b1862fa8b 100644 --- a/test/functional/eval/json_functions_spec.lua +++ b/test/functional/eval/json_functions_spec.lua @@ -507,6 +507,10 @@ describe('json_decode() function', function() it('fails to parse empty string', function() eq('Vim(call):E474: Attempt to decode a blank string', exc_exec('call json_decode("")')) + eq('Vim(call):E474: Attempt to decode a blank string', + exc_exec('call json_decode([])')) + eq('Vim(call):E474: Attempt to decode a blank string', + exc_exec('call json_decode([""])')) eq('Vim(call):E474: Attempt to decode a blank string', exc_exec('call json_decode(" ")')) eq('Vim(call):E474: Attempt to decode a blank string', From 494b1c9beef3755916048df29755d3d014902191 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 20 Mar 2016 21:31:49 +0300 Subject: [PATCH 75/82] *: Make set_vim_var_\* functions have proper argument types --- src/nvim/diff.c | 65 +++++++------- src/nvim/edit.c | 34 +++---- src/nvim/eval.c | 165 +++++++++++++++++++--------------- src/nvim/eval/decode.c | 4 +- src/nvim/ex_cmds.c | 105 +++++++++++----------- src/nvim/ex_cmds2.c | 14 +-- src/nvim/ex_docmd.c | 58 ++++++------ src/nvim/ex_eval.c | 38 ++++---- src/nvim/fileio.c | 58 ++++++------ src/nvim/fold.c | 12 +-- src/nvim/hardcopy.c | 8 +- src/nvim/indent.c | 2 +- src/nvim/main.c | 13 +-- src/nvim/memline.c | 2 +- src/nvim/message.c | 38 ++++---- src/nvim/misc1.c | 2 +- src/nvim/misc2.c | 5 +- src/nvim/msgpack_rpc/server.c | 2 +- src/nvim/normal.c | 11 ++- src/nvim/option.c | 47 +++++----- src/nvim/path.c | 15 ++-- src/nvim/quickfix.c | 38 ++++---- src/nvim/tag.c | 2 +- 23 files changed, 381 insertions(+), 357 deletions(-) diff --git a/src/nvim/diff.c b/src/nvim/diff.c index 1149ca1e62..a3063de869 100644 --- a/src/nvim/diff.c +++ b/src/nvim/diff.c @@ -658,9 +658,9 @@ void ex_diffupdate(exarg_T *eap) } // We need three temp file names. - char_u *tmp_orig = vim_tempname(); - char_u *tmp_new = vim_tempname(); - char_u *tmp_diff = vim_tempname(); + char *tmp_orig = (char *) vim_tempname(); + char *tmp_new = (char *) vim_tempname(); + char *tmp_diff = (char *) vim_tempname(); if ((tmp_orig == NULL) || (tmp_new == NULL) || (tmp_diff == NULL)) { goto theend; @@ -670,11 +670,11 @@ void ex_diffupdate(exarg_T *eap) // are no differences. Can't use the return value, it's non-zero when // there are differences. // May try twice, first with "-a" and then without. - int io_error = FALSE; - int ok = FALSE; + int io_error = false; + bool ok = false; for (;;) { - ok = FALSE; - FILE *fd = mch_fopen((char *)tmp_orig, "w"); + ok = false; + FILE *fd = mch_fopen(tmp_orig, "w"); if (fd == NULL) { io_error = TRUE; @@ -683,7 +683,7 @@ void ex_diffupdate(exarg_T *eap) io_error = TRUE; } fclose(fd); - fd = mch_fopen((char *)tmp_new, "w"); + fd = mch_fopen(tmp_new, "w"); if (fd == NULL) { io_error = TRUE; @@ -693,7 +693,7 @@ void ex_diffupdate(exarg_T *eap) } fclose(fd); diff_file(tmp_orig, tmp_new, tmp_diff); - fd = mch_fopen((char *)tmp_diff, "r"); + fd = mch_fopen(tmp_diff, "r"); if (fd == NULL) { io_error = TRUE; @@ -712,10 +712,10 @@ void ex_diffupdate(exarg_T *eap) } fclose(fd); } - os_remove((char *)tmp_diff); - os_remove((char *)tmp_new); + os_remove(tmp_diff); + os_remove(tmp_new); } - os_remove((char *)tmp_orig); + os_remove(tmp_orig); } // When using 'diffexpr' break here. @@ -756,7 +756,7 @@ void ex_diffupdate(exarg_T *eap) // Write the first buffer to a tempfile. buf_T *buf = curtab->tp_diffbuf[idx_orig]; - if (diff_write(buf, tmp_orig) == FAIL) { + if (diff_write(buf, (char_u *) tmp_orig) == FAIL) { goto theend; } @@ -767,17 +767,17 @@ void ex_diffupdate(exarg_T *eap) continue; // skip buffer that isn't loaded } - if (diff_write(buf, tmp_new) == FAIL) { + if (diff_write(buf, (char_u *) tmp_new) == FAIL) { continue; } diff_file(tmp_orig, tmp_new, tmp_diff); // Read the diff output and add each entry to the diff list. - diff_read(idx_orig, idx_new, tmp_diff); - os_remove((char *)tmp_diff); - os_remove((char *)tmp_new); + diff_read(idx_orig, idx_new, (char_u *) tmp_diff); + os_remove(tmp_diff); + os_remove(tmp_new); } - os_remove((char *)tmp_orig); + os_remove(tmp_orig); // force updating cursor position on screen curwin->w_valid_cursor.lnum = 0; @@ -795,15 +795,16 @@ theend: /// @param tmp_orig /// @param tmp_new /// @param tmp_diff -static void diff_file(char_u *tmp_orig, char_u *tmp_new, char_u *tmp_diff) +static void diff_file(const char *const tmp_orig, const char *const tmp_new, + const char *const tmp_diff) { if (*p_dex != NUL) { // Use 'diffexpr' to generate the diff file. eval_diff(tmp_orig, tmp_new, tmp_diff); } else { - size_t len = STRLEN(tmp_orig) + STRLEN(tmp_new) + STRLEN(tmp_diff) - + STRLEN(p_srr) + 27; - char_u *cmd = xmalloc(len); + const size_t len = (strlen(tmp_orig) + strlen(tmp_new) + strlen(tmp_diff) + + STRLEN(p_srr) + 27); + char *const cmd = xmalloc(len); /* We don't want $DIFF_OPTIONS to get in the way. */ if (os_getenv("DIFF_OPTIONS")) { @@ -813,19 +814,17 @@ static void diff_file(char_u *tmp_orig, char_u *tmp_new, char_u *tmp_diff) /* Build the diff command and execute it. Always use -a, binary * differences are of no use. Ignore errors, diff returns * non-zero when differences have been found. */ - vim_snprintf((char *)cmd, len, "diff %s%s%s%s%s %s", - diff_a_works == FALSE ? "" : "-a ", + vim_snprintf(cmd, len, "diff %s%s%s%s%s %s", + diff_a_works ? "-a " : "", "", (diff_flags & DIFF_IWHITE) ? "-b " : "", (diff_flags & DIFF_ICASE) ? "-i " : "", tmp_orig, tmp_new); - append_redir(cmd, len, p_srr, tmp_diff); + append_redir(cmd, len, (char *) p_srr, tmp_diff); block_autocmds(); // Avoid ShellCmdPost stuff - (void)call_shell( - cmd, - kShellOptFilter | kShellOptSilent | kShellOptDoOut, - NULL - ); + (void)call_shell((char_u *) cmd, + kShellOptFilter | kShellOptSilent | kShellOptDoOut, + NULL); unblock_autocmds(); xfree(cmd); } @@ -902,9 +901,11 @@ void ex_diffpatch(exarg_T *eap) if (*p_pex != NUL) { // Use 'patchexpr' to generate the new file. #ifdef UNIX - eval_patch(tmp_orig, fullname != NULL ? fullname : eap->arg, tmp_new); + eval_patch((char *) tmp_orig, + (char *) (fullname != NULL ? fullname : eap->arg), + (char *) tmp_new); #else - eval_patch(tmp_orig, eap->arg, tmp_new); + eval_patch((char *) tmp_orig, (char *) eap->arg, (char *) tmp_new); #endif // ifdef UNIX } else { // Build the patch command and execute it. Ignore errors. Switch to diff --git a/src/nvim/edit.c b/src/nvim/edit.c index 614a5d43be..36c88df570 100644 --- a/src/nvim/edit.c +++ b/src/nvim/edit.c @@ -199,7 +199,7 @@ typedef struct insert_state { int did_restart_edit; // remember if insert mode was restarted // after a ctrl+o bool nomove; - uint8_t *ptr; + char_u *ptr; } InsertState; @@ -270,8 +270,8 @@ static void insert_enter(InsertState *s) s->ptr = (char_u *)"i"; } - set_vim_var_string(VV_INSERTMODE, s->ptr, 1); - set_vim_var_string(VV_CHAR, NULL, -1); /* clear v:char */ + set_vim_var_string(VV_INSERTMODE, (char *) s->ptr, 1); + set_vim_var_string(VV_CHAR, NULL, -1); apply_autocmds(EVENT_INSERTENTER, NULL, NULL, false, curbuf); // Make sure the cursor didn't move. Do call check_cursor_col() in @@ -7239,15 +7239,15 @@ static void ins_insert(int replaceState) return; } - set_vim_var_string(VV_INSERTMODE, - (char_u *)((State & REPLACE_FLAG) ? "i" : - replaceState == VREPLACE ? "v" : - "r"), 1); - apply_autocmds(EVENT_INSERTCHANGE, NULL, NULL, FALSE, curbuf); - if (State & REPLACE_FLAG) + set_vim_var_string(VV_INSERTMODE, ((State & REPLACE_FLAG) ? "i" : + replaceState == VREPLACE ? "v" : + "r"), 1); + apply_autocmds(EVENT_INSERTCHANGE, NULL, NULL, false, curbuf); + if (State & REPLACE_FLAG) { State = INSERT | (State & LANGMAP); - else + } else { State = replaceState | (State & LANGMAP); + } AppendCharToRedobuff(K_INS); showmode(); ui_cursor_shape(); /* may show different cursor shape */ @@ -8480,22 +8480,22 @@ static colnr_T get_nolist_virtcol(void) */ static char_u *do_insert_char_pre(int c) { - char_u buf[MB_MAXBYTES + 1]; + char buf[MB_MAXBYTES + 1]; // Return quickly when there is nothing to do. if (!has_event(EVENT_INSERTCHARPRE)) { return NULL; } if (has_mbyte) { - buf[(*mb_char2bytes)(c, buf)] = NUL; + buf[(*mb_char2bytes)(c, (char_u *) buf)] = NUL; } else { buf[0] = c; buf[1] = NUL; } - /* Lock the text to avoid weird things from happening. */ - ++textlock; - set_vim_var_string(VV_CHAR, buf, -1); /* set v:char */ + // Lock the text to avoid weird things from happening. + textlock++; + set_vim_var_string(VV_CHAR, buf, -1); char_u *res = NULL; if (apply_autocmds(EVENT_INSERTCHARPRE, NULL, NULL, FALSE, curbuf)) { @@ -8506,8 +8506,8 @@ static char_u *do_insert_char_pre(int c) res = vim_strsave(get_vim_var_str(VV_CHAR)); } - set_vim_var_string(VV_CHAR, NULL, -1); /* clear v:char */ - --textlock; + set_vim_var_string(VV_CHAR, NULL, -1); + textlock--; return res; } diff --git a/src/nvim/eval.c b/src/nvim/eval.c index e23702ac16..204c0fb1ef 100644 --- a/src/nvim/eval.c +++ b/src/nvim/eval.c @@ -465,6 +465,8 @@ const list_T *eval_msgpack_type_lists[] = { */ void eval_init(void) { + vimvars[VV_VERSION].vv_nr = VIM_VERSION_100; + jobs = pmap_new(uint64_t)(); struct vimvar *p; @@ -768,45 +770,50 @@ void var_redir_stop(void) redir_varname = NULL; } -int eval_charconvert(char_u *enc_from, char_u *enc_to, char_u *fname_from, char_u *fname_to) +int eval_charconvert(const char *const enc_from, const char *const enc_to, + const char *const fname_from, const char *const fname_to) { - int err = FALSE; + int err = false; set_vim_var_string(VV_CC_FROM, enc_from, -1); set_vim_var_string(VV_CC_TO, enc_to, -1); set_vim_var_string(VV_FNAME_IN, fname_from, -1); set_vim_var_string(VV_FNAME_OUT, fname_to, -1); - if (eval_to_bool(p_ccv, &err, NULL, FALSE)) - err = TRUE; + if (eval_to_bool(p_ccv, &err, NULL, false)) { + err = true; + } set_vim_var_string(VV_CC_FROM, NULL, -1); set_vim_var_string(VV_CC_TO, NULL, -1); set_vim_var_string(VV_FNAME_IN, NULL, -1); set_vim_var_string(VV_FNAME_OUT, NULL, -1); - if (err) - return FAIL; - return OK; -} - -int eval_printexpr(char_u *fname, char_u *args) -{ - int err = FALSE; - - set_vim_var_string(VV_FNAME_IN, fname, -1); - set_vim_var_string(VV_CMDARG, args, -1); - if (eval_to_bool(p_pexpr, &err, NULL, FALSE)) - err = TRUE; - set_vim_var_string(VV_FNAME_IN, NULL, -1); - set_vim_var_string(VV_CMDARG, NULL, -1); - if (err) { - os_remove((char *)fname); return FAIL; } return OK; } -void eval_diff(char_u *origfile, char_u *newfile, char_u *outfile) +int eval_printexpr(const char *const fname, const char *const args) +{ + int err = false; + + set_vim_var_string(VV_FNAME_IN, fname, -1); + set_vim_var_string(VV_CMDARG, args, -1); + if (eval_to_bool(p_pexpr, &err, NULL, false)) { + err = true; + } + set_vim_var_string(VV_FNAME_IN, NULL, -1); + set_vim_var_string(VV_CMDARG, NULL, -1); + + if (err) { + os_remove(fname); + return FAIL; + } + return OK; +} + +void eval_diff(const char *const origfile, const char *const newfile, + const char *const outfile) { int err = FALSE; @@ -819,7 +826,8 @@ void eval_diff(char_u *origfile, char_u *newfile, char_u *outfile) set_vim_var_string(VV_FNAME_OUT, NULL, -1); } -void eval_patch(char_u *origfile, char_u *difffile, char_u *outfile) +void eval_patch(const char *const origfile, const char *const difffile, + const char *const outfile) { int err; @@ -17083,20 +17091,6 @@ static int eval_isnamec1(int c) return ASCII_ISALPHA(c) || c == '_'; } -/* - * Set number v: variable to "val". - */ -void set_vim_var_nr(int idx, long val) -{ - vimvars[idx].vv_nr = val; -} - -/// Set special v: variable to "val" -void set_vim_var_special(const int idx, const SpecialVarValue val) -{ - vimvars[idx].vv_special = val; -} - /* * Get number v: variable value. */ @@ -17134,11 +17128,11 @@ dict_T *get_vim_var_dict(int idx) FUNC_ATTR_PURE */ void set_vim_var_char(int c) { - char_u buf[MB_MAXBYTES + 1]; + char buf[MB_MAXBYTES + 1]; - if (has_mbyte) - buf[(*mb_char2bytes)(c, buf)] = NUL; - else { + if (has_mbyte) { + buf[(*mb_char2bytes)(c, (char_u *) buf)] = NUL; + } else { buf[0] = c; buf[1] = NUL; } @@ -17157,47 +17151,68 @@ void set_vcount(long count, long count1, int set_prevcount) vimvars[VV_COUNT1].vv_nr = count1; } -/* - * Set string v: variable to a copy of "val". - */ -void set_vim_var_string ( - int idx, - char_u *val, - int len /* length of "val" to use or -1 (whole string) */ -) +/// Set number v: variable to the given value +/// +/// @param[in] idx Index of variable to set. +/// @param[in] val Value to set to. +void set_vim_var_nr(const VimVarIndex idx, const varnumber_T val) { - /* Need to do this (at least) once, since we can't initialize a union. - * Will always be invoked when "v:progname" is set. */ - vimvars[VV_VERSION].vv_nr = VIM_VERSION_100; - - xfree(vimvars[idx].vv_str); - if (val == NULL) - vimvars[idx].vv_str = NULL; - else if (len == -1) - vimvars[idx].vv_str = vim_strsave(val); - else - vimvars[idx].vv_str = vim_strnsave(val, len); + vimvars[idx].vv_nr = val; } -/* - * Set List v: variable to "val". - */ -void set_vim_var_list(int idx, list_T *val) +/// Set special v: variable to the given value +/// +/// @param[in] idx Index of variable to set. +/// @param[in] val Value to set to. +void set_vim_var_special(const VimVarIndex idx, const SpecialVarValue val) +{ + vimvars[idx].vv_special = val; +} + +/// Set string v: variable to the given string +/// +/// @param[in] idx Index of variable to set. +/// @param[in] val Value to set to. Will be copied. +/// @param[in] len Legth of that value or -1 in which case strlen() will be +/// used. +void set_vim_var_string(const VimVarIndex idx, const char *const val, + const ptrdiff_t len) +{ + xfree(vimvars[idx].vv_str); + if (val == NULL) { + vimvars[idx].vv_str = NULL; + } else if (len == -1) { + vimvars[idx].vv_str = (char_u *) xstrdup(val); + } else { + vimvars[idx].vv_str = (char_u *) xstrndup(val, (size_t) len); + } +} + +/// Set list v: variable to the given list +/// +/// @param[in] idx Index of variable to set. +/// @param[in,out] val Value to set to. Reference count will be incremented. +void set_vim_var_list(const VimVarIndex idx, list_T *const val) { list_unref(vimvars[idx].vv_list); vimvars[idx].vv_list = val; - if (val != NULL) - ++val->lv_refcount; + if (val != NULL) { + val->lv_refcount++; + } } -/// Set Dictionary v: variable to "val". -void set_vim_var_dict(int idx, dict_T *val) +/// Set Dictionary v: variable to the given dictionary +/// +/// @param[in] idx Index of variable to set. +/// @param[in,out] val Value to set to. Reference count will be incremented. +/// Also keys of the dictionary will be made read-only. +void set_vim_var_dict(const VimVarIndex idx, dict_T *const val) { dict_unref(vimvars[idx].vv_dict); vimvars[idx].vv_dict = val; if (val != NULL) { - ++val->dv_refcount; + val->dv_refcount++; // Set readonly dict_set_keys_readonly(val); } @@ -17208,15 +17223,17 @@ void set_vim_var_dict(int idx, dict_T *val) */ void set_reg_var(int c) { - char_u regname; + char regname; - if (c == 0 || c == ' ') + if (c == 0 || c == ' ') { regname = '"'; - else + } else { regname = c; - /* Avoid free/alloc when the value is already right. */ - if (vimvars[VV_REG].vv_str == NULL || vimvars[VV_REG].vv_str[0] != c) + } + // Avoid free/alloc when the value is already right. + if (vimvars[VV_REG].vv_str == NULL || vimvars[VV_REG].vv_str[0] != c) { set_vim_var_string(VV_REG, ®name, 1); + } } /* diff --git a/src/nvim/eval/decode.c b/src/nvim/eval/decode.c index 10dd36c137..0774ef515f 100644 --- a/src/nvim/eval/decode.c +++ b/src/nvim/eval/decode.c @@ -575,8 +575,8 @@ parse_json_number_check: tv.vval.v_number = (varnumber_T) nr; } if (json_decoder_pop(OBJ(tv, false, *didcomma, *didcolon), - stack, container_stack, - &p, next_map_special, didcomma, didcolon) == FAIL) { + stack, container_stack, + &p, next_map_special, didcomma, didcolon) == FAIL) { goto parse_json_number_fail; } if (*next_map_special) { diff --git a/src/nvim/ex_cmds.c b/src/nvim/ex_cmds.c index 5ea5beb478..d344daed11 100644 --- a/src/nvim/ex_cmds.c +++ b/src/nvim/ex_cmds.c @@ -1326,15 +1326,17 @@ char_u *make_filter_cmd(char_u *cmd, char_u *itmp, char_u *otmp) #endif size_t len = STRLEN(cmd) + 1; // At least enough space for cmd + NULL. - + len += is_fish_shell ? sizeof("begin; ""; end") - 1 : sizeof("("")") - 1; - if (itmp != NULL) + if (itmp != NULL) { len += STRLEN(itmp) + sizeof(" { "" < "" } ") - 1; - if (otmp != NULL) + } + if (otmp != NULL) { len += STRLEN(otmp) + STRLEN(p_srr) + 2; // two extra spaces (" "), - char_u *buf = xmalloc(len); + } + char *const buf = xmalloc(len); #if defined(UNIX) // Put delimiters around the command (for concatenated commands) when @@ -1342,19 +1344,19 @@ char_u *make_filter_cmd(char_u *cmd, char_u *itmp, char_u *otmp) if (itmp != NULL || otmp != NULL) { char *fmt = is_fish_shell ? "begin; %s; end" : "(%s)"; - vim_snprintf((char *)buf, len, fmt, (char *)cmd); + vim_snprintf(buf, len, fmt, (char *)cmd); } else { - STRCPY(buf, cmd); + strncpy(buf, (char *) cmd, len); } if (itmp != NULL) { - STRCAT(buf, " < "); - STRCAT(buf, itmp); + strncat(buf, " < ", len); + strncat(buf, (char *) itmp, len); } #else // For shells that don't understand braces around commands, at least allow // the use of commands in a pipe. - STRCPY(buf, cmd); + strncpy(buf, cmd, len); if (itmp != NULL) { char_u *p; @@ -1362,55 +1364,56 @@ char_u *make_filter_cmd(char_u *cmd, char_u *itmp, char_u *otmp) // Don't do this when 'shellquote' is not empty, otherwise the // redirection would be inside the quotes. if (*p_shq == NUL) { - p = vim_strchr(buf, '|'); - if (p != NULL) - *p = NUL; - } - STRCAT(buf, " < "); - STRCAT(buf, itmp); - if (*p_shq == NUL) { - p = vim_strchr(cmd, '|'); + p = strchr(buf, '|'); if (p != NULL) { - STRCAT(buf, " "); // Insert a space before the '|' for DOS - STRCAT(buf, p); + *p = NUL; + } + } + strncat(buf, " < ", len); + strncat(buf, (char *) itmp, len); + if (*p_shq == NUL) { + p = strchr(cmd, '|'); + if (p != NULL) { + strncat(buf, " ", len); // Insert a space before the '|' for DOS + strncat(buf, p, len); } } } #endif if (otmp != NULL) { - append_redir(buf, len, p_srr, otmp); + append_redir(buf, len, (char *) p_srr, (char *) otmp); } - return buf; + return (char_u *) buf; } -/* - * Append output redirection for file "fname" to the end of string buffer - * "buf[buflen]" - * Works with the 'shellredir' and 'shellpipe' options. - * The caller should make sure that there is enough room: - * STRLEN(opt) + STRLEN(fname) + 3 - */ -void append_redir(char_u *buf, size_t buflen, char_u *opt, char_u *fname) +/// Append output redirection for the given file to the end of the buffer +/// +/// @param[out] buf Buffer to append to. +/// @param[in] buflen Buffer length. +/// @param[in] opt Separator or format string to append: will append +/// `printf(' ' . opt, fname)` if `%s` is found in `opt` or +/// a space, opt, a space and then fname if `%s` is not found +/// there. +/// @param[in] fname File name to append. +void append_redir(char *const buf, const size_t buflen, + const char *const opt, const char *const fname) { - char_u *p; - char_u *end; - - end = buf + STRLEN(buf); - /* find "%s" */ - for (p = opt; (p = vim_strchr(p, '%')) != NULL; ++p) { - if (p[1] == 's') /* found %s */ + char *const end = buf + strlen(buf); + // find "%s" + const char *p = opt; + for (; (p = strchr(p, '%')) != NULL; p++) { + if (p[1] == 's') { // found %s break; - if (p[1] == '%') /* skip %% */ - ++p; + } else if (p[1] == '%') { // skip %% + p++; + } } if (p != NULL) { - *end = ' '; /* not really needed? Not with sh, ksh or bash */ - vim_snprintf((char *)end + 1, (size_t)(buflen - (end + 1 - buf)), - (char *)opt, (char *)fname); - } else - vim_snprintf((char *)end, (size_t)(buflen - (end - buf)), - " %s %s", - (char *)opt, (char *)fname); + *end = ' '; // not really needed? Not with sh, ksh or bash + vim_snprintf(end + 1, (size_t) (buflen - (end + 1 - buf)), opt, fname); + } else { + vim_snprintf(end, (size_t) (buflen - (end - buf)), " %s %s", opt, fname); + } } void print_line_no_prefix(linenr_T lnum, int use_number, int list) @@ -2093,15 +2096,13 @@ do_ecmd ( if ((command != NULL || newlnum > (linenr_T)0) && *get_vim_var_str(VV_SWAPCOMMAND) == NUL) { - char_u *p; - - /* Set v:swapcommand for the SwapExists autocommands. */ - size_t len = (command != NULL) ? STRLEN(command) + 3 : 30; - p = xmalloc(len); + // Set v:swapcommand for the SwapExists autocommands. + const size_t len = (command != NULL) ? STRLEN(command) + 3 : 30; + char *const p = xmalloc(len); if (command != NULL) { - vim_snprintf((char *)p, len, ":%s\r", command); + vim_snprintf(p, len, ":%s\r", command); } else { - vim_snprintf((char *)p, len, "%" PRId64 "G", (int64_t)newlnum); + vim_snprintf(p, len, "%" PRId64 "G", (int64_t)newlnum); } set_vim_var_string(VV_SWAPCOMMAND, p, -1); did_set_swapcommand = TRUE; diff --git a/src/nvim/ex_cmds2.c b/src/nvim/ex_cmds2.c index 71ea170e1c..2c8271c696 100644 --- a/src/nvim/ex_cmds2.c +++ b/src/nvim/ex_cmds2.c @@ -3168,27 +3168,27 @@ static char_u *get_mess_env(void) */ void set_lang_var(void) { - char_u *loc; + const char *loc; # ifdef HAVE_GET_LOCALE_VAL - loc = (char_u *)get_locale_val(LC_CTYPE); + loc = get_locale_val(LC_CTYPE); # else - /* setlocale() not supported: use the default value */ - loc = (char_u *)"C"; + // setlocale() not supported: use the default value + loc = "C"; # endif set_vim_var_string(VV_CTYPE, loc, -1); /* When LC_MESSAGES isn't defined use the value from $LC_MESSAGES, fall * back to LC_CTYPE if it's empty. */ # ifdef HAVE_WORKING_LIBINTL - loc = get_mess_env(); + loc = (char *) get_mess_env(); # else - loc = (char_u *)get_locale_val(LC_MESSAGES); + loc = get_locale_val(LC_MESSAGES); # endif set_vim_var_string(VV_LANG, loc, -1); # ifdef HAVE_GET_LOCALE_VAL - loc = (char_u *)get_locale_val(LC_TIME); + loc = get_locale_val(LC_TIME); # endif set_vim_var_string(VV_LC_TIME, loc, -1); } diff --git a/src/nvim/ex_docmd.c b/src/nvim/ex_docmd.c index a7e98e7f04..6391e023c3 100644 --- a/src/nvim/ex_docmd.c +++ b/src/nvim/ex_docmd.c @@ -7421,10 +7421,10 @@ static int mksession_nl = FALSE; /* use NL only in put_eol() */ static void ex_mkrc(exarg_T *eap) { FILE *fd; - int failed = FALSE; - int view_session = FALSE; - int using_vdir = FALSE; /* using 'viewdir'? */ - char_u *viewFile = NULL; + int failed = false; + int view_session = false; + int using_vdir = false; // using 'viewdir'? + char *viewFile = NULL; unsigned *flagp; if (eap->cmdidx == CMD_mksession || eap->cmdidx == CMD_mkview) { @@ -7435,32 +7435,34 @@ static void ex_mkrc(exarg_T *eap) * short file name when 'acd' is set, that is checked later. */ did_lcd = FALSE; - char_u *fname; - /* ":mkview" or ":mkview 9": generate file name with 'viewdir' */ + char *fname; + // ":mkview" or ":mkview 9": generate file name with 'viewdir' if (eap->cmdidx == CMD_mkview && (*eap->arg == NUL || (ascii_isdigit(*eap->arg) && eap->arg[1] == NUL))) { - eap->forceit = TRUE; - fname = (char_u *)get_view_file(*eap->arg); - if (fname == NULL) + eap->forceit = true; + fname = get_view_file(*eap->arg); + if (fname == NULL) { return; + } viewFile = fname; - using_vdir = TRUE; - } else if (*eap->arg != NUL) - fname = eap->arg; - else if (eap->cmdidx == CMD_mkvimrc) - fname = (char_u *)VIMRC_FILE; - else if (eap->cmdidx == CMD_mksession) - fname = (char_u *)SESSION_FILE; - else - fname = (char_u *)EXRC_FILE; + using_vdir = true; + } else if (*eap->arg != NUL) { + fname = (char *) eap->arg; + } else if (eap->cmdidx == CMD_mkvimrc) { + fname = VIMRC_FILE; + } else if (eap->cmdidx == CMD_mksession) { + fname = SESSION_FILE; + } else { + fname = EXRC_FILE; + } /* When using 'viewdir' may have to create the directory. */ if (using_vdir && !os_isdir(p_vdir)) { vim_mkdir_emsg(p_vdir, 0755); } - fd = open_exfile(fname, eap->forceit, WRITEBIN); + fd = open_exfile((char_u *) fname, eap->forceit, WRITEBIN); if (fd != NULL) { if (eap->cmdidx == CMD_mkview) flagp = &vop_flags; @@ -7504,8 +7506,9 @@ static void ex_mkrc(exarg_T *eap) || os_chdir((char *)dirnow) != 0) *dirnow = NUL; if (*dirnow != NUL && (ssop_flags & SSOP_SESDIR)) { - if (vim_chdirfile(fname) == OK) - shorten_fnames(TRUE); + if (vim_chdirfile((char_u *) fname) == OK) { + shorten_fnames(true); + } } else if (*dirnow != NUL && (ssop_flags & SSOP_CURDIR) && globaldir != NULL) { if (os_chdir((char *)globaldir) == 0) @@ -7550,15 +7553,14 @@ static void ex_mkrc(exarg_T *eap) failed |= fclose(fd); - if (failed) + if (failed) { EMSG(_(e_write)); - else if (eap->cmdidx == CMD_mksession) { - /* successful session write - set this_session var */ - char_u *tbuf; - - tbuf = xmalloc(MAXPATHL); - if (vim_FullName((char *)fname, (char *)tbuf, MAXPATHL, FALSE) == OK) + } else if (eap->cmdidx == CMD_mksession) { + // successful session write - set this_session var + char *const tbuf = xmalloc(MAXPATHL); + if (vim_FullName(fname, tbuf, MAXPATHL, false) == OK) { set_vim_var_string(VV_THIS_SESSION, tbuf, -1); + } xfree(tbuf); } #ifdef MKSESSION_NL diff --git a/src/nvim/ex_eval.c b/src/nvim/ex_eval.c index 6fc74c7ad0..41ad96a378 100644 --- a/src/nvim/ex_eval.c +++ b/src/nvim/ex_eval.c @@ -569,17 +569,19 @@ static void catch_exception(except_T *excp) { excp->caught = caught_stack; caught_stack = excp; - set_vim_var_string(VV_EXCEPTION, excp->value, -1); + set_vim_var_string(VV_EXCEPTION, (char *) excp->value, -1); if (*excp->throw_name != NUL) { - if (excp->throw_lnum != 0) + if (excp->throw_lnum != 0) { vim_snprintf((char *)IObuff, IOSIZE, _("%s, line %" PRId64), - excp->throw_name, (int64_t)excp->throw_lnum); - else + excp->throw_name, (int64_t)excp->throw_lnum); + } else { vim_snprintf((char *)IObuff, IOSIZE, "%s", excp->throw_name); - set_vim_var_string(VV_THROWPOINT, IObuff, -1); - } else - /* throw_name not set on an exception from a command that was typed. */ + } + set_vim_var_string(VV_THROWPOINT, (char *) IObuff, -1); + } else { + // throw_name not set on an exception from a command that was typed. set_vim_var_string(VV_THROWPOINT, NULL, -1); + } if (p_verbose >= 13 || debug_break_level > 0) { int save_msg_silent = msg_silent; @@ -614,20 +616,22 @@ static void finish_exception(except_T *excp) EMSG(_(e_internal)); caught_stack = caught_stack->caught; if (caught_stack != NULL) { - set_vim_var_string(VV_EXCEPTION, caught_stack->value, -1); + set_vim_var_string(VV_EXCEPTION, (char *) caught_stack->value, -1); if (*caught_stack->throw_name != NUL) { - if (caught_stack->throw_lnum != 0) + if (caught_stack->throw_lnum != 0) { vim_snprintf((char *)IObuff, IOSIZE, - _("%s, line %" PRId64), caught_stack->throw_name, - (int64_t)caught_stack->throw_lnum); - else + _("%s, line %" PRId64), caught_stack->throw_name, + (int64_t)caught_stack->throw_lnum); + } else { vim_snprintf((char *)IObuff, IOSIZE, "%s", - caught_stack->throw_name); - set_vim_var_string(VV_THROWPOINT, IObuff, -1); - } else - /* throw_name not set on an exception from a command that was - * typed. */ + caught_stack->throw_name); + } + set_vim_var_string(VV_THROWPOINT, (char *) IObuff, -1); + } else { + // throw_name not set on an exception from a command that was + // typed. set_vim_var_string(VV_THROWPOINT, NULL, -1); + } } else { set_vim_var_string(VV_EXCEPTION, NULL, -1); set_vim_var_string(VV_THROWPOINT, NULL, -1); diff --git a/src/nvim/fileio.c b/src/nvim/fileio.c index 383cd47dbe..f4fc744e04 100644 --- a/src/nvim/fileio.c +++ b/src/nvim/fileio.c @@ -2139,9 +2139,10 @@ readfile_charconvert ( else { close(*fdp); /* close the input file, ignore errors */ *fdp = -1; - if (eval_charconvert(fenc, enc_utf8 ? (char_u *)"utf-8" : p_enc, - fname, tmpname) == FAIL) + if (eval_charconvert((char *) fenc, enc_utf8 ? "utf-8" : (char *) p_enc, + (char *) fname, (char *) tmpname) == FAIL) { errmsg = (char_u *)_("Conversion with 'charconvert' failed"); + } if (errmsg == NULL && (*fdp = os_open((char *)tmpname, O_RDONLY, 0)) < 0) { errmsg = (char_u *)_("can't read output of 'charconvert'"); } @@ -3435,9 +3436,9 @@ restore_backup: * with 'charconvert' to (overwrite) the output file. */ if (end != 0) { - if (eval_charconvert(enc_utf8 ? (char_u *)"utf-8" : p_enc, fenc, - wfname, fname) == FAIL) { - write_info.bw_conv_error = TRUE; + if (eval_charconvert(enc_utf8 ? "utf-8" : (char *) p_enc, (char *) fenc, + (char *) wfname, (char *) fname) == FAIL) { + write_info.bw_conv_error = true; end = 0; } } @@ -4740,7 +4741,6 @@ buf_check_timestamp ( { int retval = 0; char_u *path; - char_u *tbuf; char *mesg = NULL; char *mesg2 = ""; int helpmesg = FALSE; @@ -4810,14 +4810,12 @@ buf_check_timestamp ( else reason = "time"; - /* - * Only give the warning if there are no FileChangedShell - * autocommands. - * Avoid being called recursively by setting "busy". - */ - busy = TRUE; - set_vim_var_string(VV_FCS_REASON, (char_u *)reason, -1); - set_vim_var_string(VV_FCS_CHOICE, (char_u *)"", -1); + // Only give the warning if there are no FileChangedShell + // autocommands. + // Avoid being called recursively by setting "busy". + busy = true; + set_vim_var_string(VV_FCS_REASON, reason, -1); + set_vim_var_string(VV_FCS_CHOICE, "", -1); ++allbuf_lock; n = apply_autocmds(EVENT_FILECHANGEDSHELL, buf->b_fname, buf->b_fname, FALSE, buf); @@ -4876,35 +4874,39 @@ buf_check_timestamp ( if (mesg != NULL) { path = home_replace_save(buf, buf->b_fname); - if (!helpmesg) + if (!helpmesg) { mesg2 = ""; - tbuf = xmalloc(STRLEN(path) + STRLEN(mesg) + STRLEN(mesg2) + 2); - sprintf((char *)tbuf, mesg, path); - /* Set warningmsg here, before the unimportant and output-specific - * mesg2 has been appended. */ + } + const size_t tbuf_len = STRLEN(path) + STRLEN(mesg) + STRLEN(mesg2) + 2; + char *const tbuf = xmalloc(tbuf_len); + snprintf(tbuf, tbuf_len, mesg, path); + // Set warningmsg here, before the unimportant and output-specific + // mesg2 has been appended. set_vim_var_string(VV_WARNINGMSG, tbuf, -1); if (can_reload) { if (*mesg2 != NUL) { - STRCAT(tbuf, "\n"); - STRCAT(tbuf, mesg2); + strncat(tbuf, "\n", tbuf_len); + strncat(tbuf, mesg2, tbuf_len); + } + if (do_dialog(VIM_WARNING, (char_u *) _("Warning"), (char_u *) tbuf, + (char_u *) _("&OK\n&Load File"), 1, NULL, true) == 2) { + reload = true; } - if (do_dialog(VIM_WARNING, (char_u *)_("Warning"), tbuf, - (char_u *)_("&OK\n&Load File"), 1, NULL, TRUE) == 2) - reload = TRUE; } else if (State > NORMAL_BUSY || (State & CMDLINE) || already_warned) { if (*mesg2 != NUL) { - STRCAT(tbuf, "; "); - STRCAT(tbuf, mesg2); + strncat(tbuf, "; ", tbuf_len); + strncat(tbuf, mesg2, tbuf_len); } EMSG(tbuf); retval = 2; } else { if (!autocmd_busy) { msg_start(); - msg_puts_attr(tbuf, hl_attr(HLF_E) + MSG_HIST); - if (*mesg2 != NUL) + msg_puts_attr((char_u *) tbuf, hl_attr(HLF_E) + MSG_HIST); + if (*mesg2 != NUL) { msg_puts_attr((char_u *)mesg2, hl_attr(HLF_W) + MSG_HIST); + } msg_clr_eos(); (void)msg_end(); if (emsg_silent == 0) { diff --git a/src/nvim/fold.c b/src/nvim/fold.c index 70ab4ced75..7f46a37315 100644 --- a/src/nvim/fold.c +++ b/src/nvim/fold.c @@ -1699,14 +1699,14 @@ char_u *get_foldtext(win_T *wp, linenr_T lnum, linenr_T lnume, did_emsg = FALSE; if (*wp->w_p_fdt != NUL) { - char_u dashes[MAX_LEVEL + 2]; + char dashes[MAX_LEVEL + 2]; win_T *save_curwin; int level; char_u *p; - /* Set "v:foldstart" and "v:foldend". */ - set_vim_var_nr(VV_FOLDSTART, lnum); - set_vim_var_nr(VV_FOLDEND, lnume); + // Set "v:foldstart" and "v:foldend". + set_vim_var_nr(VV_FOLDSTART, (varnumber_T) lnum); + set_vim_var_nr(VV_FOLDEND, (varnumber_T) lnume); /* Set "v:folddashes" to a string of "level" dashes. */ /* Set "v:foldlevel" to "level". */ @@ -1716,7 +1716,7 @@ char_u *get_foldtext(win_T *wp, linenr_T lnum, linenr_T lnume, memset(dashes, '-', (size_t)level); dashes[level] = NUL; set_vim_var_string(VV_FOLDDASHES, dashes, -1); - set_vim_var_nr(VV_FOLDLEVEL, (long)level); + set_vim_var_nr(VV_FOLDLEVEL, (varnumber_T) level); /* skip evaluating foldtext on errors */ if (!got_fdt_error) { @@ -2676,7 +2676,7 @@ static void foldlevelExpr(fline_T *flp) win = curwin; curwin = flp->wp; curbuf = flp->wp->w_buffer; - set_vim_var_nr(VV_LNUM, lnum); + set_vim_var_nr(VV_LNUM, (varnumber_T) lnum); flp->start = 0; flp->had_end = flp->end; diff --git a/src/nvim/hardcopy.c b/src/nvim/hardcopy.c index ab8959239b..cc49bcd074 100644 --- a/src/nvim/hardcopy.c +++ b/src/nvim/hardcopy.c @@ -2780,11 +2780,13 @@ void mch_print_end(prt_settings_T *psettings) } prt_message((char_u *)_("Sending to printer...")); - /* Not printing to a file: use 'printexpr' to print the file. */ - if (eval_printexpr(prt_ps_file_name, psettings->arguments) == FAIL) + // Not printing to a file: use 'printexpr' to print the file. + if (eval_printexpr((char *) prt_ps_file_name, (char *) psettings->arguments) + == FAIL) { EMSG(_("E365: Failed to print PostScript file")); - else + } else { prt_message((char_u *)_("Print job sent.")); + } } mch_print_cleanup(); diff --git a/src/nvim/indent.c b/src/nvim/indent.c index d3008185dc..f197669a97 100644 --- a/src/nvim/indent.c +++ b/src/nvim/indent.c @@ -529,7 +529,7 @@ int get_expr_indent(void) save_pos = curwin->w_cursor; save_curswant = curwin->w_curswant; save_set_curswant = curwin->w_set_curswant; - set_vim_var_nr(VV_LNUM, curwin->w_cursor.lnum); + set_vim_var_nr(VV_LNUM, (varnumber_T) curwin->w_cursor.lnum); if (use_sandbox) { sandbox++; diff --git a/src/nvim/main.c b/src/nvim/main.c index 23ced5ebe5..71a972e8f6 100644 --- a/src/nvim/main.c +++ b/src/nvim/main.c @@ -238,8 +238,8 @@ int main(int argc, char **argv) check_and_set_isatty(¶ms); // Get the name with which Nvim was invoked, with and without path. - set_vim_var_string(VV_PROGPATH, (char_u *)argv[0], -1); - set_vim_var_string(VV_PROGNAME, path_tail((char_u *)argv[0]), -1); + set_vim_var_string(VV_PROGPATH, argv[0], -1); + set_vim_var_string(VV_PROGNAME, (char *) path_tail((char_u *) argv[0]), -1); event_init(); /* @@ -1141,10 +1141,11 @@ scripterror: /* If there is a "+123" or "-c" command, set v:swapcommand to the first * one. */ if (parmp->n_commands > 0) { - p = xmalloc(STRLEN(parmp->commands[0]) + 3); - sprintf((char *)p, ":%s\r", parmp->commands[0]); - set_vim_var_string(VV_SWAPCOMMAND, p, -1); - xfree(p); + const size_t swcmd_len = STRLEN(parmp->commands[0]) + 3; + char *const swcmd = xmalloc(swcmd_len); + snprintf(swcmd, swcmd_len, ":%s\r", parmp->commands[0]); + set_vim_var_string(VV_SWAPCOMMAND, swcmd, -1); + xfree(swcmd); } TIME_MSG("parsing arguments"); } diff --git a/src/nvim/memline.c b/src/nvim/memline.c index b568279d7d..10176752d5 100644 --- a/src/nvim/memline.c +++ b/src/nvim/memline.c @@ -3194,7 +3194,7 @@ attention_message ( */ static int do_swapexists(buf_T *buf, char_u *fname) { - set_vim_var_string(VV_SWAPNAME, fname, -1); + set_vim_var_string(VV_SWAPNAME, (char *) fname, -1); set_vim_var_string(VV_SWAPCHOICE, NULL, -1); /* Trigger SwapExists autocommands with set to the file being diff --git a/src/nvim/message.c b/src/nvim/message.c index e0806d8234..97b098c6d2 100644 --- a/src/nvim/message.c +++ b/src/nvim/message.c @@ -143,10 +143,11 @@ msg_attr_keep ( { static int entered = 0; int retval; - char_u *buf = NULL; + char_u *buf = NULL; - if (attr == 0) - set_vim_var_string(VV_STATUSMSG, s, -1); + if (attr == 0) { + set_vim_var_string(VV_STATUSMSG, (char *) s, -1); + } /* * It is possible that displaying a messages causes a problem (e.g., @@ -497,8 +498,8 @@ int emsg(char_u *s) return TRUE; } - /* set "v:errmsg", also when using ":silent! cmd" */ - set_vim_var_string(VV_ERRMSG, s, -1); + // set "v:errmsg", also when using ":silent! cmd" + set_vim_var_string(VV_ERRMSG, (char *) s, -1); /* * When using ":silent! cmd" ignore error messages. @@ -1755,25 +1756,24 @@ static void msg_scroll_up(void) static void inc_msg_scrolled(void) { if (*get_vim_var_str(VV_SCROLLSTART) == NUL) { - char_u *p = sourcing_name; - char_u *tofree = NULL; - int len; + char *p = (char *) sourcing_name; + char *tofree = NULL; - /* v:scrollstart is empty, set it to the script/function name and line - * number */ - if (p == NULL) - p = (char_u *)_("Unknown"); - else { - len = (int)STRLEN(p) + 40; + // v:scrollstart is empty, set it to the script/function name and line + // number + if (p == NULL) { + p = _("Unknown"); + } else { + size_t len = strlen(p) + 40; tofree = xmalloc(len); - vim_snprintf((char *)tofree, len, _("%s line %" PRId64), - p, (int64_t)sourcing_lnum); + vim_snprintf(tofree, len, _("%s line %" PRId64), + p, (int64_t) sourcing_lnum); p = tofree; } set_vim_var_string(VV_SCROLLSTART, p, -1); xfree(tofree); } - ++msg_scrolled; + msg_scrolled++; } static msgchunk_T *last_msgchunk = NULL; /* last displayed text */ @@ -2540,7 +2540,7 @@ void give_warning(char_u *message, bool hl) FUNC_ATTR_NONNULL_ARG(1) /* Don't want a hit-enter prompt here. */ ++no_wait_return; - set_vim_var_string(VV_WARNINGMSG, message, -1); + set_vim_var_string(VV_WARNINGMSG, (char *) message, -1); xfree(keep_msg); keep_msg = NULL; if (hl) @@ -3054,7 +3054,7 @@ int vim_snprintf_add(char *str, size_t str_m, char *fmt, ...) return str_l; } -int vim_snprintf(char *str, size_t str_m, char *fmt, ...) +int vim_snprintf(char *str, size_t str_m, const char *fmt, ...) { va_list ap; int str_l; diff --git a/src/nvim/misc1.c b/src/nvim/misc1.c index db303fd54a..cd29cd6c7e 100644 --- a/src/nvim/misc1.c +++ b/src/nvim/misc1.c @@ -2250,7 +2250,7 @@ change_warning ( msg_col = col; msg_source(hl_attr(HLF_W)); MSG_PUTS_ATTR(_(w_readonly), hl_attr(HLF_W) | MSG_HIST); - set_vim_var_string(VV_WARNINGMSG, (char_u *)_(w_readonly), -1); + set_vim_var_string(VV_WARNINGMSG, _(w_readonly), -1); msg_clr_eos(); (void)msg_end(); if (msg_silent == 0 && !silent_mode) { diff --git a/src/nvim/misc2.c b/src/nvim/misc2.c index 3c0a1414a6..4b64de1be0 100644 --- a/src/nvim/misc2.c +++ b/src/nvim/misc2.c @@ -327,9 +327,10 @@ int call_shell(char_u *cmd, ShellOpts opts, char_u *extra_shell_arg) } } - set_vim_var_nr(VV_SHELL_ERROR, (long)retval); - if (do_profiling == PROF_YES) + set_vim_var_nr(VV_SHELL_ERROR, (varnumber_T) retval); + if (do_profiling == PROF_YES) { prof_child_exit(&wait_time); + } return retval; } diff --git a/src/nvim/msgpack_rpc/server.c b/src/nvim/msgpack_rpc/server.c index 474e25ffeb..bf384e3379 100644 --- a/src/nvim/msgpack_rpc/server.c +++ b/src/nvim/msgpack_rpc/server.c @@ -59,7 +59,7 @@ static void set_vservername(garray_T *srvs) char *default_server = (srvs->ga_len > 0) ? ((SocketWatcher **)srvs->ga_data)[0]->addr : NULL; - set_vim_var_string(VV_SEND_SERVER, (char_u *)default_server, -1); + set_vim_var_string(VV_SEND_SERVER, default_server, -1); } /// Teardown the server module diff --git a/src/nvim/normal.c b/src/nvim/normal.c index 2895816b8f..5b7c4b68b1 100644 --- a/src/nvim/normal.c +++ b/src/nvim/normal.c @@ -7057,18 +7057,17 @@ static void nv_operator(cmdarg_T *cap) */ static void set_op_var(int optype) { - char_u opchars[3]; - - if (optype == OP_NOP) + if (optype == OP_NOP) { set_vim_var_string(VV_OP, NULL, 0); - else { + } else { + char opchars[3]; int opchar0 = get_op_char(optype); assert(opchar0 >= 0 && opchar0 <= UCHAR_MAX); - opchars[0] = (char_u)opchar0; + opchars[0] = (char) opchar0; int opchar1 = get_extra_op_char(optype); assert(opchar1 >= 0 && opchar1 <= UCHAR_MAX); - opchars[1] = (char_u)opchar1; + opchars[1] = (char) opchar1; opchars[2] = NUL; set_vim_var_string(VV_OP, opchars, -1); diff --git a/src/nvim/option.c b/src/nvim/option.c index 5efd71444a..f9d1cdbaec 100644 --- a/src/nvim/option.c +++ b/src/nvim/option.c @@ -1448,7 +1448,7 @@ do_set ( char_u *oldval = NULL; // previous value if *varp char_u *newval; char_u *origval = NULL; - char_u *saved_origval = NULL; + char *saved_origval = NULL; unsigned newlen; int comma; int bs; @@ -1725,7 +1725,7 @@ do_set ( if (!starting && origval != NULL) { // origval may be freed by // did_set_string_option(), make a copy. - saved_origval = vim_strsave(origval); + saved_origval = xstrdup((char *) origval); } /* Handle side effects, and set the global value for @@ -1740,11 +1740,10 @@ do_set ( } if (saved_origval != NULL) { - char_u buf_type[7]; - vim_snprintf((char *)buf_type, ARRAY_SIZE(buf_type), "%s", + char buf_type[7]; + vim_snprintf(buf_type, ARRAY_SIZE(buf_type), "%s", (opt_flags & OPT_LOCAL) ? "local" : "global"); - set_vim_var_string(VV_OPTION_NEW, - *(char_u **)varp, -1); + set_vim_var_string(VV_OPTION_NEW, *(char **) varp, -1); set_vim_var_string(VV_OPTION_OLD, saved_origval, -1); set_vim_var_string(VV_OPTION_TYPE, buf_type, -1); apply_autocmds(EVENT_OPTIONSET, @@ -2324,7 +2323,7 @@ set_string_option ( char_u *s; char_u **varp; char_u *oldval; - char_u *saved_oldval = NULL; + char *saved_oldval = NULL; char_u *r = NULL; if (options[opt_idx].var == NULL) /* don't set hidden option */ @@ -2340,7 +2339,7 @@ set_string_option ( *varp = s; if (!starting) { - saved_oldval = vim_strsave(oldval); + saved_oldval = xstrdup((char *) oldval); } if ((r = did_set_string_option(opt_idx, varp, (int)true, oldval, NULL, @@ -2349,10 +2348,10 @@ set_string_option ( // call autocommand after handling side effects if (saved_oldval != NULL) { - char_u buf_type[7]; - vim_snprintf((char *)buf_type, ARRAY_SIZE(buf_type), "%s", + char buf_type[7]; + vim_snprintf(buf_type, ARRAY_SIZE(buf_type), "%s", (opt_flags & OPT_LOCAL) ? "local" : "global"); - set_vim_var_string(VV_OPTION_NEW, *varp, -1); + set_vim_var_string(VV_OPTION_NEW, (char *) (*varp), -1); set_vim_var_string(VV_OPTION_OLD, saved_oldval, -1); set_vim_var_string(VV_OPTION_TYPE, buf_type, -1); apply_autocmds(EVENT_OPTIONSET, @@ -3800,7 +3799,7 @@ set_bool_option ( msg_source(hl_attr(HLF_W)); MSG_ATTR(_(w_arabic), hl_attr(HLF_W)); - set_vim_var_string(VV_WARNINGMSG, (char_u *)_(w_arabic), -1); + set_vim_var_string(VV_WARNINGMSG, _(w_arabic), -1); } /* set 'delcombine' */ @@ -3847,14 +3846,14 @@ set_bool_option ( options[opt_idx].flags |= P_WAS_SET; if (!starting) { - char_u buf_old[2]; - char_u buf_new[2]; - char_u buf_type[7]; - vim_snprintf((char *)buf_old, ARRAY_SIZE(buf_old), "%d", + char buf_old[2]; + char buf_new[2]; + char buf_type[7]; + vim_snprintf(buf_old, ARRAY_SIZE(buf_old), "%d", old_value ? true: false); - vim_snprintf((char *)buf_new, ARRAY_SIZE(buf_new), "%d", + vim_snprintf(buf_new, ARRAY_SIZE(buf_new), "%d", value ? true: false); - vim_snprintf((char *)buf_type, ARRAY_SIZE(buf_type), "%s", + vim_snprintf(buf_type, ARRAY_SIZE(buf_type), "%s", (opt_flags & OPT_LOCAL) ? "local" : "global"); set_vim_var_string(VV_OPTION_NEW, buf_new, -1); set_vim_var_string(VV_OPTION_OLD, buf_old, -1); @@ -4237,12 +4236,12 @@ set_num_option ( options[opt_idx].flags |= P_WAS_SET; if (!starting && errmsg == NULL) { - char_u buf_old[NUMBUFLEN]; - char_u buf_new[NUMBUFLEN]; - char_u buf_type[7]; - vim_snprintf((char *)buf_old, ARRAY_SIZE(buf_old), "%ld", old_value); - vim_snprintf((char *)buf_new, ARRAY_SIZE(buf_new), "%ld", value); - vim_snprintf((char *)buf_type, ARRAY_SIZE(buf_type), "%s", + char buf_old[NUMBUFLEN]; + char buf_new[NUMBUFLEN]; + char buf_type[7]; + vim_snprintf(buf_old, ARRAY_SIZE(buf_old), "%ld", old_value); + vim_snprintf(buf_new, ARRAY_SIZE(buf_new), "%ld", value); + vim_snprintf(buf_type, ARRAY_SIZE(buf_type), "%s", (opt_flags & OPT_LOCAL) ? "local" : "global"); set_vim_var_string(VV_OPTION_NEW, buf_new, -1); set_vim_var_string(VV_OPTION_OLD, buf_old, -1); diff --git a/src/nvim/path.c b/src/nvim/path.c index 22a3f96cfa..aaf54bc5b4 100644 --- a/src/nvim/path.c +++ b/src/nvim/path.c @@ -1495,13 +1495,12 @@ void simplify_filename(char_u *filename) } while (*p != NUL); } -static char_u *eval_includeexpr(char_u *ptr, size_t len) +static char *eval_includeexpr(const char *const ptr, const size_t len) { - assert(len <= INT_MAX); - set_vim_var_string(VV_FNAME, ptr, (int)len); - char_u *res = eval_to_string_safe(curbuf->b_p_inex, NULL, - was_set_insecurely((char_u *)"includeexpr", - OPT_LOCAL)); + set_vim_var_string(VV_FNAME, ptr, (ptrdiff_t) len); + char *res = (char *) eval_to_string_safe( + curbuf->b_p_inex, NULL, was_set_insecurely((char_u *)"includeexpr", + OPT_LOCAL)); set_vim_var_string(VV_FNAME, NULL, 0); return res; } @@ -1523,7 +1522,7 @@ find_file_name_in_path ( char_u *tofree = NULL; if ((options & FNAME_INCL) && *curbuf->b_p_inex != NUL) { - tofree = eval_includeexpr(ptr, len); + tofree = (char_u *) eval_includeexpr((char *) ptr, len); if (tofree != NULL) { ptr = tofree; len = STRLEN(ptr); @@ -1540,7 +1539,7 @@ find_file_name_in_path ( */ if (file_name == NULL && !(options & FNAME_INCL) && *curbuf->b_p_inex != NUL) { - tofree = eval_includeexpr(ptr, len); + tofree = (char_u *) eval_includeexpr((char *) ptr, len); if (tofree != NULL) { ptr = tofree; len = STRLEN(ptr); diff --git a/src/nvim/quickfix.c b/src/nvim/quickfix.c index 3bc6d46dd9..28c0425e2c 100644 --- a/src/nvim/quickfix.c +++ b/src/nvim/quickfix.c @@ -2440,8 +2440,6 @@ int grep_internal(cmdidx_T cmdidx) void ex_make(exarg_T *eap) { char_u *fname; - char_u *cmd; - size_t len; win_T *wp = NULL; qf_info_T *qi = &ql_info; int res; @@ -2479,30 +2477,28 @@ void ex_make(exarg_T *eap) return; os_remove((char *)fname); // in case it's not unique - /* - * If 'shellpipe' empty: don't redirect to 'errorfile'. - */ - len = STRLEN(p_shq) * 2 + STRLEN(eap->arg) + 1; + // If 'shellpipe' empty: don't redirect to 'errorfile'. + const size_t len = (STRLEN(p_shq) * 2 + STRLEN(eap->arg) + 1 + + (*p_sp == NUL + ? 0 + : STRLEN(p_sp) + STRLEN(fname) + 3)); + char *const cmd = xmalloc(len); + snprintf(cmd, len, "%s%s%s", (char *)p_shq, (char *)eap->arg, + (char *)p_shq); if (*p_sp != NUL) { - len += STRLEN(p_sp) + STRLEN(fname) + 3; + append_redir(cmd, len, (char *) p_sp, (char *) fname); + } + // Output a newline if there's something else than the :make command that + // was typed (in which case the cursor is in column 0). + if (msg_col == 0) { + msg_didout = false; } - cmd = xmalloc(len); - sprintf((char *)cmd, "%s%s%s", (char *)p_shq, (char *)eap->arg, - (char *)p_shq); - if (*p_sp != NUL) - append_redir(cmd, len, p_sp, fname); - /* - * Output a newline if there's something else than the :make command that - * was typed (in which case the cursor is in column 0). - */ - if (msg_col == 0) - msg_didout = FALSE; msg_start(); MSG_PUTS(":!"); - msg_outtrans(cmd); /* show what we are doing */ + msg_outtrans((char_u *) cmd); // show what we are doing - /* let the shell know if we are redirecting output or not */ - do_shell(cmd, *p_sp != NUL ? kShellOptDoOut : 0); + // let the shell know if we are redirecting output or not + do_shell((char_u *) cmd, *p_sp != NUL ? kShellOptDoOut : 0); res = qf_init(wp, fname, (eap->cmdidx != CMD_make diff --git a/src/nvim/tag.c b/src/nvim/tag.c index 8fcb02c3b6..26a89094aa 100644 --- a/src/nvim/tag.c +++ b/src/nvim/tag.c @@ -872,7 +872,7 @@ do_tag ( /* Let the SwapExists event know what tag we are jumping to. */ vim_snprintf((char *)IObuff, IOSIZE, ":ta %s\r", name); - set_vim_var_string(VV_SWAPCOMMAND, IObuff, -1); + set_vim_var_string(VV_SWAPCOMMAND, (char *) IObuff, -1); /* * Jump to the desired match. From fd92e648ac206340752c420ad639f2a6dab2a579 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 20 Mar 2016 23:24:53 +0300 Subject: [PATCH 76/82] eval/encode: Dump FF character correctly --- src/nvim/eval/encode.c | 1 + test/functional/eval/json_functions_spec.lua | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/src/nvim/eval/encode.c b/src/nvim/eval/encode.c index d21347cca6..c3941924e1 100644 --- a/src/nvim/eval/encode.c +++ b/src/nvim/eval/encode.c @@ -866,6 +866,7 @@ static const char escapes[][3] = { [CAR] = "\\r", ['"'] = "\\\"", ['\\'] = "\\\\", + [FF] = "\\f", }; static const char xdigits[] = "0123456789ABCDEF"; diff --git a/test/functional/eval/json_functions_spec.lua b/test/functional/eval/json_functions_spec.lua index 0b1862fa8b..bdd3306993 100644 --- a/test/functional/eval/json_functions_spec.lua +++ b/test/functional/eval/json_functions_spec.lua @@ -779,4 +779,9 @@ describe('json_encode() function', function() eq('Vim(call):E474: UTF-8 string contains code point which belongs to a surrogate pair: \xED\xAF\xBF', exc_exec('call json_encode("\xED\xAF\xBF")')) end) + + it('dumps control characters as expected', function() + eq([["\u0000\u0001\u0002\u0003\u0004\u0005\u0006\u0007\b\t\n\u000B\f\r\u000E\u000F\u0010\u0011\u0012\u0013"]], + eval('json_encode({"_TYPE": v:msgpack_types.string, "_VAL": ["\n\1\2\3\4\5\6\7\8\9", "\11\12\13\14\15\16\17\18\19"]})')) + end) end) From 3e435df42cd1f8237bd11a4b08407a9ec3f81dba Mon Sep 17 00:00:00 2001 From: ZyX Date: Mon, 21 Mar 2016 00:19:41 +0300 Subject: [PATCH 77/82] functests: Replace \xXX escapes with \DDD in lua code --- test/functional/eval/json_functions_spec.lua | 80 ++++++++++---------- test/unit/eval/decode_spec.lua | 2 +- 2 files changed, 41 insertions(+), 41 deletions(-) diff --git a/test/functional/eval/json_functions_spec.lua b/test/functional/eval/json_functions_spec.lua index bdd3306993..b788c2cf07 100644 --- a/test/functional/eval/json_functions_spec.lua +++ b/test/functional/eval/json_functions_spec.lua @@ -356,12 +356,12 @@ describe('json_decode() function', function() eq({ '«', 'ફ', - '\xF0\x90\x80\x80', + '\240\144\128\128', }, funcs.json_decode({ '[', '"«",', '"ફ",', - '"\xF0\x90\x80\x80"', + '"\240\144\128\128"', ']', })) end) @@ -387,54 +387,54 @@ describe('json_decode() function', function() eq('Vim(call):E474: Only UTF-8 strings allowed: \240\144\128"', exc_exec('call json_decode("\\t\\"\\xF0\\x90\\x80\\"")')) -- 0xF9 0x80 0x80 0x80 starts 5-byte unicode character - eq('Vim(call):E474: Only UTF-8 strings allowed: \xF9"', + eq('Vim(call):E474: Only UTF-8 strings allowed: \249"', exc_exec('call json_decode("\\t\\"\\xF9\\"")')) - eq('Vim(call):E474: Only UTF-8 strings allowed: \xF9\x80"', + eq('Vim(call):E474: Only UTF-8 strings allowed: \249\128"', exc_exec('call json_decode("\\t\\"\\xF9\\x80\\"")')) - eq('Vim(call):E474: Only UTF-8 strings allowed: \xF9\x80\x80"', + eq('Vim(call):E474: Only UTF-8 strings allowed: \249\128\128"', exc_exec('call json_decode("\\t\\"\\xF9\\x80\\x80\\"")')) - eq('Vim(call):E474: Only UTF-8 strings allowed: \xF9\x80\x80\x80"', + eq('Vim(call):E474: Only UTF-8 strings allowed: \249\128\128\128"', exc_exec('call json_decode("\\t\\"\\xF9\\x80\\x80\\x80\\"")')) -- 0xFC 0x90 0x80 0x80 0x80 starts 6-byte unicode character - eq('Vim(call):E474: Only UTF-8 strings allowed: \xFC"', + eq('Vim(call):E474: Only UTF-8 strings allowed: \252"', exc_exec('call json_decode("\\t\\"\\xFC\\"")')) - eq('Vim(call):E474: Only UTF-8 strings allowed: \xFC\x90"', + eq('Vim(call):E474: Only UTF-8 strings allowed: \252\144"', exc_exec('call json_decode("\\t\\"\\xFC\\x90\\"")')) - eq('Vim(call):E474: Only UTF-8 strings allowed: \xFC\x90\x80"', + eq('Vim(call):E474: Only UTF-8 strings allowed: \252\144\128"', exc_exec('call json_decode("\\t\\"\\xFC\\x90\\x80\\"")')) - eq('Vim(call):E474: Only UTF-8 strings allowed: \xFC\x90\x80\x80"', + eq('Vim(call):E474: Only UTF-8 strings allowed: \252\144\128\128"', exc_exec('call json_decode("\\t\\"\\xFC\\x90\\x80\\x80\\"")')) - eq('Vim(call):E474: Only UTF-8 strings allowed: \xFC\x90\x80\x80\x80"', + eq('Vim(call):E474: Only UTF-8 strings allowed: \252\144\128\128\128"', exc_exec('call json_decode("\\t\\"\\xFC\\x90\\x80\\x80\\x80\\"")')) -- Specification does not allow unquoted characters above 0x10FFFF - eq('Vim(call):E474: Only UTF-8 code points up to U+10FFFF are allowed to appear unescaped: \xF9\x80\x80\x80\x80"', + eq('Vim(call):E474: Only UTF-8 code points up to U+10FFFF are allowed to appear unescaped: \249\128\128\128\128"', exc_exec('call json_decode("\\t\\"\\xF9\\x80\\x80\\x80\\x80\\"")')) - eq('Vim(call):E474: Only UTF-8 code points up to U+10FFFF are allowed to appear unescaped: \xFC\x90\x80\x80\x80\x80"', + eq('Vim(call):E474: Only UTF-8 code points up to U+10FFFF are allowed to appear unescaped: \252\144\128\128\128\128"', exc_exec('call json_decode("\\t\\"\\xFC\\x90\\x80\\x80\\x80\\x80\\"")')) - -- '"\xF9\x80\x80\x80\x80"', - -- '"\xFC\x90\x80\x80\x80\x80"', + -- '"\249\128\128\128\128"', + -- '"\252\144\128\128\128\128"', end) it('parses surrogate pairs properly', function() - eq('\xF0\x90\x80\x80', funcs.json_decode('"\\uD800\\uDC00"')) - eq('\xED\xA0\x80a\xED\xB0\x80', funcs.json_decode('"\\uD800a\\uDC00"')) - eq('\xED\xA0\x80\t\xED\xB0\x80', funcs.json_decode('"\\uD800\\t\\uDC00"')) + eq('\240\144\128\128', funcs.json_decode('"\\uD800\\uDC00"')) + eq('\237\160\128a\237\176\128', funcs.json_decode('"\\uD800a\\uDC00"')) + eq('\237\160\128\t\237\176\128', funcs.json_decode('"\\uD800\\t\\uDC00"')) - eq('\xED\xA0\x80', funcs.json_decode('"\\uD800"')) - eq('\xED\xA0\x80a', funcs.json_decode('"\\uD800a"')) - eq('\xED\xA0\x80\t', funcs.json_decode('"\\uD800\\t"')) + eq('\237\160\128', funcs.json_decode('"\\uD800"')) + eq('\237\160\128a', funcs.json_decode('"\\uD800a"')) + eq('\237\160\128\t', funcs.json_decode('"\\uD800\\t"')) - eq('\xED\xB0\x80', funcs.json_decode('"\\uDC00"')) - eq('\xED\xB0\x80a', funcs.json_decode('"\\uDC00a"')) - eq('\xED\xB0\x80\t', funcs.json_decode('"\\uDC00\\t"')) + eq('\237\176\128', funcs.json_decode('"\\uDC00"')) + eq('\237\176\128a', funcs.json_decode('"\\uDC00a"')) + eq('\237\176\128\t', funcs.json_decode('"\\uDC00\\t"')) - eq('\xED\xB0\x80', funcs.json_decode('"\\uDC00"')) - eq('a\xED\xB0\x80', funcs.json_decode('"a\\uDC00"')) - eq('\t\xED\xB0\x80', funcs.json_decode('"\\t\\uDC00"')) + eq('\237\176\128', funcs.json_decode('"\\uDC00"')) + eq('a\237\176\128', funcs.json_decode('"a\\uDC00"')) + eq('\t\237\176\128', funcs.json_decode('"\\t\\uDC00"')) - eq('\xED\xA0\x80¬', funcs.json_decode('"\\uD800\\u00AC"')) + eq('\237\160\128¬', funcs.json_decode('"\\uD800\\u00AC"')) - eq('\xED\xA0\x80\xED\xA0\x80', funcs.json_decode('"\\uD800\\uD800"')) + eq('\237\160\128\237\160\128', funcs.json_decode('"\\uD800\\uD800"')) end) local sp_decode_eq = function(expected, json) @@ -490,8 +490,8 @@ describe('json_decode() function', function() it('converts strings to latin1 when &encoding is latin1', function() restart('set encoding=latin1') - eq('\xAB', funcs.json_decode('"\\u00AB"')) - sp_decode_eq({_TYPE='string', _VAL={'\n\xAB\n'}}, '"\\u0000\\u00AB\\u0000"') + eq('\171', funcs.json_decode('"\\u00AB"')) + sp_decode_eq({_TYPE='string', _VAL={'\n\171\n'}}, '"\\u0000\\u00AB\\u0000"') end) it('fails to convert string to latin1 if it is impossible', function() @@ -501,7 +501,7 @@ describe('json_decode() function', function() end) it('parses U+00C3 correctly', function() - eq('\xC3\x83', funcs.json_decode('"\xC3\x83"')) + eq('\195\131', funcs.json_decode('"\195\131"')) end) it('fails to parse empty string', function() @@ -763,21 +763,21 @@ describe('json_encode() function', function() it('converts strings from latin1 when &encoding is latin1', function() clear('set encoding=latin1') - eq('"\\u00AB"', funcs.json_encode('\xAB')) - eq('"\\u0000\\u00AB\\u0000"', eval('json_encode({"_TYPE": v:msgpack_types.string, "_VAL": ["\\n\xAB\\n"]})')) + eq('"\\u00AB"', funcs.json_encode('\171')) + eq('"\\u0000\\u00AB\\u0000"', eval('json_encode({"_TYPE": v:msgpack_types.string, "_VAL": ["\\n\171\\n"]})')) end) it('ignores improper values in &isprint', function() meths.set_option('isprint', '1') - eq(1, eval('"\x01" =~# "\\\\p"')) - eq('"\\u0001"', funcs.json_encode('\x01')) + eq(1, eval('"\1" =~# "\\\\p"')) + eq('"\\u0001"', funcs.json_encode('\1')) end) it('fails when using surrogate character in a UTF-8 string', function() - eq('Vim(call):E474: UTF-8 string contains code point which belongs to a surrogate pair: \xED\xA0\x80', - exc_exec('call json_encode("\xED\xA0\x80")')) - eq('Vim(call):E474: UTF-8 string contains code point which belongs to a surrogate pair: \xED\xAF\xBF', - exc_exec('call json_encode("\xED\xAF\xBF")')) + eq('Vim(call):E474: UTF-8 string contains code point which belongs to a surrogate pair: \237\160\128', + exc_exec('call json_encode("\237\160\128")')) + eq('Vim(call):E474: UTF-8 string contains code point which belongs to a surrogate pair: \237\175\191', + exc_exec('call json_encode("\237\175\191")')) end) it('dumps control characters as expected', function() diff --git a/test/unit/eval/decode_spec.lua b/test/unit/eval/decode_spec.lua index 44471c1877..d94d809c14 100644 --- a/test/unit/eval/decode_spec.lua +++ b/test/unit/eval/decode_spec.lua @@ -115,7 +115,7 @@ describe('json_decode_string()', function() '"\t"test', 3, 'E474: ASCII control characters cannot be present inside string: \t"') check_failure('"\194"test', 3, 'E474: Only UTF-8 strings allowed: \194"') - check_failure('"\xFC\x90\x80\x80\x80\x80"test', 8, 'E474: Only UTF-8 code points up to U+10FFFF are allowed to appear unescaped: \xFC\x90\x80\x80\x80\x80"') + check_failure('"\252\144\128\128\128\128"test', 8, 'E474: Only UTF-8 code points up to U+10FFFF are allowed to appear unescaped: \252\144\128\128\128\128"') check_failure('"test', 1, 'E474: Expected string end: "') decode.p_enc = to_cstr('latin1') check_failure('"\\uABCD"test', 8, From c4f1b5a9383c00e0a23fdfdca096c569f05e8a1c Mon Sep 17 00:00:00 2001 From: ZyX Date: Sat, 2 Apr 2016 01:18:58 +0300 Subject: [PATCH 78/82] eval/encode: Adjust buffer sizes passed to vim_snprintf --- src/nvim/eval/encode.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/nvim/eval/encode.c b/src/nvim/eval/encode.c index c3941924e1..0bde6562b8 100644 --- a/src/nvim/eval/encode.c +++ b/src/nvim/eval/encode.c @@ -658,7 +658,7 @@ encode_vim_to_##name##_error_ret: \ #define CONV_NUMBER(num) \ do { \ char numbuf[NUMBUFLEN]; \ - vim_snprintf(numbuf, NUMBUFLEN - 1, "%" PRId64, (int64_t) (num)); \ + vim_snprintf(numbuf, ARRAY_SIZE(numbuf), "%" PRId64, (int64_t) (num)); \ ga_concat(gap, numbuf); \ } while (0) @@ -679,7 +679,7 @@ encode_vim_to_##name##_error_ret: \ } \ default: { \ char numbuf[NUMBUFLEN]; \ - vim_snprintf(numbuf, NUMBUFLEN - 1, "%g", flt_); \ + vim_snprintf(numbuf, ARRAY_SIZE(numbuf), "%g", flt_); \ ga_concat(gap, (char_u *) numbuf); \ } \ } \ @@ -754,7 +754,7 @@ encode_vim_to_##name##_error_ret: \ } \ } \ } \ - vim_snprintf(ebuf, NUMBUFLEN + 6, "{E724@%zu}", backref); \ + vim_snprintf(ebuf, ARRAY_SIZE(ebuf), "{E724@%zu}", backref); \ ga_concat(gap, &ebuf[0]); \ return OK; \ } while (0) @@ -783,9 +783,9 @@ DEFINE_VIML_CONV_FUNCTIONS(static, string, garray_T *const, gap) } \ } \ if (conv_type == kMPConvDict) { \ - vim_snprintf(ebuf, NUMBUFLEN + 6, "{...@%zu}", backref); \ + vim_snprintf(ebuf, ARRAY_SIZE(ebuf), "{...@%zu}", backref); \ } else { \ - vim_snprintf(ebuf, NUMBUFLEN + 6, "[...@%zu]", backref); \ + vim_snprintf(ebuf, ARRAY_SIZE(ebuf), "[...@%zu]", backref); \ } \ ga_concat(gap, &ebuf[0]); \ return OK; \ @@ -821,7 +821,7 @@ DEFINE_VIML_CONV_FUNCTIONS(, echo, garray_T *const, gap) #define CONV_UNSIGNED_NUMBER(num) \ do { \ char numbuf[NUMBUFLEN]; \ - vim_snprintf(numbuf, sizeof(numbuf), "%" PRIu64, (num)); \ + vim_snprintf(numbuf, ARRAY_SIZE(numbuf), "%" PRIu64, (num)); \ ga_concat(gap, numbuf); \ } while (0) @@ -840,7 +840,7 @@ DEFINE_VIML_CONV_FUNCTIONS(, echo, garray_T *const, gap) } \ default: { \ char numbuf[NUMBUFLEN]; \ - vim_snprintf(numbuf, NUMBUFLEN - 1, "%g", flt_); \ + vim_snprintf(numbuf, ARRAY_SIZE(numbuf), "%g", flt_); \ ga_concat(gap, (char_u *) numbuf); \ break; \ } \ From bda0165514a582978c2da672b528562df78a2d1a Mon Sep 17 00:00:00 2001 From: ZyX Date: Mon, 4 Apr 2016 04:53:07 +0300 Subject: [PATCH 79/82] eval/encode: Make sure that encoder can encode NULL variables Adds two undocumented v: variables: _null_list and _null_dict because I do not know a reproducible way to get such lists (though I think I heard about this) and dictionaries (do not remember hearing about them). NULL strings are obtained using $XXX_UNEXISTENT_VAR_XXX. Fixes crash in json_encode($XXX_UNEXISTENT_VAR_XXX). Other added tests worked fine before this commit. --- src/nvim/eval.c | 2 + src/nvim/eval.h | 2 + src/nvim/eval/encode.c | 5 ++- test/functional/eval/json_functions_spec.lua | 12 ++++++ .../eval/msgpack_functions_spec.lua | 14 ++++++ test/functional/eval/string_spec.lua | 12 ++++++ test/unit/eval/tricks_spec.lua | 43 +++++++++++++++++++ 7 files changed, 89 insertions(+), 1 deletion(-) create mode 100644 test/unit/eval/tricks_spec.lua diff --git a/src/nvim/eval.c b/src/nvim/eval.c index 204c0fb1ef..aaad9fab34 100644 --- a/src/nvim/eval.c +++ b/src/nvim/eval.c @@ -379,6 +379,8 @@ static struct vimvar { VV(VV_FALSE, "false", VAR_SPECIAL, VV_RO), VV(VV_TRUE, "true", VAR_SPECIAL, VV_RO), VV(VV_NULL, "null", VAR_SPECIAL, VV_RO), + VV(VV__NULL_LIST, "_null_list", VAR_LIST, VV_RO), + VV(VV__NULL_DICT, "_null_dict", VAR_DICT, VV_RO), }; #undef VV diff --git a/src/nvim/eval.h b/src/nvim/eval.h index f81eb5d063..d6800afd52 100644 --- a/src/nvim/eval.h +++ b/src/nvim/eval.h @@ -122,6 +122,8 @@ typedef enum { VV_FALSE, VV_TRUE, VV_NULL, + VV__NULL_LIST, // List with NULL value. For test purposes only. + VV__NULL_DICT, // Dictionary with NULL value. For test purposes only. } VimVarIndex; /// All recognized msgpack types diff --git a/src/nvim/eval/encode.c b/src/nvim/eval/encode.c index 0bde6562b8..88c731e92a 100644 --- a/src/nvim/eval/encode.c +++ b/src/nvim/eval/encode.c @@ -287,6 +287,9 @@ int encode_read_from_list(ListReaderState *const state, char *const buf, (val)->copyID_attr = copyID; \ } while (0) +#define TV_STRLEN(tv) \ + (tv->vval.v_string == NULL ? 0 : STRLEN(tv->vval.v_string)) + /// Define functions which convert VimL value to something else /// /// Creates function `vim_to_{name}(firstargtype firstargname, typval_T *const @@ -306,7 +309,7 @@ static int name##_convert_one_value(firstargtype firstargname, \ { \ switch (tv->v_type) { \ case VAR_STRING: { \ - CONV_STRING(tv->vval.v_string, STRLEN(tv->vval.v_string)); \ + CONV_STRING(tv->vval.v_string, TV_STRLEN(tv)); \ break; \ } \ case VAR_NUMBER: { \ diff --git a/test/functional/eval/json_functions_spec.lua b/test/functional/eval/json_functions_spec.lua index b788c2cf07..0a7c4cf669 100644 --- a/test/functional/eval/json_functions_spec.lua +++ b/test/functional/eval/json_functions_spec.lua @@ -784,4 +784,16 @@ describe('json_encode() function', function() eq([["\u0000\u0001\u0002\u0003\u0004\u0005\u0006\u0007\b\t\n\u000B\f\r\u000E\u000F\u0010\u0011\u0012\u0013"]], eval('json_encode({"_TYPE": v:msgpack_types.string, "_VAL": ["\n\1\2\3\4\5\6\7\8\9", "\11\12\13\14\15\16\17\18\19"]})')) end) + + it('can dump NULL string', function() + eq('""', eval('json_encode($XXX_UNEXISTENT_VAR_XXX)')) + end) + + it('can dump NULL list', function() + eq('[]', eval('json_encode(v:_null_list)')) + end) + + it('can dump NULL dictionary', function() + eq('{}', eval('json_encode(v:_null_dict)')) + end) end) diff --git a/test/functional/eval/msgpack_functions_spec.lua b/test/functional/eval/msgpack_functions_spec.lua index a602bad86f..9e501353a5 100644 --- a/test/functional/eval/msgpack_functions_spec.lua +++ b/test/functional/eval/msgpack_functions_spec.lua @@ -686,4 +686,18 @@ describe('msgpackdump() function', function() exc_exec('call msgpackdump(' .. val .. ')')) end end) + + it('can dump NULL string', function() + eq({'\196\n'}, eval('msgpackdump([$XXX_UNEXISTENT_VAR_XXX])')) + eq({'\196\n'}, eval('msgpackdump([{"_TYPE": v:msgpack_types.binary, "_VAL": [$XXX_UNEXISTENT_VAR_XXX]}])')) + eq({'\160'}, eval('msgpackdump([{"_TYPE": v:msgpack_types.string, "_VAL": [$XXX_UNEXISTENT_VAR_XXX]}])')) + end) + + it('can dump NULL list', function() + eq({'\144'}, eval('msgpackdump([v:_null_list])')) + end) + + it('can dump NULL dictionary', function() + eq({'\128'}, eval('msgpackdump([v:_null_dict])')) + end) end) diff --git a/test/functional/eval/string_spec.lua b/test/functional/eval/string_spec.lua index 0c4ff87231..20a800cde0 100644 --- a/test/functional/eval/string_spec.lua +++ b/test/functional/eval/string_spec.lua @@ -84,6 +84,18 @@ describe('string() function', function() eq('\'\'\'b\'\'\'\'d\'', funcs.string('\'b\'\'d')) eq('\'a\'\'b\'\'c\'\'d\'', funcs.string('a\'b\'c\'d')) end) + + it('dumps NULL strings', function() + eq('\'\'', eval('string($XXX_UNEXISTENT_VAR_XXX)')) + end) + + it('dumps NULL lists', function() + eq('[]', eval('string(v:_null_list)')) + end) + + it('dumps NULL dictionaries', function() + eq('{}', eval('string(v:_null_dict)')) + end) end) describe('used to represent funcrefs', function() diff --git a/test/unit/eval/tricks_spec.lua b/test/unit/eval/tricks_spec.lua new file mode 100644 index 0000000000..4c5184995c --- /dev/null +++ b/test/unit/eval/tricks_spec.lua @@ -0,0 +1,43 @@ +local helpers = require('test.unit.helpers') + +local cimport = helpers.cimport +local to_cstr = helpers.to_cstr +local ffi = helpers.ffi +local eq = helpers.eq + +local eval = cimport('./src/nvim/eval.h', './src/nvim/memory.h') + +local eval_expr = function(expr) + return ffi.gc(eval.eval_expr(to_cstr(expr), nil), function(tv) + eval.clear_tv(tv) + eval.xfree(tv) + end) +end + +describe('NULL typval_T', function() + it('is produced by $XXX_UNEXISTENT_VAR_XXX', function() + -- Required for various tests which need to check whether typval_T with NULL + -- string works correctly. This test checks that unexistent environment + -- variable produces NULL string, not that some specific environment + -- variable does not exist. Last bit is left for the test writers. + local unexistent_env = 'XXX_UNEXISTENT_VAR_XXX' + while os.getenv(unexistent_env) ~= nil do + unexistent_env = unexistent_env .. '_XXX' + end + local rettv = eval_expr('$' .. unexistent_env) + eq(eval.VAR_STRING, rettv.v_type) + eq(nil, rettv.vval.v_string) + end) + + it('is produced by v:_null_list', function() + local rettv = eval_expr('v:_null_list') + eq(eval.VAR_LIST, rettv.v_type) + eq(nil, rettv.vval.v_list) + end) + + it('is produced by v:_null_dict', function() + local rettv = eval_expr('v:_null_dict') + eq(eval.VAR_DICT, rettv.v_type) + eq(nil, rettv.vval.v_dict) + end) +end) From 45304b482cbeae01af3a89358d96b0d1511213c3 Mon Sep 17 00:00:00 2001 From: ZyX Date: Mon, 4 Apr 2016 04:58:21 +0300 Subject: [PATCH 80/82] eval/encode: Simplify loop in encode_list_write Patch made up by oni-link. --- src/nvim/eval/encode.c | 48 ++++++++++++++++++++++-------------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/src/nvim/eval/encode.c b/src/nvim/eval/encode.c index 88c731e92a..c651a50be9 100644 --- a/src/nvim/eval/encode.c +++ b/src/nvim/eval/encode.c @@ -75,36 +75,38 @@ int encode_list_write(void *data, const char *buf, size_t len) const char *const end = buf + len; const char *line_end = buf; listitem_T *li = list->lv_last; - do { + + // Continue the last list element + if (li != NULL) { + line_end = xmemscan(buf, NL, len); + if (line_end != buf) { + const size_t line_length = (size_t)(line_end - buf); + char *str = (char *)li->li_tv.vval.v_string; + const size_t li_len = (str == NULL ? 0 : strlen(str)); + li->li_tv.vval.v_string = xrealloc(str, li_len + line_length + 1); + str = (char *)li->li_tv.vval.v_string + li_len; + memcpy(str, buf, line_length); + str[line_length] = 0; + memchrsub(str, NUL, NL, line_length); + } + line_end++; + } + + while (line_end < end) { const char *line_start = line_end; line_end = xmemscan(line_start, NL, (size_t) (end - line_start)); char *str = NULL; if (line_end != line_start) { - const size_t line_length = (size_t) (line_end - line_start); - if (li == NULL) { - str = xmemdupz(line_start, line_length); - } else { - const size_t li_len = (li->li_tv.vval.v_string == NULL - ? 0 - : STRLEN(li->li_tv.vval.v_string)); - li->li_tv.vval.v_string = xrealloc(li->li_tv.vval.v_string, - li_len + line_length + 1); - str = (char *) li->li_tv.vval.v_string + li_len; - memcpy(str, line_start, line_length); - str[line_length] = 0; - } + const size_t line_length = (size_t)(line_end - line_start); + str = xmemdupz(line_start, line_length); memchrsub(str, NUL, NL, line_length); } - if (li == NULL) { - list_append_allocated_string(list, str); - } else { - li = NULL; - } - if (line_end == end - 1) { - list_append_allocated_string(list, NULL); - } + list_append_allocated_string(list, str); line_end++; - } while (line_end < end); + } + if (line_end == end) { + list_append_allocated_string(list, NULL); + } return 0; } From 28275fe5c3eedd7d5c5954178a28884a7a7b9483 Mon Sep 17 00:00:00 2001 From: ZyX Date: Mon, 18 Apr 2016 01:32:00 +0300 Subject: [PATCH 81/82] *: Fix preincrement lint errors --- src/nvim/eval.c | 12 ++++++------ src/nvim/fileio.c | 8 ++++---- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/nvim/eval.c b/src/nvim/eval.c index aaad9fab34..9d370632a1 100644 --- a/src/nvim/eval.c +++ b/src/nvim/eval.c @@ -478,7 +478,7 @@ void eval_init(void) hash_init(&compat_hashtab); hash_init(&func_hashtab); - for (size_t i = 0; i < ARRAY_SIZE(vimvars); ++i) { + for (size_t i = 0; i < ARRAY_SIZE(vimvars); i++) { p = &vimvars[i]; STRCPY(p->vv_di.di_key, p->vv_name); if (p->vv_flags & VV_RO) @@ -537,7 +537,7 @@ void eval_clear(void) { struct vimvar *p; - for (size_t i = 0; i < ARRAY_SIZE(vimvars); ++i) { + for (size_t i = 0; i < ARRAY_SIZE(vimvars); i++) { p = &vimvars[i]; if (p->vv_di.di_tv.v_type == VAR_STRING) { xfree(p->vv_str); @@ -20069,10 +20069,10 @@ call_user_func ( msg_outnum((long)argvars[i].vval.v_number); } else { // Do not want errors such as E724 here. - ++emsg_off; + emsg_off++; char_u *s = (char_u *) encode_tv2string(&argvars[i], NULL); char_u *tofree = s; - --emsg_off; + emsg_off--; if (s != NULL) { if (vim_strsize(s) > MSG_BUF_CLEN) { trunc_string(s, buf, MSG_BUF_CLEN, MSG_BUF_LEN); @@ -20163,10 +20163,10 @@ call_user_func ( // The value may be very long. Skip the middle part, so that we // have some idea how it starts and ends. smsg() would always // truncate it at the end. Don't want errors such as E724 here. - ++emsg_off; + emsg_off++; char_u *s = (char_u *) encode_tv2string(fc->rettv, NULL); char_u *tofree = s; - --emsg_off; + emsg_off--; if (s != NULL) { if (vim_strsize(s) > MSG_BUF_CLEN) { trunc_string(s, buf, MSG_BUF_CLEN, MSG_BUF_LEN); diff --git a/src/nvim/fileio.c b/src/nvim/fileio.c index f4fc744e04..32e1b645d0 100644 --- a/src/nvim/fileio.c +++ b/src/nvim/fileio.c @@ -4816,11 +4816,11 @@ buf_check_timestamp ( busy = true; set_vim_var_string(VV_FCS_REASON, reason, -1); set_vim_var_string(VV_FCS_CHOICE, "", -1); - ++allbuf_lock; + allbuf_lock++; n = apply_autocmds(EVENT_FILECHANGEDSHELL, - buf->b_fname, buf->b_fname, FALSE, buf); - --allbuf_lock; - busy = FALSE; + buf->b_fname, buf->b_fname, false, buf); + allbuf_lock--; + busy = false; if (n) { if (!buf_valid(buf)) EMSG(_("E246: FileChangedShell autocommand deleted buffer")); From a64114eba017c0db3d1849186c9c54fb09308761 Mon Sep 17 00:00:00 2001 From: ZyX Date: Mon, 18 Apr 2016 01:37:21 +0300 Subject: [PATCH 82/82] functests: Make json_functions_spec use new NIL where appropriate --- test/functional/api/vim_spec.lua | 2 +- test/functional/eval/json_functions_spec.lua | 4 ++-- test/functional/eval/special_vars_spec.lua | 14 +++++++------- test/functional/eval/string_spec.lua | 3 +++ test/functional/plugin/msgpack_spec.lua | 4 ++-- 5 files changed, 15 insertions(+), 12 deletions(-) diff --git a/test/functional/api/vim_spec.lua b/test/functional/api/vim_spec.lua index d7faaa9d2d..389badb423 100644 --- a/test/functional/api/vim_spec.lua +++ b/test/functional/api/vim_spec.lua @@ -92,7 +92,7 @@ describe('vim_* functions', function() it('del_var returns the old value', function() local val1 = {1, 2, {['3'] = 1}} local val2 = {4, 7} - eq(nil, meths.set_var('lua', val1)) + eq(NIL, meths.set_var('lua', val1)) eq(val1, meths.set_var('lua', val2)) eq(val2, meths.del_var('lua')) end) diff --git a/test/functional/eval/json_functions_spec.lua b/test/functional/eval/json_functions_spec.lua index 0a7c4cf669..1cece78ce1 100644 --- a/test/functional/eval/json_functions_spec.lua +++ b/test/functional/eval/json_functions_spec.lua @@ -79,7 +79,7 @@ describe('json_decode() function', function() end) it('parses null, true, false', function() - eq(nil, funcs.json_decode('null')) + eq(NIL, funcs.json_decode('null')) eq(true, funcs.json_decode('true')) eq(false, funcs.json_decode('false')) end) @@ -309,7 +309,7 @@ describe('json_decode() function', function() it('parses containers', function() eq({1}, funcs.json_decode('[1]')) - eq({nil, 1}, funcs.json_decode('[null, 1]')) + eq({NIL, 1}, funcs.json_decode('[null, 1]')) eq({['1']=2}, funcs.json_decode('{"1": 2}')) eq({['1']=2, ['3']={{['4']={['5']={{}, 1}}}}}, funcs.json_decode('{"1": 2, "3": [{"4": {"5": [[], 1]}}]}')) diff --git a/test/functional/eval/special_vars_spec.lua b/test/functional/eval/special_vars_spec.lua index b003dceb04..6ab4e6a7d7 100644 --- a/test/functional/eval/special_vars_spec.lua +++ b/test/functional/eval/special_vars_spec.lua @@ -27,13 +27,13 @@ describe('Special values', function() it('work with empty()', function() eq(0, funcs.empty(true)) eq(1, funcs.empty(false)) - eq(1, eval('empty(v:null)')) + eq(1, funcs.empty(NIL)) end) it('can be stringified and eval’ed back', function() eq(true, funcs.eval(funcs.string(true))) eq(false, funcs.eval(funcs.string(false))) - eq(nil, eval('eval(string(v:null))')) + eq(NIL, funcs.eval(funcs.string(NIL))) end) it('work with is/isnot properly', function() @@ -133,17 +133,17 @@ describe('Special values', function() it('work with type()', function() eq(6, funcs.type(true)) eq(6, funcs.type(false)) - eq(7, eval('type(v:null)')) + eq(7, funcs.type(NIL)) end) it('work with copy() and deepcopy()', function() eq(true, funcs.deepcopy(true)) eq(false, funcs.deepcopy(false)) - eq(nil, eval('deepcopy(v:null)')) + eq(NIL, funcs.deepcopy(NIL)) eq(true, funcs.copy(true)) eq(false, funcs.copy(false)) - eq(nil, eval('copy(v:null)')) + eq(NIL, funcs.copy(NIL)) end) it('fails in index', function() @@ -155,11 +155,11 @@ describe('Special values', function() it('is accepted by assert_true and assert_false', function() funcs.assert_false(false) funcs.assert_false(true) - eval('assert_false(v:null)') + funcs.assert_false(NIL) funcs.assert_true(false) funcs.assert_true(true) - eval('assert_true(v:null)') + funcs.assert_true(NIL) eq({ 'Expected False but got v:true', diff --git a/test/functional/eval/string_spec.lua b/test/functional/eval/string_spec.lua index 20a800cde0..0fd7587edb 100644 --- a/test/functional/eval/string_spec.lua +++ b/test/functional/eval/string_spec.lua @@ -32,6 +32,9 @@ describe('string() function', function() eq('v:true', eval('string(v:true)')) eq('v:false', eval('string(v:false)')) eq('v:null', eval('string(v:null)')) + eq('v:true', funcs.string(true)) + eq('v:false', funcs.string(false)) + eq('v:null', funcs.string(NIL)) end) it('dumps values with at most six digits after the decimal point', diff --git a/test/functional/plugin/msgpack_spec.lua b/test/functional/plugin/msgpack_spec.lua index 197a1a92e5..246b26188f 100644 --- a/test/functional/plugin/msgpack_spec.lua +++ b/test/functional/plugin/msgpack_spec.lua @@ -534,7 +534,7 @@ describe('In autoload/msgpack.vim', function() it('works for special v: values like v:true', function() meths.set_var('true', true) meths.set_var('false', false) - nvim_command('let nil = v:null') + meths.set_var('nil', NIL) nvim_command('let true2 = msgpack#deepcopy(true)') nvim_command('let false2 = msgpack#deepcopy(false)') @@ -542,7 +542,7 @@ describe('In autoload/msgpack.vim', function() eq(true, meths.get_var('true')) eq(false, meths.get_var('false')) - eq(nil, meths.get_var('nil')) + eq(NIL, meths.get_var('nil')) end) end)