kevin/neovim
1
Fork 0
mirror of https://github.com/neovim/neovim.git synced 2024-12-20 11:15:14 -07:00
Code Issues Packages Projects Releases Wiki Activity

eval/*code,shada: Drop support for converting UTF-8 from/to p_enc

Browse Source 
Not needed any longer since p_enc is always utf-8.
This commit is contained in:
ZyX 2017-04-08 18:03:56 +03:00
parent 6006cb74ef
commit e814695223
3 changed files with 45 additions and 284 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
src/nvim/eval/decode.c
View File

@ -222,8 +222,6 @@ static inline int json_decoder_pop(ValuesStackItem obj,
/// Parse JSON double-quoted string
///
/// @param[in] conv Defines conversion necessary to convert UTF-8 string to
/// &encoding.
/// @param[in] buf Buffer being converted.
/// @param[in] buf_len Length of the buffer.
/// @param[in,out] pp Pointer to the start of the string. Must point to '"'.
@ -240,8 +238,7 @@ static inline int json_decoder_pop(ValuesStackItem obj,
/// value when decoder is restarted, otherwise unused.
///
/// @return OK in case of success, FAIL in case of error.
static inline int parse_json_string(vimconv_T *const conv,
const char *const buf, const size_t buf_len,
static inline int parse_json_string(const char *const buf, const size_t buf_len,
const char **const pp,
ValuesStack *const stack,
ContainerStack *const container_stack,
@ -416,20 +413,6 @@ static inline int parse_json_string(vimconv_T *const conv,
}
PUT_FST_IN_PAIR(fst_in_pair, str_end);
#undef PUT_FST_IN_PAIR
if (conv->vc_type != CONV_NONE) {
size_t str_len = (size_t) (str_end - str);
char *const new_str = (char *) string_convert(conv, (char_u *) str,
&str_len);
if (new_str == NULL) {
emsgf(_("E474: Failed to convert string \"%.*s\" from UTF-8"),
(int) str_len, str);
xfree(str);
goto parse_json_string_fail;
}
xfree(str);
str = new_str;
str_end = new_str + str_len;
}
if (hasnul) {
typval_T obj;
list_T *const list = tv_list_alloc();
@ -606,17 +589,6 @@ parse_json_number_ret:
} \
} while (0)
/// Last used p_enc value
///
/// Generic pointer: it is not used as a string, only pointer comparisons are
/// performed. Must not be freed.
static const void *last_p_enc = NULL;
/// Conversion setup for converting from UTF-8 to last_p_enc
static vimconv_T p_enc_conv = {
.vc_type = CONV_NONE,
};
/// Convert JSON string into VimL object
///
/// @param[in] buf String to convert. UTF-8 encoding is assumed.
@ -637,12 +609,7 @@ int json_decode_string(const char *const buf, const size_t buf_len,
EMSG(_("E474: Attempt to decode a blank string"));
return FAIL;
}
if (last_p_enc != (const void *)p_enc) {
p_enc_conv.vc_type = CONV_NONE;
convert_setup(&p_enc_conv, (char_u *)"utf-8", p_enc);
p_enc_conv.vc_fail = true;
last_p_enc = p_enc;
}
assert(STRCMP(p_enc, "utf-8") == 0);
int ret = OK;
ValuesStack stack = KV_INITIAL_VALUE;
ContainerStack container_stack = KV_INITIAL_VALUE;
@ -789,7 +756,7 @@ json_decode_string_cycle_start:
}
case '"': {
if (parse_json_string(
&p_enc_conv, buf, buf_len, &p, &stack, &container_stack,
buf, buf_len, &p, &stack, &container_stack,
&next_map_special, &didcomma, &didcolon) == FAIL) {
// Error message was already given
goto json_decode_string_fail;

56
src/nvim/eval/encode.c
View File

@ -11,7 +11,7 @@
#include <math.h>
#include "nvim/eval/encode.h"
#include "nvim/buffer_defs.h" // vimconv_T
#include "nvim/buffer_defs.h"
#include "nvim/eval.h"
#include "nvim/eval/typval.h"
#include "nvim/garray.h"
@ -29,10 +29,6 @@
#define utf_ptr2char(b) utf_ptr2char((char_u *)b)
#define utf_ptr2len(b) ((size_t)utf_ptr2len((char_u *)b))
#define utf_char2len(b) ((size_t)utf_char2len(b))
#define string_convert(a, b, c) \
((char *)string_convert((vimconv_T *)a, (char_u *)b, c))
#define convert_setup(vcp, from, to) \
(convert_setup(vcp, (char_u *)from, (char_u *)to))
const char *const encode_special_var_names[] = {
[kSpecialVarNull] = "null",
@ -537,17 +533,6 @@ int encode_read_from_list(ListReaderState *const state, char *const buf,
} \
} while (0)
/// Last used p_enc value
///
/// Generic pointer: it is not used as a string, only pointer comparisons are
/// performed. Must not be freed.
static const void *last_p_enc = NULL;
/// Conversion setup for converting from last_p_enc to UTF-8
static vimconv_T p_enc_conv = {
.vc_type = CONV_NONE,
};
/// Escape sequences used in JSON
static const char escapes[][3] = {
[BS] = "\\b",
@ -579,33 +564,16 @@ static inline int convert_to_json_string(garray_T *const gap,
} else {
size_t utf_len = len;
char *tofree = NULL;
if (last_p_enc != (const void *) p_enc) {
p_enc_conv.vc_type = CONV_NONE;
convert_setup(&p_enc_conv, p_enc, "utf-8");
p_enc_conv.vc_fail = true;
last_p_enc = p_enc;
}
if (p_enc_conv.vc_type != CONV_NONE) {
tofree = string_convert(&p_enc_conv, buf, &utf_len);
if (tofree == NULL) {
emsgf(_("E474: Failed to convert string \"%.*s\" to UTF-8"),
utf_len, utf_buf);
return FAIL;
}
utf_buf = tofree;
}
assert(STRCMP(p_enc, "utf-8") == 0);
size_t str_len = 0;
// Encode character as \u0000 if
// 1. It is an ASCII control character (0x0 .. 0x1F, 0x7F).
// 2. &encoding is not UTF-8 and code point is above 0x7F.
// 3. &encoding is UTF-8 and code point is not printable according to
// utf_printable().
// This is done to make it possible to :echo values when &encoding is not
// UTF-8.
#define ENCODE_RAW(p_enc_conv, ch) \
(ch >= 0x20 && (p_enc_conv.vc_type == CONV_NONE \
? utf_printable(ch) \
: ch < 0x7F))
// Encode character as \uNNNN if
// 1. It is an ASCII control character (0x0 .. 0x1F; 0x7F not
// utf_printable and thus not checked specially).
// 2. Code point is not printable according to utf_printable().
// This is done to make resulting values displayable on screen also not from
// Neovim.
#define ENCODE_RAW(ch) \
(ch >= 0x20 && utf_printable(ch))
for (size_t i = 0; i < utf_len;) {
const int ch = utf_ptr2char(utf_buf + i);
const size_t shift = (ch == 0? 1: utf_ptr2len(utf_buf + i));
@ -636,7 +604,7 @@ static inline int convert_to_json_string(garray_T *const gap,
utf_len - (i - shift), utf_buf + i - shift);
xfree(tofree);
return FAIL;
} else if (ENCODE_RAW(p_enc_conv, ch)) {
} else if (ENCODE_RAW(ch)) {
str_len += shift;
} else {
str_len += ((sizeof("\\u1234") - 1)
@ -666,7 +634,7 @@ static inline int convert_to_json_string(garray_T *const gap,
break;
}
default: {
if (ENCODE_RAW(p_enc_conv, ch)) {
if (ENCODE_RAW(ch)) {
ga_concat_len(gap, utf_buf + i, shift);
} else if (ch < SURROGATE_FIRST_CHAR) {
ga_concat_len(gap, ((const char[]) {

234
src/nvim/shada.c
View File

@ -73,15 +73,10 @@ KHASH_SET_INIT_STR(strset)
(vim_rename((char_u *)a, (char_u *)b))
#define mb_strnicmp(a, b, c) \
(mb_strnicmp((char_u *)a, (char_u *)b, c))
#define has_non_ascii(a) (has_non_ascii((char_u *)a))
#define string_convert(a, b, c) \
((char *)string_convert((vimconv_T *)a, (char_u *)b, c))
#define path_shorten_fname_if_possible(b) \
((char *)path_shorten_fname_if_possible((char_u *)b))
#define buflist_new(ffname, sfname, ...) \
(buflist_new((char_u *)ffname, (char_u *)sfname, __VA_ARGS__))
#define convert_setup(vcp, from, to) \
(convert_setup(vcp, (char_u *)from, (char_u *)to))
#define os_isdir(f) (os_isdir((char_u *) f))
#define regtilde(s, m) ((char *) regtilde((char_u *) s, m))
#define path_tail_with_sep(f) ((char *) path_tail_with_sep((char_u *)f))
@ -413,8 +408,6 @@ typedef struct sd_read_def {
const char *error; ///< Error message in case of error.
uintmax_t fpos; ///< Current position (amount of bytes read since
///< reader structure initialization). May overflow.
vimconv_T sd_conv; ///< Structure used for converting encodings of some
///< items.
} ShaDaReadDef;
struct sd_write_def;
@ -435,8 +428,6 @@ typedef struct sd_write_def {
ShaDaWriteCloser close; ///< Close function.
void *cookie; ///< Data describing object written to.
const char *error; ///< Error message in case of error.
vimconv_T sd_conv; ///< Structure used for converting encodings of some
///< items.
} ShaDaWriteDef;
#ifdef INCLUDE_GENERATED_DECLARATIONS
@ -709,7 +700,6 @@ static ptrdiff_t write_file(ShaDaWriteDef *const sd_writer,
static void close_sd_reader(ShaDaReadDef *const sd_reader)
FUNC_ATTR_NONNULL_ALL
{
convert_setup(&sd_reader->sd_conv, NULL, NULL);
close_file(sd_reader->cookie);
}
@ -717,7 +707,6 @@ static void close_sd_reader(ShaDaReadDef *const sd_reader)
static void close_sd_writer(ShaDaWriteDef *const sd_writer)
FUNC_ATTR_NONNULL_ALL
{
convert_setup(&sd_writer->sd_conv, NULL, NULL);
close_file(sd_writer->cookie);
}
@ -800,13 +789,12 @@ static int open_shada_file_for_reading(const char *const fname,
.eof = false,
.fpos = 0,
.cookie = file_open_new(&error, fname, kFileReadOnly, 0),
.sd_conv.vc_type = CONV_NONE,
};
if (sd_reader->cookie == NULL) {
return error;
}
convert_setup(&sd_reader->sd_conv, "utf-8", p_enc);
assert(STRCMP(p_enc, "utf-8") == 0);
return 0;
}
@ -1902,127 +1890,24 @@ shada_pack_entry_error:
}
#undef PACK_STRING
/// Write single ShaDa entry, converting it if needed
/// Write single ShaDa entry and free it afterwards
///
/// @warning Frees entry after packing.
/// Will not free if entry could not be freed.
///
/// @param[in] packer Packer used to write entry.
/// @param[in] sd_conv Conversion definitions.
/// @param[in] entry Entry written. If entry.can_free_entry is false then
/// it assumes that entry was not converted, otherwise it
/// is assumed that entry was already converted.
/// @param[in] entry Entry written.
/// @param[in] max_kbyte Maximum size of an item in KiB. Zero means no
/// restrictions.
static ShaDaWriteResult shada_pack_encoded_entry(msgpack_packer *const packer,
const vimconv_T *const sd_conv,
PossiblyFreedShadaEntry entry,
const size_t max_kbyte)
FUNC_ATTR_NONNULL_ALL
static inline ShaDaWriteResult shada_pack_pfreed_entry(
msgpack_packer *const packer, PossiblyFreedShadaEntry entry,
const size_t max_kbyte)
FUNC_ATTR_NONNULL_ALL FUNC_ATTR_ALWAYS_INLINE
{
ShaDaWriteResult ret = kSDWriteSuccessfull;
ret = shada_pack_entry(packer, entry.data, max_kbyte);
if (entry.can_free_entry) {
ret = shada_pack_entry(packer, entry.data, max_kbyte);
shada_free_shada_entry(&entry.data);
return ret;
}
#define RUN_WITH_CONVERTED_STRING(cstr, code) \
do { \
bool did_convert = false; \
if (sd_conv->vc_type != CONV_NONE && has_non_ascii((cstr))) { \
char *const converted_string = string_convert(sd_conv, (cstr), NULL); \
if (converted_string != NULL) { \
(cstr) = converted_string; \
did_convert = true; \
} \
} \
code \
if (did_convert) { \
xfree((cstr)); \
} \
} while (0)
switch (entry.data.type) {
case kSDItemUnknown:
case kSDItemMissing: {
assert(false);
}
case kSDItemSearchPattern: {
RUN_WITH_CONVERTED_STRING(entry.data.data.search_pattern.pat, {
ret = shada_pack_entry(packer, entry.data, max_kbyte);
});
break;
}
case kSDItemHistoryEntry: {
RUN_WITH_CONVERTED_STRING(entry.data.data.history_item.string, {
ret = shada_pack_entry(packer, entry.data, max_kbyte);
});
break;
}
case kSDItemSubString: {
RUN_WITH_CONVERTED_STRING(entry.data.data.sub_string.sub, {
ret = shada_pack_entry(packer, entry.data, max_kbyte);
});
break;
}
case kSDItemVariable: {
if (sd_conv->vc_type != CONV_NONE) {
typval_T tgttv;
var_item_copy(sd_conv, &entry.data.data.global_var.value, &tgttv,
true, 0);
tv_clear(&entry.data.data.global_var.value);
entry.data.data.global_var.value = tgttv;
}
ret = shada_pack_entry(packer, entry.data, max_kbyte);
break;
}
case kSDItemRegister: {
bool did_convert = false;
if (sd_conv->vc_type != CONV_NONE) {
size_t first_non_ascii = 0;
for (size_t i = 0; i < entry.data.data.reg.contents_size; i++) {
if (has_non_ascii(entry.data.data.reg.contents[i])) {
first_non_ascii = i;
did_convert = true;
break;
}
}
if (did_convert) {
entry.data.data.reg.contents =
xmemdup(entry.data.data.reg.contents,
(entry.data.data.reg.contents_size
* sizeof(entry.data.data.reg.contents[0])));
for (size_t i = 0; i < entry.data.data.reg.contents_size; i++) {
if (i >= first_non_ascii) {
entry.data.data.reg.contents[i] = get_converted_string(
sd_conv,
entry.data.data.reg.contents[i],
strlen(entry.data.data.reg.contents[i]));
} else {
entry.data.data.reg.contents[i] =
xstrdup(entry.data.data.reg.contents[i]);
}
}
}
}
ret = shada_pack_entry(packer, entry.data, max_kbyte);
if (did_convert) {
for (size_t i = 0; i < entry.data.data.reg.contents_size; i++) {
xfree(entry.data.data.reg.contents[i]);
}
xfree(entry.data.data.reg.contents);
}
break;
}
case kSDItemHeader:
case kSDItemGlobalMark:
case kSDItemJump:
case kSDItemBufferList:
case kSDItemLocalMark:
case kSDItemChange: {
ret = shada_pack_entry(packer, entry.data, max_kbyte);
break;
}
}
#undef RUN_WITH_CONVERTED_STRING
return ret;
}
@ -2559,11 +2444,7 @@ static ShaDaWriteResult shada_write(ShaDaWriteDef *const sd_writer,
break;
}
typval_T tgttv;
if (sd_writer->sd_conv.vc_type != CONV_NONE) {
var_item_copy(&sd_writer->sd_conv, &vartv, &tgttv, true, 0);
} else {
tv_copy(&vartv, &tgttv);
}
tv_copy(&vartv, &tgttv);
ShaDaWriteResult spe_ret;
if ((spe_ret = shada_pack_entry(packer, (ShadaEntry) {
.type = kSDItemVariable,
@ -2814,9 +2695,8 @@ static ShaDaWriteResult shada_write(ShaDaWriteDef *const sd_writer,
do { \
for (size_t i_ = 0; i_ < ARRAY_SIZE(wms_array); i_++) { \
if (wms_array[i_].data.type != kSDItemMissing) { \
if (shada_pack_encoded_entry(packer, &sd_writer->sd_conv, \
wms_array[i_], \
max_kbyte) == kSDWriteFailed) { \
if (shada_pack_pfreed_entry(packer, wms_array[i_], max_kbyte) \
== kSDWriteFailed) { \
ret = kSDWriteFailed; \
goto shada_write_exit; \
} \
@ -2826,8 +2706,8 @@ static ShaDaWriteResult shada_write(ShaDaWriteDef *const sd_writer,
PACK_WMS_ARRAY(wms->global_marks);
PACK_WMS_ARRAY(wms->registers);
for (size_t i = 0; i < wms->jumps_size; i++) {
if (shada_pack_encoded_entry(packer, &sd_writer->sd_conv, wms->jumps[i],
max_kbyte) == kSDWriteFailed) {
if (shada_pack_pfreed_entry(packer, wms->jumps[i], max_kbyte)
== kSDWriteFailed) {
ret = kSDWriteFailed;
goto shada_write_exit;
}
@ -2835,8 +2715,8 @@ static ShaDaWriteResult shada_write(ShaDaWriteDef *const sd_writer,
#define PACK_WMS_ENTRY(wms_entry) \
do { \
if (wms_entry.data.type != kSDItemMissing) { \
if (shada_pack_encoded_entry(packer, &sd_writer->sd_conv, wms_entry, \
max_kbyte) == kSDWriteFailed) { \
if (shada_pack_pfreed_entry(packer, wms_entry, max_kbyte) \
== kSDWriteFailed) { \
ret = kSDWriteFailed; \
goto shada_write_exit; \
} \
@ -2863,9 +2743,8 @@ static ShaDaWriteResult shada_write(ShaDaWriteDef *const sd_writer,
for (size_t i = 0; i < file_markss_to_dump; i++) {
PACK_WMS_ARRAY(all_file_markss[i]->marks);
for (size_t j = 0; j < all_file_markss[i]->changes_size; j++) {
if (shada_pack_encoded_entry(packer, &sd_writer->sd_conv,
all_file_markss[i]->changes[j],
max_kbyte) == kSDWriteFailed) {
if (shada_pack_pfreed_entry(packer, all_file_markss[i]->changes[j],
max_kbyte) == kSDWriteFailed) {
ret = kSDWriteFailed;
goto shada_write_exit;
}
@ -2889,8 +2768,8 @@ static ShaDaWriteResult shada_write(ShaDaWriteDef *const sd_writer,
if (dump_one_history[i]) {
hms_insert_whole_neovim_history(&wms->hms[i]);
HMS_ITER(&wms->hms[i], cur_entry, {
if (shada_pack_encoded_entry(
packer, &sd_writer->sd_conv, (PossiblyFreedShadaEntry) {
if (shada_pack_pfreed_entry(
packer, (PossiblyFreedShadaEntry) {
.data = cur_entry->data,
.can_free_entry = cur_entry->can_free_entry,
}, max_kbyte) == kSDWriteFailed) {
@ -2940,7 +2819,6 @@ int shada_write_file(const char *const file, bool nomerge)
.write = &write_file,
.close = &close_sd_writer,
.error = NULL,
.sd_conv.vc_type = CONV_NONE,
};
ShaDaReadDef sd_reader = { .close = NULL };
@ -3042,7 +2920,7 @@ shada_write_file_nomerge: {}
verbose_leave();
}
convert_setup(&sd_writer.sd_conv, p_enc, "utf-8");
assert(STRCMP(p_enc, "utf-8") == 0);
const ShaDaWriteResult sw_ret = shada_write(&sd_writer, (nomerge
? NULL
@ -3331,29 +3209,6 @@ static ShaDaReadResult msgpack_read_uint64(ShaDaReadDef *const sd_reader,
return kSDReadStatusSuccess;
}
/// Convert or copy and return a string
///
/// @param[in] sd_conv Conversion definition.
/// @param[in] str String to convert.
/// @param[in] len String length.
///
/// @return [allocated] converted string or copy of the original string.
static inline char *get_converted_string(const vimconv_T *const sd_conv,
const char *const str,
const size_t len)
FUNC_ATTR_NONNULL_ALL FUNC_ATTR_MALLOC FUNC_ATTR_WARN_UNUSED_RESULT
{
if (!has_non_ascii_len(str, len)) {
return xmemdupz(str, len);
}
size_t new_len = len;
char *const new_str = string_convert(sd_conv, str, &new_len);
if (new_str == NULL) {
return xmemdupz(str, len);
}
return new_str;
}
#define READERR(entry_name, error_desc) \
RERR "Error while reading ShaDa file: " \
entry_name " entry at position %" PRIu64 " " \
@ -3431,10 +3286,7 @@ static inline char *get_converted_string(const vimconv_T *const sd_conv,
sizeof(*unpacked.data.via.map.ptr)); \
ad_ga.ga_len++; \
}
#define CONVERTED(str, len) ( \
sd_reader->sd_conv.vc_type != CONV_NONE \
? get_converted_string(&sd_reader->sd_conv, (str), (len)) \
: xmemdupz((str), (len)))
#define CONVERTED(str, len) (xmemdupz((str), (len)))
#define BIN_CONVERTED(b) CONVERTED(b.ptr, b.size)
#define SET_ADDITIONAL_DATA(tgt, name) \
do { \
@ -3807,30 +3659,14 @@ shada_read_next_item_start:
(char) unpacked.data.via.array.ptr[2].via.u64;
}
size_t strsize;
if (sd_reader->sd_conv.vc_type == CONV_NONE
|| !has_non_ascii_len(unpacked.data.via.array.ptr[1].via.bin.ptr,
unpacked.data.via.array.ptr[1].via.bin.size)) {
shada_read_next_item_hist_no_conv:
strsize = (
unpacked.data.via.array.ptr[1].via.bin.size
+ 1 // Zero byte
+ 1); // Separator character
entry->data.history_item.string = xmalloc(strsize);
memcpy(entry->data.history_item.string,
unpacked.data.via.array.ptr[1].via.bin.ptr,
unpacked.data.via.array.ptr[1].via.bin.size);
} else {
size_t len = unpacked.data.via.array.ptr[1].via.bin.size;
char *const converted = string_convert(
&sd_reader->sd_conv, unpacked.data.via.array.ptr[1].via.bin.ptr,
&len);
if (converted != NULL) {
strsize = len + 2;
entry->data.history_item.string = xrealloc(converted, strsize);
} else {
goto shada_read_next_item_hist_no_conv;
}
}
strsize = (
unpacked.data.via.array.ptr[1].via.bin.size
+ 1 // Zero byte
+ 1); // Separator character
entry->data.history_item.string = xmalloc(strsize);
memcpy(entry->data.history_item.string,
unpacked.data.via.array.ptr[1].via.bin.ptr,
unpacked.data.via.array.ptr[1].via.bin.size);
entry->data.history_item.string[strsize - 2] = 0;
entry->data.history_item.string[strsize - 1] =
entry->data.history_item.sep;
@ -3863,16 +3699,6 @@ shada_read_next_item_hist_no_conv:
"be converted to the VimL value")), initial_fpos);
goto shada_read_next_item_error;
}
if (sd_reader->sd_conv.vc_type != CONV_NONE) {
typval_T tgttv;
var_item_copy(&sd_reader->sd_conv,
&entry->data.global_var.value,
&tgttv,
true,
0);
tv_clear(&entry->data.global_var.value);
entry->data.global_var.value = tgttv;
}
SET_ADDITIONAL_ELEMENTS(unpacked.data.via.array, 2,
entry->data.global_var.additional_elements,
"variable");