Remove the charlen field from strings, calculating it when required

Rosuav · Rosuav · commit 5473e1a1dba2 · 2014-06-08T07:18:42.000+10:00
diff --git a/py/makeqstrdata.py b/py/makeqstrdata.py
@@ -59,8 +59,7 @@ def do_work(infiles):
     for order, ident, qstr in sorted(qstrs.values(), key=lambda x: x[0]):
         qhash = compute_hash(qstr)
         qlen = len(qstr)
-        qchlen = len(qstr.decode("utf-8"))
-        print('Q({}, (const byte*)"\\x{:02x}\\x{:02x}\\x{:02x}\\x{:02x}\\x{:02x}\\x{:02x}" "{}")'.format(ident, qhash & 0xff, (qhash >> 8) & 0xff, qlen & 0xff, (qlen >> 8) & 0xff, qchlen & 0xff, (qchlen >> 8) & 0xff, qstr))
+        print('Q({}, (const byte*)"\\x{:02x}\\x{:02x}\\x{:02x}\\x{:02x}" "{}")'.format(ident, qhash & 0xff, (qhash >> 8) & 0xff, qlen & 0xff, (qlen >> 8) & 0xff, qstr))
 
     return True
 
diff --git a/py/objstr.c b/py/objstr.c
@@ -52,9 +52,6 @@ const mp_obj_t mp_const_empty_bytes;
 // use this macro to extract the string data and length
 #define GET_STR_DATA_LEN(str_obj_in, str_data, str_len) const byte *str_data; uint str_len; if (MP_OBJ_IS_QSTR(str_obj_in)) { str_data = qstr_data(MP_OBJ_QSTR_VALUE(str_obj_in), &str_len); } else { str_len = ((mp_obj_str_t*)str_obj_in)->len; str_data = ((mp_obj_str_t*)str_obj_in)->data; }
 
-// use this macro to extract the string data and both lengths
-#define GET_STR_INFO(str_obj_in, str_data, str_len, str_charlen) const byte *str_data; uint str_len, str_charlen; if (MP_OBJ_IS_QSTR(str_obj_in)) { str_data = qstr_data(MP_OBJ_QSTR_VALUE(str_obj_in), &str_len); str_charlen = qstr_charlen(MP_OBJ_QSTR_VALUE(str_obj_in)); } else { str_len = ((mp_obj_str_t*)str_obj_in)->len; str_charlen = ((mp_obj_str_t*)str_obj_in)->charlen; str_data = ((mp_obj_str_t*)str_obj_in)->data; }
-
 STATIC mp_obj_t mp_obj_new_str_iterator(mp_obj_t str);
 STATIC mp_obj_t mp_obj_new_bytes_iterator(mp_obj_t str);
 STATIC NORETURN void bad_implicit_conversion(mp_obj_t self_in);
@@ -365,7 +362,7 @@ STATIC mp_obj_t str_binary_op(int op, mp_obj_t lhs_in, mp_obj_t rhs_in) {
 
 STATIC mp_obj_t str_subscr(mp_obj_t self_in, mp_obj_t index, mp_obj_t value) {
     mp_obj_type_t *type = mp_obj_get_type(self_in);
-    GET_STR_INFO(self_in, self_data, self_len, self_charlen);
+    GET_STR_DATA_LEN(self_in, self_data, self_len);
     if (value == MP_OBJ_SENTINEL) {
         // load
 #if MICROPY_PY_BUILTINS_SLICE
@@ -378,7 +375,8 @@ STATIC mp_obj_t str_subscr(mp_obj_t self_in, mp_obj_t index, mp_obj_t value) {
             return mp_obj_new_str_of_type(type, self_data + slice.start, slice.stop - slice.start);
         }
 #endif
-        uint index_val = mp_get_index(type, self_charlen, index, false);
+        // TODO: Don't use mp_get_index() here
+        uint index_val = mp_get_index(type, unichar_charlen((const char *)self_data, self_len), index, false);
         if (type == &mp_type_bytes) {
             return MP_OBJ_NEW_SMALL_INT((mp_small_int_t)self_data[index_val]);
         } else {
@@ -1734,7 +1732,7 @@ const mp_obj_type_t mp_type_bytes = {
 };
 
 // the zero-length bytes
-STATIC const mp_obj_str_t empty_bytes_obj = {{&mp_type_bytes}, 0, 0, 0, NULL};
+STATIC const mp_obj_str_t empty_bytes_obj = {{&mp_type_bytes}, 0, 0, NULL};
 const mp_obj_t mp_const_empty_bytes = (mp_obj_t)&empty_bytes_obj;
 
 mp_obj_t mp_obj_str_builder_start(const mp_obj_type_t *type, uint len, byte **data) {
@@ -1761,20 +1759,6 @@ mp_obj_t mp_obj_new_str_of_type(const mp_obj_type_t *type, const byte* data, uin
     o->base.type = type;
     o->len = len;
     if (data) {
-        if (MP_OBJ_IS_STR(o)) {
-            // Count non-continuation bytes so we know how long the string is in characters.
-            const byte *endptr, *top = data + len;
-            uint charlen = 0;
-            for (endptr = data; endptr < top; ++endptr) {
-                if (!UTF8_IS_CONT(*endptr)) {
-                    ++charlen;
-                }
-            }
-            o->charlen = charlen;
-        } else {
-            // For byte strings, the 'character' length (really the "exposed length" or "Python length") equals the byte length.
-            o->charlen = len;
-        }
         o->hash = qstr_compute_hash(data, len);
         byte *p = m_new(byte, len + 1);
         o->data = p;
@@ -1844,8 +1828,8 @@ uint mp_obj_str_get_hash(mp_obj_t self_in) {
 uint mp_obj_str_get_len(mp_obj_t self_in) {
     // TODO This has a double check for the type, one in obj.c and one here
     if (MP_OBJ_IS_STR(self_in) || MP_OBJ_IS_TYPE(self_in, &mp_type_bytes)) {
-        GET_STR_INFO(self_in, self_data, self_len, self_charlen); (void)self_data;
-        return self_charlen;
+        GET_STR_DATA_LEN(self_in, self_data, self_len);
+        return unichar_charlen((const char *)self_data, self_len);
     } else {
         bad_implicit_conversion(self_in);
     }
diff --git a/py/objstr.h b/py/objstr.h
@@ -30,13 +30,10 @@ typedef struct _mp_obj_str_t {
     machine_uint_t hash : 16;
     // len == number of bytes used in data, alloc = len + 1 because (at the moment) we also append a null byte
     machine_uint_t len : 16;
-    // charlen == number of characters in the string - charlen <= len - 1, and is the value returned by len() in Python
-    machine_uint_t charlen : 16;
     const void *data; //Character data is encoded UTF-8 and should not be blindly indexed.
 } mp_obj_str_t;
 
-// This is valid ONLY for pure-ASCII strings!
-#define MP_DEFINE_STR_OBJ(obj_name, str) mp_obj_str_t obj_name = {{&mp_type_str}, 0, sizeof(str) - 1, sizeof(str) - 1, (const byte*)str};
+#define MP_DEFINE_STR_OBJ(obj_name, str) mp_obj_str_t obj_name = {{&mp_type_str}, 0, sizeof(str) - 1, (const byte*)str};
 
 mp_obj_t mp_obj_str_format(uint n_args, const mp_obj_t *args);
 mp_obj_t mp_obj_new_str_of_type(const mp_obj_type_t *type, const byte* data, uint len);
diff --git a/py/qstr.c b/py/qstr.c
@@ -46,15 +46,13 @@
 // For now we use very simple encoding, just to get the framework correct:
 //  - hash is 2 bytes (see function below)
 //  - length is 2 bytes
-//  - character length is 2 bytes
 //  - data follows
 //  - \0 terminated (for now, so they can be printed using printf)
 
 #define Q_GET_HASH(q)   ((q)[0] | ((q)[1] << 8))
-#define Q_GET_ALLOC(q)  (6 + Q_GET_LENGTH(q) + 1)
+#define Q_GET_ALLOC(q)  (4 + Q_GET_LENGTH(q) + 1)
 #define Q_GET_LENGTH(q) ((q)[2] | ((q)[3] << 8))
-#define Q_GET_CHARLEN(q) ((q)[4] | ((q)[5] << 8))
-#define Q_GET_DATA(q)   ((q) + 6)
+#define Q_GET_DATA(q)   ((q) + 4)
 
 // this must match the equivalent function in makeqstrdata.py
 // Note that this hashes the UTF-8 encoded data bytes.
@@ -158,29 +156,21 @@ qstr qstr_from_strn(const char *str, uint len) {
     qstr q = qstr_find_strn(str, len);
     if (q == 0) {
         machine_uint_t hash = qstr_compute_hash((const byte*)str, len);
-        byte *q_ptr = m_new(byte, 6 + len + 1);
-        uint charlen = 0;
-        for (const char *s = str; s < str + len; ++s) {
-            if (!UTF8_IS_CONT(*s)) {
-                ++charlen;
-            }
-        }
+        byte *q_ptr = m_new(byte, 4 + len + 1);
         q_ptr[0] = hash;
         q_ptr[1] = hash >> 8;
         q_ptr[2] = len;
         q_ptr[3] = len >> 8;
-        q_ptr[4] = charlen;
-        q_ptr[5] = charlen >> 8;
-        memcpy(q_ptr + 6, str, len);
-        q_ptr[6 + len] = '\0';
+        memcpy(q_ptr + 4, str, len);
+        q_ptr[4 + len] = '\0';
         q = qstr_add(q_ptr);
     }
     return q;
 }
 
 byte *qstr_build_start(uint len, byte **q_ptr) {
     assert(len <= 65535);
-    *q_ptr = m_new(byte, 7 + len + 1);
+    *q_ptr = m_new(byte, 4 + len + 1);
     (*q_ptr)[2] = len;
     (*q_ptr)[3] = len >> 8;
     return Q_GET_DATA(*q_ptr);
@@ -194,15 +184,7 @@ qstr qstr_build_end(byte *q_ptr) {
         machine_uint_t hash = qstr_compute_hash(str, len);
         q_ptr[0] = hash;
         q_ptr[1] = hash >> 8;
-        uint charlen = 0;
-        for (const byte *s = str; s < str + len; ++s) {
-            if (!UTF8_IS_CONT(*s)) {
-                ++charlen;
-            }
-        }
-        q_ptr[4] = charlen;
-        q_ptr[5] = charlen >> 8;
-        q_ptr[6 + len] = '\0';
+        q_ptr[4 + len] = '\0';
         q = qstr_add(q_ptr);
     } else {
         m_del(byte, q_ptr, Q_GET_ALLOC(q_ptr));
@@ -219,11 +201,6 @@ uint qstr_len(qstr q) {
     return Q_GET_LENGTH(qd);
 }
 
-uint qstr_charlen(qstr q) {
-    const byte *qd = find_qstr(q);
-    return Q_GET_CHARLEN(qd);
-}
-
 // XXX to remove!
 const char *qstr_str(qstr q) {
     const byte *qd = find_qstr(q);
diff --git a/py/qstr.h b/py/qstr.h
@@ -59,7 +59,6 @@ qstr qstr_build_end(byte *q_ptr);
 machine_uint_t qstr_hash(qstr q);
 const char* qstr_str(qstr q);
 uint qstr_len(qstr q);
-uint qstr_charlen(qstr q);
 const byte* qstr_data(qstr q, uint *len);
 
 void qstr_pool_info(uint *n_pool, uint *n_qstr, uint *n_str_data_bytes, uint *n_total_bytes);
