Improvements to utf8 handling.

Makefile

@@ -8,22 +8,16 @@
 #
 ########################################################################
 
-# Note: the space on the end of the next line is intentional, so it will
-# still work in unix for idiots who got ^M on the end of every line.
-default:  all 
-
-all:
-	@echo :
-	@echo : To build TinyFugue, use one of the following commands
-	@echo : appropriate to your system.  For more information, see
-	@echo : the README file in the directory for your system.
-	@echo :
-	@echo : UNIX, CYGWIN, MACOS X:
-	@echo : ./configure [options]
-	@echo : make install
-	@echo :
-
-install files tf clean uninstall: all
+
+
+LONGPATH="${PATH}:@PATH@"
+
+default: files
+
+files all install tf clean uninstall: _force_
+	@cd src; PATH="${LONGPATH}" "${MAKE}" $@
+
+_force_:
 
 # The next line is a hack to get around a bug in BSD/386 make.
 make:

src/expr.c

@@ -1415,12 +1415,23 @@ static Value *function_switch(const ExprFunc *func, int n, const char *parent)
         case FN_kbpoint:
             return newint(keyboard_pos);
 
+        case FN_kbpoint_prev:
+            return newint(utf8_prev_n_chars(keybuf->data, keybuf->len,
+                keyboard_pos, n > 0 ? opdint(1) : 1));
+
+        case FN_kbpoint_next:
+            return newint(utf8_next_n_chars(keybuf->data, keybuf->len,
+                keyboard_pos, n > 0 ? opdint(1) : 1));
+
         case FN_kbgoto:
             return newint(igoto(opdint(1)));
 
         case FN_kbdel:
             return (newint(do_kbdel(opdint(1))));
 
+        case FN_kbtranspose:
+            return newint(do_kbtranspose());
+
         case FN_kbmatch:
             return newint(do_kbmatch(n>0 ? opdint(1) : keyboard_pos));
 

src/funclist.h

@@ -51,7 +51,10 @@ funccode(kbhead,	0,	0,  0),
 funccode(kblen,		0,	0,  0),
 funccode(kbmatch,	0,	0,  1),
 funccode(kbpoint,	0,	0,  0),
+funccode(kbpoint_next,	0,	0,  1),
+funccode(kbpoint_prev,	0,	0,  1),
 funccode(kbtail,	0,	0,  0),
+funccode(kbtranspose,	0,	0,  0),
 funccode(kbwordleft,	0,	0,  1),
 funccode(kbwordright,	0,	0,  1),
 funccode(keycode,	0,	1,  1),

src/keyboard.c

@@ -26,6 +26,10 @@
 #include "cmdlist.h"
 #include "variable.h"	/* unsetvar() */
 
+#if WIDECHAR
+#include <wchar.h>
+#endif
+
 static int literal_next = FALSE;
 static TrieNode *keynode = NULL;	/* current node matched by input */
 static int kbnum_internal = 0;
@@ -140,16 +144,20 @@ int handle_keyboard_input(int read_flag)
 
     for (i = 0; i < count; i++) {
 #if !WIDECHAR
-        if (istrip) buf[i] &= 0x7F;
-        if (buf[i] & 0x80) {
-	    if (!literal_next &&
-		(meta_esc == META_ON || (!is_print(buf[i]) && meta_esc)))
-	    {
-		Stringadd(current_input, '\033');
-		buf[i] &= 0x7F;
-	    }
-	    if (!is_print(buf[i]))
-		buf[i] &= 0x7F;
+        /* Only apply 7-bit stripping to bytes 0x00-0x7F so UTF-8 (0x80-0xFF)
+         * is preserved. */
+        if ((unsigned char)buf[i] < 0x80) {
+            if (istrip) buf[i] &= 0x7F;
+            if (buf[i] & 0x80) {
+                if (!literal_next &&
+                    (meta_esc == META_ON || (!is_print(buf[i]) && meta_esc)))
+                {
+                    Stringadd(current_input, '\033');
+                    buf[i] &= 0x7F;
+                }
+                if (!is_print(buf[i]))
+                    buf[i] &= 0x7F;
+            }
         }
 #endif
         Stringadd(current_input, mapchar(buf[i]));
@@ -180,7 +188,26 @@ int handle_keyboard_input(int read_flag)
             } else if (s[key_start] == '\b' || s[key_start] == '\177') {
                 handle_input_string(s + input_start, key_start - input_start);
                 place = input_start = ++key_start;
+#if WIDECHAR
+                /*
+                 * In wide-character builds, treat backspace/delete as
+                 * operating on whole UTF-8 characters rather than raw bytes,
+                 * so we never leave the input buffer containing a partial
+                 * multibyte sequence.
+                 */
+                {
+                    int del_from = keyboard_pos;
+                    int count = kbnumval;
+
+                    if (count < 0)
+                        count = -count;
+                    del_from = utf8_prev_n_chars(keybuf->data, keybuf->len,
+                        del_from, count);
+                    do_kbdel(del_from);
+                }
+#else
                 do_kbdel(keyboard_pos - kbnumval);
+#endif
 		reset_kbnum();
             } else if (kbnum && is_digit(s[key_start]) &&
 		key_start == input_start)
@@ -386,17 +413,134 @@ int do_kbdel(int place)
     return keyboard_pos;
 }
 
+/* Transpose the character before the cursor with the character at (or before) the
+ * cursor.  At end of line, swaps the last two characters.  Returns new keyboard_pos
+ * or 0 on failure (beep).  UTF-8 aware when WIDECHAR. */
+int do_kbtranspose(void)
+{
+    int left_start, left_end, right_start, right_end;
+    char left_buf[8], right_buf[8];  /* UTF-8 char can be up to 4 bytes */
+    int left_len, right_len, tail_len;
+
+    if (!keybuf->len) {
+        dobell(1);
+        return 0;
+    }
+    if (keyboard_pos == 0) {
+        dobell(1);
+        return 0;
+    }
+
+#if WIDECHAR
+    left_end = keyboard_pos;
+    left_start = utf8_prev_char(keybuf->data, keybuf->len, left_end);
+    if (keyboard_pos < keybuf->len) {
+        right_start = keyboard_pos;
+        right_end = utf8_next_char(keybuf->data, keybuf->len, right_start);
+    } else {
+        /* At end of line: swap last two characters. */
+        right_start = left_start;
+        right_end = left_end;
+        left_start = utf8_prev_char(keybuf->data, keybuf->len, left_start);
+        left_end = right_start;
+    }
+#else
+    /* Single-byte: swap the byte before cursor with the byte at cursor (or last byte). */
+    left_start = keyboard_pos - 1;
+    left_end = keyboard_pos;
+    if (keyboard_pos < keybuf->len) {
+        right_start = keyboard_pos;
+        right_end = keyboard_pos + 1;
+    } else {
+        right_start = left_start;
+        right_end = left_end;
+        left_start = keyboard_pos - 2;
+        left_end = keyboard_pos - 1;
+        if (left_start < 0) {
+            dobell(1);
+            return 0;
+        }
+    }
+#endif
+
+    left_len = left_end - left_start;
+    right_len = right_end - right_start;
+    if (left_len > (int)sizeof(left_buf) || right_len > (int)sizeof(right_buf)) {
+        dobell(1);
+        return 0;
+    }
+
+    memcpy(left_buf, keybuf->data + left_start, left_len);
+    memcpy(right_buf, keybuf->data + right_start, right_len);
+    tail_len = keybuf->len - right_end;
+    if (tail_len > 0)
+        Stringncpy(scratch, keybuf->data + right_end, tail_len);
+
+    Stringtrunc(keybuf, left_start);
+    Stringncat(keybuf, right_buf, right_len);
+    Stringncat(keybuf, left_buf, left_len);
+    if (tail_len > 0)
+        SStringcat(keybuf, CS(scratch));
+
+    keyboard_pos = left_start + right_len;
+    sync_input_hist();
+    set_refresh_pending(REF_LOGICAL);
+    do_refresh();
+    return keyboard_pos;
+}
+
 #define is_inword(c) (is_alnum(c) || (wordpunct && strchr(wordpunct, (c))))
 
+#if WIDECHAR
+/* Return true if the UTF-8 character at byte offset pos in s[0..len-1] is a
+ * word character (alphanumeric or in wordpunct).  pos must be at the start
+ * of a character. */
+static int is_word_char_at(const char *s, int len, int pos)
+{
+    wchar_t wc;
+    mbstate_t mbs;
+    size_t n;
+
+    if (!s || pos < 0 || pos >= len)
+        return 0;
+    memset(&mbs, 0, sizeof(mbs));
+    n = mbrtowc(&wc, s + pos, (size_t)(len - pos), &mbs);
+    if (n == (size_t)-1 || n == (size_t)-2 || n == 0)
+        return 0;
+    if (wc >= 0 && wc < 128 && wordpunct != NULL && strchr(wordpunct, (char)wc))
+        return 1;
+    return iswalnum((wint_t)wc);
+}
+#endif
+
 int do_kbword(int start, int dir)
 {
     int stop = (dir < 0) ? -1 : keybuf->len;
     int place = start<0 ? 0 : start>keybuf->len ? keybuf->len : start;
     place -= (dir < 0);
 
+#if WIDECHAR
+    /* Normalize to start of character when in the middle of a multibyte. */
+    if (place >= 0 && place < keybuf->len &&
+        ((unsigned char)keybuf->data[place] & 0xC0) == 0x80)
+        place = utf8_prev_char(keybuf->data, keybuf->len, place);
+
+    /* Skip non-word characters; stop at buffer boundary. */
+    while ((dir >= 0 ? place != stop : place > 0) &&
+           !is_word_char_at(keybuf->data, keybuf->len, place))
+        place = (dir < 0) ? utf8_prev_char(keybuf->data, keybuf->len, place)
+                         : utf8_next_char(keybuf->data, keybuf->len, place);
+    /* Skip word characters. */
+    while ((dir >= 0 ? place != stop : place > 0) &&
+           is_word_char_at(keybuf->data, keybuf->len, place))
+        place = (dir < 0) ? utf8_prev_char(keybuf->data, keybuf->len, place)
+                         : utf8_next_char(keybuf->data, keybuf->len, place);
+    return place;
+#else
     while (place != stop && !is_inword(keybuf->data[place])) place += dir;
     while (place != stop && is_inword(keybuf->data[place])) place += dir;
     return place + (dir < 0);
+#endif
 }
 
 int do_kbmatch(int start)
@@ -406,18 +550,35 @@ int do_kbmatch(int start)
     int dir, stop, depth = 0;
     int place = start<0 ? 0 : start>keybuf->len ? keybuf->len : start;
 
+#if WIDECHAR
+    /* Ensure we start at a character boundary. */
+    if (place < keybuf->len && ((unsigned char)keybuf->data[place] & 0xC0) == 0x80)
+        place = utf8_prev_char(keybuf->data, keybuf->len, place);
+#endif
+
     while (1) {
         if (place >= keybuf->len) return -1;
         if ((type = strchr(braces, keybuf->data[place]))) break;
+#if WIDECHAR
+        place = utf8_next_char(keybuf->data, keybuf->len, place);
+#else
         ++place;
+#endif
     }
     dir = ((type - braces) % 2) ? -1 : 1;
     stop = (dir < 0) ? -1 : keybuf->len;
     do {
         if      (keybuf->data[place] == type[0])   depth++;
         else if (keybuf->data[place] == type[dir]) depth--;
         if (depth == 0) return place;
-    } while ((place += dir) != stop);
+#if WIDECHAR
+        place = (dir < 0) ? utf8_prev_char(keybuf->data, keybuf->len, place)
+                         : utf8_next_char(keybuf->data, keybuf->len, place);
+        if (dir < 0 && place <= 0) break;
+#else
+        place += dir;
+#endif
+    } while (place != stop);
     return -1;
 }
 

src/keyboard.h

@@ -19,6 +19,7 @@ extern int           bind_key(Macro *spec, const char *key);
 extern void          unbind_key(const char *key);
 extern struct Macro *find_key(const char *key);
 extern int           do_kbdel(int place);
+extern int           do_kbtranspose(void);
 extern int           do_kbword(int start, int dir);
 extern int           do_kbmatch(int start);
 extern int           handle_keyboard_input(int read_flag);