Re: Process output truncation when using UTF-8

[Kenichi Handa]

In article <874r3f1ocb.fsf@zamazal.org>, Milan Zamazal <pdm@zamazal.org> writes:
> When I start a process, set its input and output encoding
> to utf-8 and send a long string to it through
> process-send-string, usually some final part of the sent
> string is missing on the recipient's side.

Thank you for the report.  I've just installed the attached
change in RC and HEAD.   It should be applicable also to
Emacs 21.3.

---
Ken'ichi HANDA
handa@m17n.org

2003-05-30  Kenichi Handa  <handa@m17n.org>

        * coding.c (ccl_coding_driver): Set ccl->eight_bit_control
        properly before calling ccl_driver.

        * ccl.h (struct ccl_program) <eight_bit_control: Comment fixed.

        * ccl.c (CCL_WRITE_CHAR): Increment extra_bytes only when it is
        nonzero.
        (ccl_driver): Initialize extra_bytes to ccl->eight_bit_control.
        (setup_ccl_program): Initialize ccl->eight_bit_control to zero.

Index: coding.c
===================================================================
RCS file: /cvsroot/emacs/emacs/src/coding.c,v
retrieving revision 1.222.4.16
retrieving revision 1.222.4.17
diff -u -c -r1.222.4.16 -r1.222.4.17
cvs server: conflicting specifications of output style
*** coding.c    7 Mar 2003 04:37:21 -0000       1.222.4.16
--- coding.c    30 May 2003 08:12:19 -0000      1.222.4.17
***************
*** 4477,4483 ****
--- 4477,4486 ----
        if (ccl->eol_type ==CODING_EOL_UNDECIDED)
        ccl->eol_type = CODING_EOL_LF;
        ccl->cr_consumed = coding->spec.ccl.cr_carryover;
+       ccl->eight_bit_control = coding->dst_multibyte;
      }
+   else
+     ccl->eight_bit_control = 1;
    ccl->multibyte = coding->src_multibyte;
    if (coding->spec.ccl.eight_bit_carryover[0] != 0)
      {
Index: ccl.h
===================================================================
RCS file: /cvsroot/emacs/emacs/src/ccl.h,v
retrieving revision 1.16
retrieving revision 1.16.12.1
diff -u -c -r1.16 -r1.16.12.1
cvs server: conflicting specifications of output style
*** ccl.h       27 Feb 2001 03:29:08 -0000      1.16
--- ccl.h       30 May 2003 08:12:38 -0000      1.16.12.1
***************
*** 65,72 ****
                                   system.  */
    int suppress_error;         /* If nonzero, don't insert error
                                   message in the output.  */
!   int eight_bit_control;      /* Set to nonzero if CCL_WRITE_CHAR
!                                  writes eight-bit-control char.  */
  };
  
  /* This data type is used for the spec field of the structure
--- 65,75 ----
                                   system.  */
    int suppress_error;         /* If nonzero, don't insert error
                                   message in the output.  */
!   int eight_bit_control;      /* If nonzero, ccl_driver counts all
!                                  eight-bit-control bytes written by
!                                  CCL_WRITE_CHAR.  After execution,
!                                  if no such byte is written, set
!                                  this value to zero.  */
  };
  
  /* This data type is used for the spec field of the structure
Index: ccl.c
===================================================================
RCS file: /cvsroot/emacs/emacs/src/ccl.c,v
retrieving revision 1.71
retrieving revision 1.71.12.1
diff -u -c -r1.71 -r1.71.12.1
cvs server: conflicting specifications of output style
*** ccl.c       17 May 2001 09:09:14 -0000      1.71
--- ccl.c       30 May 2003 08:13:01 -0000      1.71.12.1
***************
*** 717,723 ****
        if (bytes == 1)                                                 \
          {                                                             \
            *dst++ = (ch);                                              \
!           if ((ch) >= 0x80 && (ch) < 0xA0)                            \
              /* We may have to convert this eight-bit char to          \
                 multibyte form later.  */                              \
              extra_bytes++;                                            \
--- 717,723 ----
        if (bytes == 1)                                                 \
          {                                                             \
            *dst++ = (ch);                                              \
!           if (extra_bytes && (ch) >= 0x80 && (ch) < 0xA0)             \
              /* We may have to convert this eight-bit char to          \
                 multibyte form later.  */                              \
              extra_bytes++;                                            \
***************
*** 731,736 ****
--- 731,737 ----
        CCL_SUSPEND (CCL_STAT_SUSPEND_BY_DST);                          \
    } while (0)
  
+ 
  /* Encode one character CH to multibyte form and write to the current
     output buffer.  The output bytes always forms a valid multibyte
     sequence.  */
***************
*** 874,880 ****
       each of them will be converted to multibyte form of 2-byte
       sequence.  For that conversion, we remember how many more bytes
       we must keep in DESTINATION in this variable.  */
!   int extra_bytes = 0;
  
    if (ic >= ccl->eof_ic)
      ic = CCL_HEADER_MAIN;
--- 875,881 ----
       each of them will be converted to multibyte form of 2-byte
       sequence.  For that conversion, we remember how many more bytes
       we must keep in DESTINATION in this variable.  */
!   int extra_bytes = ccl->eight_bit_control;
  
    if (ic >= ccl->eof_ic)
      ic = CCL_HEADER_MAIN;
***************
*** 1849,1855 ****
    ccl->ic = ic;
    ccl->stack_idx = stack_idx;
    ccl->prog = ccl_prog;
!   ccl->eight_bit_control = (extra_bytes > 0);
    if (consumed)
      *consumed = src - source;
    return (dst ? dst - destination : 0);
--- 1850,1856 ----
    ccl->ic = ic;
    ccl->stack_idx = stack_idx;
    ccl->prog = ccl_prog;
!   ccl->eight_bit_control = (extra_bytes > 1);
    if (consumed)
      *consumed = src - source;
    return (dst ? dst - destination : 0);
***************
*** 2004,2009 ****
--- 2005,2011 ----
    ccl->stack_idx = 0;
    ccl->eol_type = CODING_EOL_LF;
    ccl->suppress_error = 0;
+   ccl->eight_bit_control = 0;
    return 0;
  }