Fossil

Check-in [c9ec3d18]
Login

Check-in [c9ec3d18]

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:For better word breaking results with the (non-legacy) comment printing algorithm, make sure the lookahead to the next space character is UTF-8-aware. Also make sure the per-line remaining character count is decremented properly for UTF-8 sequences. The neuralgic points now handle UTF-8 sequences correctly, and they could be enhanced to work with the effective display width, if required (to handle combining characters, and East Asian Wide and Fullwidth characters).
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | comment-formatter-utf8
Files: files | file ages | folders
SHA1: c9ec3d1886367b546a37e674df1bff9913d8664a
User & Date: florian 2018-11-16 11:14:00
Context
2018-11-16
11:26
Coding style fixes. ... (check-in: aadbbb38 user: florian tags: comment-formatter-utf8)
11:14
For better word breaking results with the (non-legacy) comment printing algorithm, make sure the lookahead to the next space character is UTF-8-aware. Also make sure the per-line remaining character count is decremented properly for UTF-8 sequences. The neuralgic points now handle UTF-8 sequences correctly, and they could be enhanced to work with the effective display width, if required (to handle combining characters, and East Asian Wide and Fullwidth characters). ... (check-in: c9ec3d18 user: florian tags: comment-formatter-utf8)
2018-11-15
15:32
Ensure the line buffer for the legacy comment printing algorithm can hold maximum-length UTF-8 sequences. ... (check-in: 29d3a2ed user: florian tags: comment-formatter-utf8)
Changes
Unified Diff Ignore Whitespace Patch
Changes to src/comformat.c.
118
119
120
121
122
123
124
125

126
127

128
129

130







131
132
133
134
135
136



































137
138
139
140
141
142
143
** This function scans the specified comment line starting just after the
** initial index and returns the index of the next spacing character -OR-
** zero if such a character cannot be found.  For the purposes of this
** algorithm, the NUL character is treated the same as a spacing character.
*/
static int comment_next_space(
  const char *zLine, /* [in] The comment line being printed. */
  int index          /* [in] The current character index being handled. */

){
  int nextIndex = index + 1;

  for(;;){
    char c = zLine[nextIndex];

    if( c==0 || fossil_isspace(c) ){







      return nextIndex;
    }
    nextIndex++;
  }
  return 0; /* NOT REACHED */
}




































/*
** This function is called when printing a logical comment line to calculate
** the necessary indenting.  The caller needs to emit the indenting spaces.
*/
static void comment_calc_indent(
  const char *zLine, /* [in] The comment line being printed. */







|
>


>


>

>
>
>
>
>
>
>






>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
** This function scans the specified comment line starting just after the
** initial index and returns the index of the next spacing character -OR-
** zero if such a character cannot be found.  For the purposes of this
** algorithm, the NUL character is treated the same as a spacing character.
*/
static int comment_next_space(
  const char *zLine, /* [in] The comment line being printed. */
  int index,         /* [in] The current character index being handled. */
  int *distUTF8      /* [out] Distance to next space in UTF-8 sequences. */
){
  int nextIndex = index + 1;
  int fNonASCII=0;
  for(;;){
    char c = zLine[nextIndex];
    if ( (c&0x80)==0x80 ) fNonASCII=1;
    if( c==0 || fossil_isspace(c) ){
      if ( distUTF8 ){
        if ( fNonASCII!=0 ){
          *distUTF8 = strlen_utf8(&zLine[index], nextIndex-index);
        }else{
          *distUTF8 = nextIndex-index;
        }
      }
      return nextIndex;
    }
    nextIndex++;
  }
  return 0; /* NOT REACHED */
}

/*
** Count the number of UTF-8 sequences in a string. Incomplete, ill-formed and
** overlong sequences are counted as one sequence. The invalid lead bytes 0xC0
** to 0xC1 and 0xF5 to 0xF7 are allowed to initiate (ill-formed) 2- and 4-byte
** sequences, respectively, the other invalid lead bytes 0xF8 to 0xFF are
** treated as invalid 1-byte sequences (as lone trail bytes).
** Combining characters and East Asian Wide and Fullwidth characters are counted
** as one, so this function does not calculate the effective "display width".
*/
int strlen_utf8(const char *zString, int lengthBytes)
{
#if 0
  assert( lengthBytes>=0 );
#endif
  int lengthUTF8=0; /* Counted UTF-8 sequences. */
  int i;
  for( i=0; i<lengthBytes; i++ ){
    char c = zString[i];
    lengthUTF8++;
    if ( (c&0xc0)==0xc0 ){                    /* Any UTF-8 lead byte 11xxxxxx */
      int cchUTF8=1; /* Code units consumed. */
      int maxUTF8=1; /* Expected sequence length. */
      if( (c&0xe0)==0xc0 )maxUTF8=2;          /* UTF-8 lead byte 110vvvvv */
      else if( (c&0xf0)==0xe0 )maxUTF8=3;     /* UTF-8 lead byte 1110vvvv */
      else if( (c&0xf8)==0xf0 )maxUTF8=4;     /* UTF-8 lead byte 11110vvv */
      while( i<lengthBytes-1 &&
              cchUTF8<maxUTF8 &&
              (zString[i+1]&0xc0)==0x80 ){    /* UTF-8 trail byte 10vvvvvv */
        i++;
      }
    }
  }
  return lengthUTF8;
}

/*
** This function is called when printing a logical comment line to calculate
** the necessary indenting.  The caller needs to emit the indenting spaces.
*/
static void comment_calc_indent(
  const char *zLine, /* [in] The comment line being printed. */
225
226
227
228
229
230
231

232
233
234
235
236
237
238
239
240
241
242

243
244
245
246
247
248
249
250
251
      index++;
    }
    if( c=='\n' ){
      lineCnt++;
      charCnt = 0;
      useChars = 0;
    }else if( c=='\t' ){

      int nextIndex = comment_next_space(zLine, index);
      if( nextIndex<=0 || (nextIndex-index)>maxChars ){
        break;
      }
      charCnt++;
      useChars = COMMENT_TAB_WIDTH;
      if( maxChars<useChars ){
        zBuf[iBuf++] = ' ';
        break;
      }
    }else if( wordBreak && fossil_isspace(c) ){

      int nextIndex = comment_next_space(zLine, index);
      if( nextIndex<=0 || (nextIndex-index)>maxChars ){
        break;
      }
      charCnt++;
    }else{
      charCnt++;
    }
    assert( c!='\n' || charCnt==0 );







>
|
|









>
|
|







270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
      index++;
    }
    if( c=='\n' ){
      lineCnt++;
      charCnt = 0;
      useChars = 0;
    }else if( c=='\t' ){
      int distUTF8;
      int nextIndex = comment_next_space(zLine, index, &distUTF8);
      if( nextIndex<=0 || distUTF8>maxChars ){
        break;
      }
      charCnt++;
      useChars = COMMENT_TAB_WIDTH;
      if( maxChars<useChars ){
        zBuf[iBuf++] = ' ';
        break;
      }
    }else if( wordBreak && fossil_isspace(c) ){
      int distUTF8;
      int nextIndex = comment_next_space(zLine, index, &distUTF8);
      if( nextIndex<=0 || distUTF8>maxChars ){
        break;
      }
      charCnt++;
    }else{
      charCnt++;
    }
    assert( c!='\n' || charCnt==0 );
265
266
267
268
269
270
271

272
273
274
275

276
277
278
279
280
281
282
      else if( (c&0xf0)==0xe0 )maxUTF8=3;     /* UTF-8 lead byte 1110vvvv */
      else if( (c&0xf8)==0xf0 )maxUTF8=4;     /* UTF-8 lead byte 11110vvv */
      while( cchUTF8<maxUTF8 &&
              (zLine[index]&0xc0)==0x80 ){    /* UTF-8 trail byte 10vvvvvv */
        cchUTF8++;
        zBuf[iBuf++] = zLine[index++];
      }

    }
    else
      zBuf[iBuf++] = c;
    if( (c&0x80)==0 || (zLine[index+1]&0xc0)!=0xc0 ) maxChars -= useChars;

    if( maxChars<=0 ) break;
    if( c=='\n' ) break;
  }
  if( charCnt>0 ){
    zBuf[iBuf++] = '\n';
    lineCnt++;
  }







>

|

|
>







312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
      else if( (c&0xf0)==0xe0 )maxUTF8=3;     /* UTF-8 lead byte 1110vvvv */
      else if( (c&0xf8)==0xf0 )maxUTF8=4;     /* UTF-8 lead byte 11110vvv */
      while( cchUTF8<maxUTF8 &&
              (zLine[index]&0xc0)==0x80 ){    /* UTF-8 trail byte 10vvvvvv */
        cchUTF8++;
        zBuf[iBuf++] = zLine[index++];
      }
      maxChars--;
    }
    else {
      zBuf[iBuf++] = c;
      maxChars -= useChars;
    }
    if( maxChars<=0 ) break;
    if( c=='\n' ) break;
  }
  if( charCnt>0 ){
    zBuf[iBuf++] = '\n';
    lineCnt++;
  }