source: patches/coreutils-6.12-i18n-1.patch@ 4ea666a

clfs-1.2 clfs-2.1 clfs-3.0.0-systemd clfs-3.0.0-sysvinit systemd sysvinit
Last change on this file since 4ea666a was d33f80c, checked in by Jim Gifford <clfs@…>, 16 years ago

Put in Correct i18n patch for coreutils
  • Property mode set to 100644
File size: 100.4 KB
RevLine 
[577dd2d]1Submitted By: Jim Gifford <jim at cross-lfs dot org>
2Date: 2009-01-08
3Initial Package Version: 6.12
4Upstream Status: Unkown
[d33f80c]5Origin: Mandriva
[a85d737]6Description: i18n Updates
[577dd2d]7
8diff -Naur coreutils-6.12.orig/lib/linebuffer.h coreutils-6.12/lib/linebuffer.h
9--- coreutils-6.12.orig/lib/linebuffer.h 2007-10-17 06:47:26.000000000 -0700
[d33f80c]10+++ coreutils-6.12/lib/linebuffer.h 2009-01-08 20:14:37.000000000 -0800
[577dd2d]11@@ -21,6 +21,11 @@
12
13 # include <stdio.h>
14
15+/* Get mbstate_t. */
16+# if HAVE_WCHAR_H
17+# include <wchar.h>
18+# endif
19+
20 /* A `struct linebuffer' holds a line of text. */
21
22 struct linebuffer
23@@ -28,6 +33,9 @@
24 size_t size; /* Allocated. */
25 size_t length; /* Used. */
26 char *buffer;
27+# if HAVE_WCHAR_H
28+ mbstate_t state;
29+# endif
30 };
31
32 /* Initialize linebuffer LINEBUFFER for use. */
33diff -Naur coreutils-6.12.orig/src/cut.c coreutils-6.12/src/cut.c
34--- coreutils-6.12.orig/src/cut.c 2008-05-25 23:40:33.000000000 -0700
[d33f80c]35+++ coreutils-6.12/src/cut.c 2009-01-08 20:14:37.000000000 -0800
[577dd2d]36@@ -28,6 +28,11 @@
37 #include <assert.h>
38 #include <getopt.h>
39 #include <sys/types.h>
40+
41+/* Get mbstate_t, mbrtowc(). */
42+#if HAVE_WCHAR_H
43+# include <wchar.h>
44+#endif
45 #include "system.h"
46
47 #include "error.h"
48@@ -36,6 +41,18 @@
49 #include "quote.h"
50 #include "xstrndup.h"
51
52+/* MB_LEN_MAX is incorrectly defined to be 1 in at least one GCC
53+ installation; work around this configuration error. */
54+#if !defined MB_LEN_MAX || MB_LEN_MAX < 2
55+# undef MB_LEN_MAX
56+# define MB_LEN_MAX 16
57+#endif
58+
59+/* Some systems, like BeOS, have multibyte encodings but lack mbstate_t. */
60+#if HAVE_MBRTOWC && defined mbstate_t
61+# define mbrtowc(pwc, s, n, ps) (mbrtowc) (pwc, s, n, 0)
62+#endif
63+
64 /* The official name of this program (e.g., no `g' prefix). */
65 #define PROGRAM_NAME "cut"
66
67@@ -71,6 +88,52 @@
68 } \
69 while (0)
70
71+/* Refill the buffer BUF to get a multibyte character. */
72+#define REFILL_BUFFER(BUF, BUFPOS, BUFLEN, STREAM) \
73+ do \
74+ { \
75+ if (BUFLEN < MB_LEN_MAX && !feof (STREAM) && !ferror (STREAM)) \
76+ { \
77+ memmove (BUF, BUFPOS, BUFLEN); \
78+ BUFLEN += fread (BUF + BUFLEN, sizeof(char), BUFSIZ, STREAM); \
79+ BUFPOS = BUF; \
80+ } \
81+ } \
82+ while (0)
83+
84+/* Get wide character on BUFPOS. BUFPOS is not included after that.
85+ If byte sequence is not valid as a character, CONVFAIL is 1. Otherwise 0. */
86+#define GET_NEXT_WC_FROM_BUFFER(WC, BUFPOS, BUFLEN, MBLENGTH, STATE, CONVFAIL) \
87+ do \
88+ { \
89+ mbstate_t state_bak; \
90+ \
91+ if (BUFLEN < 1) \
92+ { \
93+ WC = WEOF; \
94+ break; \
95+ } \
96+ \
97+ /* Get a wide character. */ \
98+ CONVFAIL = 0; \
99+ state_bak = STATE; \
100+ MBLENGTH = mbrtowc ((wchar_t *)&WC, BUFPOS, BUFLEN, &STATE); \
101+ \
102+ switch (MBLENGTH) \
103+ { \
104+ case (size_t)-1: \
105+ case (size_t)-2: \
106+ CONVFAIL++; \
107+ STATE = state_bak; \
108+ /* Fall througn. */ \
109+ \
110+ case 0: \
111+ MBLENGTH = 1; \
112+ break; \
113+ } \
114+ } \
115+ while (0)
116+
117 struct range_pair
118 {
119 size_t lo;
120@@ -89,7 +152,7 @@
121 /* The number of bytes allocated for FIELD_1_BUFFER. */
122 static size_t field_1_bufsize;
123
124-/* The largest field or byte index used as an endpoint of a closed
125+/* The largest byte, character or field index used as an endpoint of a closed
126 or degenerate range specification; this doesn't include the starting
127 index of right-open-ended ranges. For example, with either range spec
128 `2-5,9-', `2-3,5,9-' this variable would be set to 5. */
129@@ -101,10 +164,11 @@
130
131 /* This is a bit vector.
132 In byte mode, which bytes to output.
133+ In character mode, which characters to output.
134 In field mode, which DELIM-separated fields to output.
135- Both bytes and fields are numbered starting with 1,
136+ Bytes, characters and fields are numbered starting with 1,
137 so the zeroth bit of this array is unused.
138- A field or byte K has been selected if
139+ A byte, character or field K has been selected if
140 (K <= MAX_RANGE_ENDPOINT and is_printable_field(K))
141 || (EOL_RANGE_START > 0 && K >= EOL_RANGE_START). */
142 static unsigned char *printable_field;
143@@ -113,9 +177,12 @@
144 {
145 undefined_mode,
146
147- /* Output characters that are in the given bytes. */
148+ /* Output bytes that are at the given positions. */
149 byte_mode,
150
151+ /* Output characters that are at the given positions. */
152+ character_mode,
153+
154 /* Output the given delimeter-separated fields. */
155 field_mode
156 };
157@@ -125,6 +192,13 @@
158
159 static enum operating_mode operating_mode;
160
161+/* If nonzero, when in byte mode, don't split multibyte characters. */
162+static int byte_mode_character_aware;
163+
164+/* If nonzero, the function for single byte locale is work
165+ if this program runs on multibyte locale. */
166+static int force_singlebyte_mode;
167+
168 /* If true do not output lines containing no delimeter characters.
169 Otherwise, all such lines are printed. This option is valid only
170 with field mode. */
171@@ -136,6 +210,9 @@
172
173 /* The delimeter character for field mode. */
174 static unsigned char delim;
175+#if HAVE_WCHAR_H
176+static wchar_t wcdelim;
177+#endif
178
179 /* True if the --output-delimiter=STRING option was specified. */
180 static bool output_delimiter_specified;
181@@ -209,7 +286,7 @@
182 -f, --fields=LIST select only these fields; also print any line\n\
183 that contains no delimiter character, unless\n\
184 the -s option is specified\n\
185- -n (ignored)\n\
186+ -n with -b: don't split multibyte characters\n\
187 "), stdout);
188 fputs (_("\
189 --complement complement the set of selected bytes, characters\n\
190@@ -368,7 +445,7 @@
191 in_digits = false;
192 /* Starting a range. */
193 if (dash_found)
194- FATAL_ERROR (_("invalid byte or field list"));
195+ FATAL_ERROR (_("invalid byte, character or field list"));
196 dash_found = true;
197 fieldstr++;
198
199@@ -392,14 +469,16 @@
200 if (!rhs_specified)
201 {
202 /* `n-'. From `initial' to end of line. */
203- eol_range_start = initial;
204+ if (eol_range_start == 0 ||
205+ (eol_range_start != 0 && eol_range_start > initial))
206+ eol_range_start = initial;
207 field_found = true;
208 }
209 else
210 {
211 /* `m-n' or `-n' (1-n). */
212 if (value < initial)
213- FATAL_ERROR (_("invalid decreasing range"));
214+ FATAL_ERROR (_("invalid byte, character or field list"));
215
216 /* Is there already a range going to end of line? */
217 if (eol_range_start != 0)
218@@ -479,6 +558,9 @@
219 if (operating_mode == byte_mode)
220 error (0, 0,
221 _("byte offset %s is too large"), quote (bad_num));
222+ else if (operating_mode == character_mode)
223+ error (0, 0,
224+ _("character offset %s is too large"), quote (bad_num));
225 else
226 error (0, 0,
227 _("field number %s is too large"), quote (bad_num));
228@@ -489,7 +571,7 @@
229 fieldstr++;
230 }
231 else
232- FATAL_ERROR (_("invalid byte or field list"));
233+ FATAL_ERROR (_("invalid byte, character or field list"));
234 }
235
236 max_range_endpoint = 0;
237@@ -582,6 +664,63 @@
238 }
239 }
240
241+#if HAVE_MBRTOWC
242+/* This function is in use for the following case.
243+
244+ 1. Read from the stream STREAM, printing to standard output any selected
245+ characters.
246+
247+ 2. Read from stream STREAM, printing to standard output any selected bytes,
248+ without splitting multibyte characters. */
249+
250+static void
251+cut_characters_or_cut_bytes_no_split (FILE *stream)
252+{
253+ int idx; /* number of bytes or characters in the line so far. */
254+ char buf[MB_LEN_MAX + BUFSIZ]; /* For spooling a read byte sequence. */
255+ char *bufpos; /* Next read position of BUF. */
256+ size_t buflen; /* The length of the byte sequence in buf. */
257+ wint_t wc; /* A gotten wide character. */
258+ size_t mblength; /* The byte size of a multibyte character which shows
259+ as same character as WC. */
260+ mbstate_t state; /* State of the stream. */
261+ int convfail; /* 1, when conversion is failed. Otherwise 0. */
262+
263+ idx = 0;
264+ buflen = 0;
265+ bufpos = buf;
266+ memset (&state, '\0', sizeof(mbstate_t));
267+
268+ while (1)
269+ {
270+ REFILL_BUFFER (buf, bufpos, buflen, stream);
271+
272+ GET_NEXT_WC_FROM_BUFFER (wc, bufpos, buflen, mblength, state, convfail);
273+
274+ if (wc == WEOF)
275+ {
276+ if (idx > 0)
277+ putchar ('\n');
278+ break;
279+ }
280+ else if (wc == L'\n')
281+ {
282+ putchar ('\n');
283+ idx = 0;
284+ }
285+ else
286+ {
287+ idx += (operating_mode == byte_mode) ? mblength : 1;
288+ if (print_kth (idx, NULL))
289+ fwrite (bufpos, mblength, sizeof(char), stdout);
290+ }
291+
292+ buflen -= mblength;
293+ bufpos += mblength;
294+ }
295+}
296+#endif
297+
298 /* Read from stream STREAM, printing to standard output any selected fields. */
299
300 static void
301@@ -704,13 +843,192 @@
302 }
303 }
304
305+#if HAVE_MBRTOWC
306+static void
307+cut_fields_mb (FILE *stream)
308+{
309+ int c;
310+ unsigned int field_idx;
311+ int found_any_selected_field;
312+ int buffer_first_field;
313+ int empty_input;
314+ char buf[MB_LEN_MAX + BUFSIZ]; /* For spooling a read byte sequence. */
315+ char *bufpos; /* Next read position of BUF. */
316+ size_t buflen; /* The length of the byte sequence in buf. */
317+ wint_t wc = 0; /* A gotten wide character. */
318+ size_t mblength; /* The byte size of a multibyte character which shows
319+ as same character as WC. */
320+ mbstate_t state; /* State of the stream. */
321+ int convfail; /* 1, when conversion is failed. Otherwise 0. */
322+
323+ found_any_selected_field = 0;
324+ field_idx = 1;
325+ bufpos = buf;
326+ buflen = 0;
327+ memset (&state, '\0', sizeof(mbstate_t));
328+
329+ c = getc (stream);
330+ empty_input = (c == EOF);
331+ if (c != EOF)
332+ ungetc (c, stream);
333+ else
334+ wc = WEOF;
335+
336+ /* To support the semantics of the -s flag, we may have to buffer
337+ all of the first field to determine whether it is `delimited.'
338+ But that is unnecessary if all non-delimited lines must be printed
339+ and the first field has been selected, or if non-delimited lines
340+ must be suppressed and the first field has *not* been selected.
341+ That is because a non-delimited line has exactly one field. */
342+ buffer_first_field = (suppress_non_delimited ^ !print_kth (1, NULL));
343+
344+ while (1)
345+ {
346+ if (field_idx == 1 && buffer_first_field)
347+ {
348+ int len = 0;
349+
350+ while (1)
351+ {
352+ REFILL_BUFFER (buf, bufpos, buflen, stream);
353+
354+ GET_NEXT_WC_FROM_BUFFER
355+ (wc, bufpos, buflen, mblength, state, convfail);
356+
357+ if (wc == WEOF)
358+ break;
359+
360+ field_1_buffer = xrealloc (field_1_buffer, len + mblength);
361+ memcpy (field_1_buffer + len, bufpos, mblength);
362+ len += mblength;
363+ buflen -= mblength;
364+ bufpos += mblength;
365+
366+ if (!convfail && (wc == L'\n' || wc == wcdelim))
367+ break;
368+ }
369+
370+ if (wc == WEOF)
371+ break;
372+
373+ /* If the first field extends to the end of line (it is not
374+ delimited) and we are printing all non-delimited lines,
375+ print this one. */
376+ if (convfail || (!convfail && wc != wcdelim))
377+ {
378+ if (suppress_non_delimited)
379+ {
380+ /* Empty. */
381+ }
382+ else
383+ {
384+ fwrite (field_1_buffer, sizeof (char), len, stdout);
385+ /* Make sure the output line is newline terminated. */
386+ if (convfail || (!convfail && wc != L'\n'))
387+ putchar ('\n');
388+ }
389+ continue;
390+ }
391+
392+ if (print_kth (1, NULL))
393+ {
394+ /* Print the field, but not the trailing delimiter. */
395+ fwrite (field_1_buffer, sizeof (char), len - 1, stdout);
396+ found_any_selected_field = 1;
397+ }
398+ ++field_idx;
399+ }
400+
401+ if (wc != WEOF)
402+ {
403+ if (print_kth (field_idx, NULL))
404+ {
405+ if (found_any_selected_field)
406+ {
407+ fwrite (output_delimiter_string, sizeof (char),
408+ output_delimiter_length, stdout);
409+ }
410+ found_any_selected_field = 1;
411+ }
412+
413+ while (1)
414+ {
415+ REFILL_BUFFER (buf, bufpos, buflen, stream);
416+
417+ GET_NEXT_WC_FROM_BUFFER
418+ (wc, bufpos, buflen, mblength, state, convfail);
419+
420+ if (wc == WEOF)
421+ break;
422+ else if (!convfail && (wc == wcdelim || wc == L'\n'))
423+ {
424+ buflen -= mblength;
425+ bufpos += mblength;
426+ break;
427+ }
428+
429+ if (print_kth (field_idx, NULL))
430+ fwrite (bufpos, mblength, sizeof(char), stdout);
431+
432+ buflen -= mblength;
433+ bufpos += mblength;
434+ }
435+ }
436+
437+ if ((!convfail || wc == L'\n') && buflen < 1)
438+ wc = WEOF;
439+
440+ if (!convfail && wc == wcdelim)
441+ ++field_idx;
442+ else if (wc == WEOF || (!convfail && wc == L'\n'))
443+ {
444+ if (found_any_selected_field
445+ || (!empty_input && !(suppress_non_delimited && field_idx == 1)))
446+ putchar ('\n');
447+ if (wc == WEOF)
448+ break;
449+ field_idx = 1;
450+ found_any_selected_field = 0;
451+ }
452+ }
453+}
454+#endif
455+
456 static void
457 cut_stream (FILE *stream)
458 {
459- if (operating_mode == byte_mode)
460- cut_bytes (stream);
461+#if HAVE_MBRTOWC
462+ if (MB_CUR_MAX > 1 && !force_singlebyte_mode)
463+ {
464+ switch (operating_mode)
465+ {
466+ case byte_mode:
467+ if (byte_mode_character_aware)
468+ cut_characters_or_cut_bytes_no_split (stream);
469+ else
470+ cut_bytes (stream);
471+ break;
472+
473+ case character_mode:
474+ cut_characters_or_cut_bytes_no_split (stream);
475+ break;
476+
477+ case field_mode:
478+ cut_fields_mb (stream);
479+ break;
480+
481+ default:
482+ abort ();
483+ }
484+ }
485 else
486- cut_fields (stream);
487+#endif
488+ {
489+ if (operating_mode == field_mode)
490+ cut_fields (stream);
491+ else
492+ cut_bytes (stream);
493+ }
494 }
495
496 /* Process file FILE to standard output.
497@@ -760,6 +1078,8 @@
498 bool ok;
499 bool delim_specified = false;
500 char *spec_list_string IF_LINT(= NULL);
501+ char mbdelim[MB_LEN_MAX + 1];
502+ size_t delimlen = 0;
503
504 initialize_main (&argc, &argv);
505 program_name = argv[0];
506@@ -782,7 +1102,6 @@
507 switch (optc)
508 {
509 case 'b':
510- case 'c':
511 /* Build the byte list. */
512 if (operating_mode != undefined_mode)
513 FATAL_ERROR (_("only one type of list may be specified"));
514@@ -790,6 +1109,14 @@
515 spec_list_string = optarg;
516 break;
517
518+ case 'c':
519+ /* Build the character list. */
520+ if (operating_mode != undefined_mode)
521+ FATAL_ERROR (_("only one type of list may be specified"));
522+ operating_mode = character_mode;
523+ spec_list_string = optarg;
524+ break;
525+
526 case 'f':
527 /* Build the field list. */
528 if (operating_mode != undefined_mode)
529@@ -801,10 +1128,35 @@
530 case 'd':
531 /* New delimiter. */
532 /* Interpret -d '' to mean `use the NUL byte as the delimiter.' */
533- if (optarg[0] != '\0' && optarg[1] != '\0')
534- FATAL_ERROR (_("the delimiter must be a single character"));
535- delim = optarg[0];
536- delim_specified = true;
537+ {
538+#if HAVE_MBRTOWC
539+ if(MB_CUR_MAX > 1)
540+ {
541+ mbstate_t state;
542+
543+ memset (&state, '\0', sizeof(mbstate_t));
544+ delimlen = mbrtowc (&wcdelim, optarg, strnlen(optarg, MB_LEN_MAX), &state);
545+
546+ if (delimlen == (size_t)-1 || delimlen == (size_t)-2)
547+ ++force_singlebyte_mode;
548+ else
549+ {
550+ delimlen = (delimlen < 1) ? 1 : delimlen;
551+ if (wcdelim != L'\0' && *(optarg + delimlen) != '\0')
552+ FATAL_ERROR (_("the delimiter must be a single character"));
553+ memcpy (mbdelim, optarg, delimlen);
554+ }
555+ }
556+
557+ if (MB_CUR_MAX <= 1 || force_singlebyte_mode)
558+#endif
559+ {
560+ if (optarg[0] != '\0' && optarg[1] != '\0')
561+ FATAL_ERROR (_("the delimiter must be a single character"));
562+ delim = (unsigned char) optarg[0];
563+ }
564+ delim_specified = true;
565+ }
566 break;
567
568 case OUTPUT_DELIMITER_OPTION:
569@@ -817,6 +1169,7 @@
570 break;
571
572 case 'n':
573+ byte_mode_character_aware = 1;
574 break;
575
576 case 's':
577@@ -839,7 +1192,7 @@
578 if (operating_mode == undefined_mode)
579 FATAL_ERROR (_("you must specify a list of bytes, characters, or fields"));
580
581- if (delim != '\0' && operating_mode != field_mode)
582+ if (delim_specified && operating_mode != field_mode)
583 FATAL_ERROR (_("an input delimiter may be specified only\
584 when operating on fields"));
585
586@@ -866,15 +1219,34 @@
587 }
588
589 if (!delim_specified)
590- delim = '\t';
591+ {
592+ delim = '\t';
593+#ifdef HAVE_MBRTOWC
594+ wcdelim = L'\t';
595+ mbdelim[0] = '\t';
596+ mbdelim[1] = '\0';
597+ delimlen = 1;
598+#endif
599+ }
600
601 if (output_delimiter_string == NULL)
602 {
603- static char dummy[2];
604- dummy[0] = delim;
605- dummy[1] = '\0';
606- output_delimiter_string = dummy;
607- output_delimiter_length = 1;
608+#ifdef HAVE_MBRTOWC
609+ if (MB_CUR_MAX > 1 && !force_singlebyte_mode)
610+ {
611+ output_delimiter_string = xstrdup(mbdelim);
612+ output_delimiter_length = delimlen;
613+ }
614+
615+ if (MB_CUR_MAX <= 1 || force_singlebyte_mode)
616+#endif
617+ {
618+ static char dummy[2];
619+ dummy[0] = delim;
620+ dummy[1] = '\0';
621+ output_delimiter_string = dummy;
622+ output_delimiter_length = 1;
623+ }
624 }
625
626 if (optind == argc)
627diff -Naur coreutils-6.12.orig/src/expand.c coreutils-6.12/src/expand.c
628--- coreutils-6.12.orig/src/expand.c 2008-05-25 23:40:33.000000000 -0700
[d33f80c]629+++ coreutils-6.12/src/expand.c 2009-01-08 20:14:37.000000000 -0800
[577dd2d]630@@ -37,11 +37,28 @@
631 #include <stdio.h>
632 #include <getopt.h>
633 #include <sys/types.h>
634+
635+/* Get mbstate_t, mbrtowc(), wcwidth(). */
636+#if HAVE_WCHAR_H
637+# include <wchar.h>
638+#endif
639+
640 #include "system.h"
641 #include "error.h"
642 #include "quote.h"
643 #include "xstrndup.h"
644
645+/* MB_LEN_MAX is incorrectly defined to be 1 in at least one GCC
646+ installation; work around this configuration error. */
647+#if !defined MB_LEN_MAX || MB_LEN_MAX < 2
648+# define MB_LEN_MAX 16
649+#endif
650+
651+/* Some systems, like BeOS, have multibyte encodings but lack mbstate_t. */
652+#if HAVE_MBRTOWC && defined mbstate_t
653+# define mbrtowc(pwc, s, n, ps) (mbrtowc) (pwc, s, n, 0)
654+#endif
655+
656 /* The official name of this program (e.g., no `g' prefix). */
657 #define PROGRAM_NAME "expand"
658
659@@ -182,6 +199,7 @@
660 stops = num_start + len - 1;
661 }
662 }
663+
664 else
665 {
666 error (0, 0, _("tab size contains invalid character(s): %s"),
667@@ -364,6 +382,142 @@
668 }
669 }
670
671+#if HAVE_MBRTOWC
672+static void
673+expand_multibyte (void)
674+{
675+ FILE *fp; /* Input strem. */
676+ mbstate_t i_state; /* Current shift state of the input stream. */
677+ mbstate_t i_state_bak; /* Back up the I_STATE. */
678+ mbstate_t o_state; /* Current shift state of the output stream. */
679+ char buf[MB_LEN_MAX + BUFSIZ]; /* For spooling a read byte sequence. */
680+ char *bufpos; /* Next read position of BUF. */
681+ size_t buflen = 0; /* The length of the byte sequence in buf. */
682+ wchar_t wc; /* A gotten wide character. */
683+ size_t mblength; /* The byte size of a multibyte character
684+ which shows as same character as WC. */
685+ int tab_index = 0; /* Index in `tab_list' of next tabstop. */
686+ int column = 0; /* Column on screen of the next char. */
687+ int next_tab_column; /* Column the next tab stop is on. */
688+ int convert = 1; /* If nonzero, perform translations. */
689+
690+ fp = next_file ((FILE *) NULL);
691+ if (fp == NULL)
692+ return;
693+
694+ memset (&o_state, '\0', sizeof(mbstate_t));
695+ memset (&i_state, '\0', sizeof(mbstate_t));
696+
697+ for (;;)
698+ {
699+ /* Refill the buffer BUF. */
700+ if (buflen < MB_LEN_MAX && !feof(fp) && !ferror(fp))
701+ {
702+ memmove (buf, bufpos, buflen);
703+ buflen += fread (buf + buflen, sizeof(char), BUFSIZ, fp);
704+ bufpos = buf;
705+ }
706+
707+ /* No character is left in BUF. */
708+ if (buflen < 1)
709+ {
710+ fp = next_file (fp);
711+
712+ if (fp == NULL)
713+ break; /* No more files. */
714+ else
715+ {
716+ memset (&i_state, '\0', sizeof(mbstate_t));
717+ continue;
718+ }
719+ }
720+
721+ /* Get a wide character. */
722+ i_state_bak = i_state;
723+ mblength = mbrtowc (&wc, bufpos, buflen, &i_state);
724+
725+ switch (mblength)
726+ {
727+ case (size_t)-1: /* illegal byte sequence. */
728+ case (size_t)-2:
729+ mblength = 1;
730+ i_state = i_state_bak;
731+ if (convert)
732+ {
733+ ++column;
734+ if (convert_entire_line == 0)
735+ convert = 0;
736+ }
737+ putchar (*bufpos);
738+ break;
739+
740+ case 0: /* null. */
741+ mblength = 1;
742+ if (convert && convert_entire_line == 0)
743+ convert = 0;
744+ putchar ('\0');
745+ break;
746+
747+ default:
748+ if (wc == L'\n') /* LF. */
749+ {
750+ tab_index = 0;
751+ column = 0;
752+ convert = 1;
753+ putchar ('\n');
754+ }
755+ else if (wc == L'\t' && convert) /* Tab. */
756+ {
757+ if (tab_size == 0)
758+ {
759+ /* Do not let tab_index == first_free_tab;
760+ stop when it is 1 less. */
761+ while (tab_index < first_free_tab - 1
762+ && column >= tab_list[tab_index])
763+ tab_index++;
764+ next_tab_column = tab_list[tab_index];
765+ if (tab_index < first_free_tab - 1)
766+ tab_index++;
767+ if (column >= next_tab_column)
768+ next_tab_column = column + 1;
769+ }
770+ else
771+ next_tab_column = column + tab_size - column % tab_size;
772+
773+ while (column < next_tab_column)
774+ {
775+ putchar (' ');
776+ ++column;
777+ }
778+ }
779+ else /* Others. */
780+ {
781+ if (convert)
782+ {
783+ if (wc == L'\b')
784+ {
785+ if (column > 0)
786+ --column;
787+ }
788+ else
789+ {
790+ int width; /* The width of WC. */
791+
792+ width = wcwidth (wc);
793+ column += (width > 0) ? width : 0;
794+ if (convert_entire_line == 0)
795+ convert = 0;
796+ }
797+ }
798+ fwrite (bufpos, sizeof(char), mblength, stdout);
799+ }
800+ }
801+ buflen -= mblength;
802+ bufpos += mblength;
803+ }
804+}
805+#endif
806+
807 int
808 main (int argc, char **argv)
809 {
810@@ -428,7 +582,12 @@
811
812 file_list = (optind < argc ? &argv[optind] : stdin_argv);
813
814- expand ();
815+#if HAVE_MBRTOWC
816+ if (MB_CUR_MAX > 1)
817+ expand_multibyte ();
818+ else
819+#endif
820+ expand ();
821
822 if (have_read_stdin && fclose (stdin) != 0)
823 error (EXIT_FAILURE, errno, "-");
824diff -Naur coreutils-6.12.orig/src/fold.c coreutils-6.12/src/fold.c
825--- coreutils-6.12.orig/src/fold.c 2008-05-25 23:40:33.000000000 -0700
[d33f80c]826+++ coreutils-6.12/src/fold.c 2009-01-08 20:14:37.000000000 -0800
[577dd2d]827@@ -22,11 +22,33 @@
828 #include <getopt.h>
829 #include <sys/types.h>
830
831+/* Get mbstate_t, mbrtowc(), wcwidth(). */
832+#if HAVE_WCHAR_H
833+# include <wchar.h>
834+#endif
835+
836+/* Get iswprint(), iswblank(), wcwidth(). */
837+#if HAVE_WCTYPE_H
838+# include <wctype.h>
839+#endif
840+
841 #include "system.h"
842 #include "error.h"
843 #include "quote.h"
844 #include "xstrtol.h"
845
846+/* MB_LEN_MAX is incorrectly defined to be 1 in at least one GCC
847+ installation; work around this configuration error. */
848+#if !defined MB_LEN_MAX || MB_LEN_MAX < 2
849+# undef MB_LEN_MAX
850+# define MB_LEN_MAX 16
851+#endif
852+
853+/* Some systems, like BeOS, have multibyte encodings but lack mbstate_t. */
854+#if HAVE_MBRTOWC && defined mbstate_t
855+# define mbrtowc(pwc, s, n, ps) (mbrtowc) (pwc, s, n, 0)
856+#endif
857+
858 #define TAB_WIDTH 8
859
860 /* The official name of this program (e.g., no `g' prefix). */
861@@ -34,23 +56,44 @@
862
863 #define AUTHORS proper_name ("David MacKenzie")
864
865+#define FATAL_ERROR(Message) \
866+ do \
867+ { \
868+ error (0, 0, (Message)); \
869+ usage (2); \
870+ } \
871+ while (0)
872+
873+enum operating_mode
874+{
875+ /* Fold texts by columns that are at the given positions. */
876+ column_mode,
877+
878+ /* Fold texts by bytes that are at the given positions. */
879+ byte_mode,
880+
881+ /* Fold texts by characters that are at the given positions. */
882+ character_mode,
883+};
884+
885 /* The name this program was run with. */
886 char *program_name;
887
888+/* The argument shows current mode. (Default: column_mode) */
889+static enum operating_mode operating_mode;
890+
891 /* If nonzero, try to break on whitespace. */
892 static bool break_spaces;
893
894-/* If nonzero, count bytes, not column positions. */
895-static bool count_bytes;
896-
897 /* If nonzero, at least one of the files we read was standard input. */
898 static bool have_read_stdin;
899
900-static char const shortopts[] = "bsw:0::1::2::3::4::5::6::7::8::9::";
901+static char const shortopts[] = "bcsw:0::1::2::3::4::5::6::7::8::9::";
902
903 static struct option const longopts[] =
904 {
905 {"bytes", no_argument, NULL, 'b'},
906+ {"characters", no_argument, NULL, 'c'},
907 {"spaces", no_argument, NULL, 's'},
908 {"width", required_argument, NULL, 'w'},
909 {GETOPT_HELP_OPTION_DECL},
910@@ -80,6 +123,7 @@
911 "), stdout);
912 fputs (_("\
913 -b, --bytes count bytes rather than columns\n\
914+ -c, --characters count characters rather than columns\n\
915 -s, --spaces break at spaces\n\
916 -w, --width=WIDTH use WIDTH columns instead of 80\n\
917 "), stdout);
918@@ -97,7 +141,7 @@
919 static size_t
920 adjust_column (size_t column, char c)
921 {
922- if (!count_bytes)
923+ if (operating_mode != byte_mode)
924 {
925 if (c == '\b')
926 {
927@@ -120,30 +164,14 @@
928 to stdout, with maximum line length WIDTH.
929 Return true if successful. */
930
931-static bool
932-fold_file (char const *filename, size_t width)
933+static void
934+fold_text (FILE *istream, size_t width, int *saved_errno)
935 {
936- FILE *istream;
937 int c;
938 size_t column = 0; /* Screen column where next char will go. */
939 size_t offset_out = 0; /* Index in `line_out' for next char. */
940 static char *line_out = NULL;
941 static size_t allocated_out = 0;
942- int saved_errno;
943-
944- if (STREQ (filename, "-"))
945- {
946- istream = stdin;
947- have_read_stdin = true;
948- }
949- else
950- istream = fopen (filename, "r");
951-
952- if (istream == NULL)
953- {
954- error (0, errno, "%s", filename);
955- return false;
956- }
957
958 while ((c = getc (istream)) != EOF)
959 {
960@@ -171,6 +199,15 @@
961 bool found_blank = false;
962 size_t logical_end = offset_out;
963
964+ /* If LINE_OUT has no wide character,
965+ put a new wide character in LINE_OUT
966+ if column is bigger than width. */
967+ if (offset_out == 0)
968+ {
969+ line_out[offset_out++] = c;
970+ continue;
971+ }
972+
973 /* Look for the last blank. */
974 while (logical_end)
975 {
976@@ -217,11 +254,225 @@
977 line_out[offset_out++] = c;
978 }
979
980- saved_errno = errno;
981+ *saved_errno = errno;
982+
983+ if (offset_out)
984+ fwrite (line_out, sizeof (char), (size_t) offset_out, stdout);
985+
986+ free(line_out);
987+}
988+
989+#if HAVE_MBRTOWC
990+static void
991+fold_multibyte_text (FILE *istream, size_t width, int *saved_errno)
992+{
993+ char buf[MB_LEN_MAX + BUFSIZ]; /* For spooling a read byte sequence. */
994+ size_t buflen = 0; /* The length of the byte sequence in buf. */
995+ char *bufpos; /* Next read position of BUF. */
996+ wint_t wc; /* A gotten wide character. */
997+ size_t mblength; /* The byte size of a multibyte character which shows
998+ as same character as WC. */
999+ mbstate_t state, state_bak; /* State of the stream. */
1000+ int convfail; /* 1, when conversion is failed. Otherwise 0. */
1001+
1002+ char *line_out = NULL;
1003+ size_t offset_out = 0; /* Index in `line_out' for next char. */
1004+ size_t allocated_out = 0;
1005+
1006+ int increment;
1007+ size_t column = 0;
1008+
1009+ size_t last_blank_pos;
1010+ size_t last_blank_column;
1011+ int is_blank_seen;
1012+ int last_blank_increment;
1013+ int is_bs_following_last_blank;
1014+ size_t bs_following_last_blank_num;
1015+ int is_cr_after_last_blank;
1016+
1017+#define CLEAR_FLAGS \
1018+ do \
1019+ { \
1020+ last_blank_pos = 0; \
1021+ last_blank_column = 0; \
1022+ is_blank_seen = 0; \
1023+ is_bs_following_last_blank = 0; \
1024+ bs_following_last_blank_num = 0; \
1025+ is_cr_after_last_blank = 0; \
1026+ } \
1027+ while (0)
1028+
1029+#define START_NEW_LINE \
1030+ do \
1031+ { \
1032+ putchar ('\n'); \
1033+ column = 0; \
1034+ offset_out = 0; \
1035+ CLEAR_FLAGS; \
1036+ } \
1037+ while (0)
1038+
1039+ CLEAR_FLAGS;
1040+ memset (&state, '\0', sizeof(mbstate_t));
1041+
1042+ for (;; bufpos += mblength, buflen -= mblength)
1043+ {
1044+ if (buflen < MB_LEN_MAX && !feof (istream) && !ferror (istream))
1045+ {
1046+ memmove (buf, bufpos, buflen);
1047+ buflen += fread (buf + buflen, sizeof(char), BUFSIZ, istream);
1048+ bufpos = buf;
1049+ }
1050+
1051+ if (buflen < 1)
1052+ break;
1053+
1054+ /* Get a wide character. */
1055+ convfail = 0;
1056+ state_bak = state;
1057+ mblength = mbrtowc ((wchar_t *)&wc, bufpos, buflen, &state);
1058+
1059+ switch (mblength)
1060+ {
1061+ case (size_t)-1:
1062+ case (size_t)-2:
1063+ convfail++;
1064+ state = state_bak;
1065+ /* Fall through. */
1066+
1067+ case 0:
1068+ mblength = 1;
1069+ break;
1070+ }
1071+
1072+rescan:
1073+ if (operating_mode == byte_mode) /* byte mode */
1074+ increment = mblength;
1075+ else if (operating_mode == character_mode) /* character mode */
1076+ increment = 1;
1077+ else /* column mode */
1078+ {
1079+ if (convfail)
1080+ increment = 1;
1081+ else
1082+ {
1083+ switch (wc)
1084+ {
1085+ case L'\n':
1086+ fwrite (line_out, sizeof(char), offset_out, stdout);
1087+ START_NEW_LINE;
1088+ continue;
1089+
1090+ case L'\b':
1091+ increment = (column > 0) ? -1 : 0;
1092+ break;
1093+
1094+ case L'\r':
1095+ increment = -1 * column;
1096+ break;
1097+
1098+ case L'\t':
1099+ increment = 8 - column % 8;
1100+ break;
1101+
1102+ default:
1103+ increment = wcwidth (wc);
1104+ increment = (increment < 0) ? 0 : increment;
1105+ }
1106+ }
1107+ }
1108+
1109+ if (column + increment > width && break_spaces && last_blank_pos)
1110+ {
1111+ fwrite (line_out, sizeof(char), last_blank_pos, stdout);
1112+ putchar ('\n');
1113+
1114+ offset_out = offset_out - last_blank_pos;
1115+ column = column - last_blank_column + ((is_cr_after_last_blank)
1116+ ? last_blank_increment : bs_following_last_blank_num);
1117+ memmove (line_out, line_out + last_blank_pos, offset_out);
1118+ CLEAR_FLAGS;
1119+ goto rescan;
1120+ }
1121+
1122+ if (column + increment > width && column != 0)
1123+ {
1124+ fwrite (line_out, sizeof(char), offset_out, stdout);
1125+ START_NEW_LINE;
1126+ goto rescan;
1127+ }
1128+
1129+ if (allocated_out < offset_out + mblength)
1130+ {
1131+ allocated_out += 1024;
1132+ line_out = xrealloc (line_out, allocated_out);
1133+ }
1134+
1135+ memcpy (line_out + offset_out, bufpos, mblength);
1136+ offset_out += mblength;
1137+ column += increment;
1138+
1139+ if (is_blank_seen && !convfail && wc == L'\r')
1140+ is_cr_after_last_blank = 1;
1141+
1142+ if (is_bs_following_last_blank && !convfail && wc == L'\b')
1143+ ++bs_following_last_blank_num;
1144+ else
1145+ is_bs_following_last_blank = 0;
1146+
1147+ if (break_spaces && !convfail && iswblank (wc))
1148+ {
1149+ last_blank_pos = offset_out;
1150+ last_blank_column = column;
1151+ is_blank_seen = 1;
1152+ last_blank_increment = increment;
1153+ is_bs_following_last_blank = 1;
1154+ bs_following_last_blank_num = 0;
1155+ is_cr_after_last_blank = 0;
1156+ }
1157+ }
1158+
1159+ *saved_errno = errno;
1160
1161 if (offset_out)
1162 fwrite (line_out, sizeof (char), (size_t) offset_out, stdout);
1163
1164+ free(line_out);
1165+}
1166+#endif
1167+
1168+/* Fold file FILENAME, or standard input if FILENAME is "-",
1169+ to stdout, with maximum line length WIDTH.
1170+ Return 0 if successful, 1 if an error occurs. */
1171+
1172+static bool
1173+fold_file (char *filename, size_t width)
1174+{
1175+ FILE *istream;
1176+ int saved_errno;
1177+
1178+ if (STREQ (filename, "-"))
1179+ {
1180+ istream = stdin;
1181+ have_read_stdin = 1;
1182+ }
1183+ else
1184+ istream = fopen (filename, "r");
1185+
1186+ if (istream == NULL)
1187+ {
1188+ error (0, errno, "%s", filename);
1189+ return 1;
1190+ }
1191+
1192+ /* Define how ISTREAM is being folded. */
1193+#if HAVE_MBRTOWC
1194+ if (MB_CUR_MAX > 1)
1195+ fold_multibyte_text (istream, width, &saved_errno);
1196+ else
1197+#endif
1198+ fold_text (istream, width, &saved_errno);
1199+
1200 if (ferror (istream))
1201 {
1202 error (0, saved_errno, "%s", filename);
1203@@ -254,7 +505,8 @@
1204
1205 atexit (close_stdout);
1206
1207- break_spaces = count_bytes = have_read_stdin = false;
1208+ operating_mode = column_mode;
1209+ break_spaces = have_read_stdin = false;
1210
1211 while ((optc = getopt_long (argc, argv, shortopts, longopts, NULL)) != -1)
1212 {
1213@@ -263,7 +515,15 @@
1214 switch (optc)
1215 {
1216 case 'b': /* Count bytes rather than columns. */
1217- count_bytes = true;
1218+ if (operating_mode != column_mode)
1219+ FATAL_ERROR (_("only one way of folding may be specified"));
1220+ operating_mode = byte_mode;
1221+ break;
1222+
1223+ case 'c':
1224+ if (operating_mode != column_mode)
1225+ FATAL_ERROR (_("only one way of folding may be specified"));
1226+ operating_mode = character_mode;
1227 break;
1228
1229 case 's': /* Break at word boundaries. */
1230diff -Naur coreutils-6.12.orig/src/join.c coreutils-6.12/src/join.c
1231--- coreutils-6.12.orig/src/join.c 2008-05-25 23:40:32.000000000 -0700
[d33f80c]1232+++ coreutils-6.12/src/join.c 2009-01-08 20:14:37.000000000 -0800
[577dd2d]1233@@ -22,17 +22,31 @@
1234 #include <sys/types.h>
1235 #include <getopt.h>
1236
1237+/* Get mbstate_t, mbrtowc(), mbrtowc(), wcwidth(). */
1238+#if HAVE_WCHAR_H
1239+# include <wchar.h>
1240+#endif
1241+
1242+/* Get iswblank(), towupper. */
1243+#if HAVE_WCTYPE_H
1244+# include <wctype.h>
1245+#endif
1246+
1247 #include "system.h"
1248 #include "error.h"
1249 #include "hard-locale.h"
1250 #include "linebuffer.h"
1251-#include "memcasecmp.h"
1252 #include "quote.h"
1253 #include "stdio--.h"
1254 #include "xmemcoll.h"
1255 #include "xstrtol.h"
1256 #include "argmatch.h"
1257
1258+/* Some systems, like BeOS, have multibyte encodings but lack mbstate_t. */
1259+#if HAVE_MBRTOWC && defined mbstate_t
1260+# define mbrtowc(pwc, s, n, ps) (mbrtowc) (pwc, s, n, 0)
1261+#endif
1262+
1263 /* The official name of this program (e.g., no `g' prefix). */
1264 #define PROGRAM_NAME "join"
1265
1266@@ -113,10 +127,12 @@
1267 /* Last element in `outlist', where a new element can be added. */
1268 static struct outlist *outlist_end = &outlist_head;
1269
1270-/* Tab character separating fields. If negative, fields are separated
1271- by any nonempty string of blanks, otherwise by exactly one
1272- tab character whose value (when cast to unsigned char) equals TAB. */
1273-static int tab = -1;
1274+/* Tab character separating fields. If NULL, fields are separated
1275+ by any nonempty string of blanks. */
1276+static char *tab = NULL;
1277+
1278+/* The number of bytes used for tab. */
1279+static size_t tablen = 0;
1280
1281 /* If nonzero, check that the input is correctly ordered. */
1282 static enum
[d33f80c]1283@@ -221,6 +237,8 @@
1284
1285 /* Fill in the `fields' structure in LINE. */
1286
1287+/* Fill in the `fields' structure in LINE. */
1288+
1289 static void
1290 xfields (struct line *line)
1291 {
1292@@ -230,10 +248,11 @@
[577dd2d]1293 if (ptr == lim)
1294 return;
1295
1296- if (0 <= tab)
1297+ if (tab != NULL)
1298 {
1299+ unsigned char t = tab[0];
1300 char *sep;
1301- for (; (sep = memchr (ptr, tab, lim - ptr)) != NULL; ptr = sep + 1)
1302+ for (; (sep = memchr (ptr, t, lim - ptr)) != NULL; ptr = sep + 1)
1303 extract_field (line, ptr, sep - ptr);
1304 }
1305 else
[d33f80c]1306@@ -305,56 +324,115 @@
1307 size_t jf_1, size_t jf_2)
1308 {
1309 /* Start of field to compare in each file. */
1310- char *beg1;
1311- char *beg2;
1312-
1313- size_t len1;
1314- size_t len2; /* Length of fields to compare. */
1315+ char *beg[2];
1316+ char *copy[2];
1317+ size_t len[2]; /* Length of fields to compare. */
1318 int diff;
1319+ int i, j;
1320
1321 if (jf_1 < line1->nfields)
1322 {
1323- beg1 = line1->fields[jf_1].beg;
1324- len1 = line1->fields[jf_1].len;
1325+ beg[0] = line1->fields[jf_1].beg;
1326+ len[0] = line1->fields[jf_1].len;
1327 }
1328 else
1329 {
1330- beg1 = NULL;
1331- len1 = 0;
1332+ beg[0] = NULL;
1333+ len[0] = 0;
1334 }
1335
1336 if (jf_2 < line2->nfields)
1337 {
1338- beg2 = line2->fields[jf_2].beg;
1339- len2 = line2->fields[jf_2].len;
1340+ beg[1] = line2->fields[jf_2].beg;
1341+ len[1] = line2->fields[jf_2].len;
1342 }
1343 else
1344 {
1345- beg2 = NULL;
1346- len2 = 0;
1347+ beg[1] = NULL;
1348+ len[1] = 0;
1349 }
1350
1351- if (len1 == 0)
1352- return len2 == 0 ? 0 : -1;
1353- if (len2 == 0)
1354+ if (len[0] == 0)
1355+ return len[1] == 0 ? 0 : -1;
1356+ if (len[1] == 0)
1357 return 1;
1358
1359 if (ignore_case)
1360 {
1361- /* FIXME: ignore_case does not work with NLS (in particular,
1362- with multibyte chars). */
1363- diff = memcasecmp (beg1, beg2, MIN (len1, len2));
1364+#ifdef HAVE_MBRTOWC
1365+ if (MB_CUR_MAX > 1)
1366+ {
1367+ size_t mblength;
1368+ wchar_t wc, uwc;
1369+ mbstate_t state, state_bak;
1370+
1371+ memset (&state, '\0', sizeof (mbstate_t));
1372+
1373+ for (i = 0; i < 2; i++)
1374+ {
1375+ copy[i] = alloca (len[i] + 1);
1376+
1377+ for (j = 0; j < MIN (len[0], len[1]);)
1378+ {
1379+ state_bak = state;
1380+ mblength = mbrtowc (&wc, beg[i] + j, len[i] - j, &state);
1381+
1382+ switch (mblength)
1383+ {
1384+ case (size_t) -1:
1385+ case (size_t) -2:
1386+ state = state_bak;
1387+ /* Fall through */
1388+ case 0:
1389+ mblength = 1;
1390+ break;
1391+
1392+ default:
1393+ uwc = towupper (wc);
1394+
1395+ if (uwc != wc)
1396+ {
1397+ mbstate_t state_wc;
1398+
1399+ memset (&state_wc, '\0', sizeof (mbstate_t));
1400+ wcrtomb (copy[i] + j, uwc, &state_wc);
1401+ }
1402+ else
1403+ memcpy (copy[i] + j, beg[i] + j, mblength);
1404+ }
1405+ j += mblength;
1406+ }
1407+ copy[i][j] = '\0';
1408+ }
1409+ }
1410+ else
1411+#endif
1412+ {
1413+ for (i = 0; i < 2; i++)
1414+ {
1415+ copy[i] = alloca (len[i] + 1);
1416+
1417+ for (j = 0; j < MIN (len[0], len[1]); j++)
1418+ copy[i][j] = toupper (beg[i][j]);
1419+
1420+ copy[i][j] = '\0';
1421+ }
1422+ }
1423 }
1424 else
1425 {
1426- if (hard_LC_COLLATE)
1427- return xmemcoll (beg1, len1, beg2, len2);
1428- diff = memcmp (beg1, beg2, MIN (len1, len2));
1429+ copy[0] = (unsigned char *) beg[0];
1430+ copy[1] = (unsigned char *) beg[1];
1431 }
1432
1433+ if (hard_LC_COLLATE)
1434+ return xmemcoll ((char *) copy[0], len[0], (char *) copy[1], len[1]);
1435+ diff = memcmp (copy[0], copy[1], MIN (len[0], len[1]));
1436+
1437+
1438 if (diff)
1439 return diff;
1440- return len1 < len2 ? -1 : len1 != len2;
1441+ return len[0] - len[1];
1442 }
1443
1444 /* Check that successive input lines PREV and CURRENT from input file
1445@@ -393,6 +471,148 @@
1446 }
[577dd2d]1447 }
1448
1449+#if HAVE_MBRTOWC
1450+static void
1451+xfields_multibyte (struct line *line)
1452+{
1453+ char *ptr = line->buf.buffer;
1454+ char const *lim = ptr + line->buf.length - 1;
1455+ wchar_t wc = 0;
1456+ size_t mblength = 1;
1457+ mbstate_t state, state_bak;
1458+
1459+ memset (&state, 0, sizeof (mbstate_t));
1460+
1461+ if (ptr == lim)
1462+ return;
1463+
1464+ if (tab != NULL)
1465+ {
1466+ unsigned char t = tab[0];
1467+ char *sep = ptr;
1468+ for (; ptr < lim; ptr = sep + mblength)
1469+ {
1470+ sep = ptr;
1471+ while (sep < lim)
1472+ {
1473+ state_bak = state;
1474+ mblength = mbrtowc (&wc, sep, lim - sep + 1, &state);
1475+
1476+ if (mblength == (size_t)-1 || mblength == (size_t)-2)
1477+ {
1478+ mblength = 1;
1479+ state = state_bak;
1480+ }
1481+ mblength = (mblength < 1) ? 1 : mblength;
1482+
1483+ if (mblength == tablen && !memcmp (sep, tab, mblength))
1484+ break;
1485+ else
1486+ {
1487+ sep += mblength;
1488+ continue;
1489+ }
1490+ }
1491+
1492+ if (sep == lim)
1493+ break;
1494+
1495+ extract_field (line, ptr, sep - ptr);
1496+ }
1497+ }
1498+ else
1499+ {
1500+ /* Skip leading blanks before the first field. */
1501+ while(ptr < lim)
1502+ {
1503+ state_bak = state;
1504+ mblength = mbrtowc (&wc, ptr, lim - ptr + 1, &state);
1505+
1506+ if (mblength == (size_t)-1 || mblength == (size_t)-2)
1507+ {
1508+ mblength = 1;
1509+ state = state_bak;
1510+ break;
1511+ }
1512+ mblength = (mblength < 1) ? 1 : mblength;
1513+
1514+ if (!iswblank(wc))
1515+ break;
1516+ ptr += mblength;
1517+ }
1518+
1519+ do
1520+ {
1521+ char *sep;
1522+ state_bak = state;
1523+ mblength = mbrtowc (&wc, ptr, lim - ptr + 1, &state);
1524+ if (mblength == (size_t)-1 || mblength == (size_t)-2)
1525+ {
1526+ mblength = 1;
1527+ state = state_bak;
1528+ break;
1529+ }
1530+ mblength = (mblength < 1) ? 1 : mblength;
1531+
1532+ sep = ptr + mblength;
1533+ while (sep != lim)
1534+ {
1535+ state_bak = state;
1536+ mblength = mbrtowc (&wc, sep, lim - sep + 1, &state);
1537+ if (mblength == (size_t)-1 || mblength == (size_t)-2)
1538+ {
1539+ mblength = 1;
1540+ state = state_bak;
1541+ break;
1542+ }
1543+ mblength = (mblength < 1) ? 1 : mblength;
1544+
1545+ if (iswblank (wc))
1546+ break;
1547+
1548+ sep += mblength;
1549+ }
1550+
1551+ extract_field (line, ptr, sep - ptr);
1552+ if (sep == lim)
1553+ return;
1554+
1555+ state_bak = state;
1556+ mblength = mbrtowc (&wc, sep, lim - sep + 1, &state);
1557+ if (mblength == (size_t)-1 || mblength == (size_t)-2)
1558+ {
1559+ mblength = 1;
1560+ state = state_bak;
1561+ break;
1562+ }
1563+ mblength = (mblength < 1) ? 1 : mblength;
1564+
1565+ ptr = sep + mblength;
1566+ while (ptr != lim)
1567+ {
1568+ state_bak = state;
1569+ mblength = mbrtowc (&wc, ptr, lim - ptr + 1, &state);
1570+ if (mblength == (size_t)-1 || mblength == (size_t)-2)
1571+ {
1572+ mblength = 1;
1573+ state = state_bak;
1574+ break;
1575+ }
1576+ mblength = (mblength < 1) ? 1 : mblength;
1577+
1578+ if (!iswblank (wc))
1579+ break;
1580+
1581+ ptr += mblength;
1582+ }
1583+ }
1584+ while (ptr != lim);
1585+ }
1586+
1587+ extract_field (line, ptr, lim - ptr);
1588+}
1589+#endif
1590+
[d33f80c]1591 /* Read a line from FP into LINE and split it into fields.
1592 Return true if successful. */
[577dd2d]1593
[d33f80c]1594@@ -413,6 +633,11 @@
[577dd2d]1595 line->nfields_allocated = 0;
1596 line->nfields = 0;
1597 line->fields = NULL;
1598+#if HAVE_MBRTOWC
1599+ if (MB_CUR_MAX > 1)
1600+ xfields_multibyte (line);
1601+ else
1602+#endif
1603 xfields (line);
1604
1605 if (prevline[which - 1])
[d33f80c]1606@@ -509,11 +734,18 @@
[577dd2d]1607
1608 /* Print the join of LINE1 and LINE2. */
1609
1610+#define PUT_TAB_CHAR \
1611+ do \
1612+ { \
1613+ (tab != NULL) ? \
1614+ fwrite(tab, sizeof(char), tablen, stdout) : putchar (' '); \
1615+ } \
1616+ while (0)
1617+
1618 static void
1619 prjoin (struct line const *line1, struct line const *line2)
1620 {
1621 const struct outlist *outlist;
1622- char output_separator = tab < 0 ? ' ' : tab;
1623
1624 outlist = outlist_head.next;
1625 if (outlist)
[d33f80c]1626@@ -529,12 +761,12 @@
[577dd2d]1627 if (o->file == 0)
1628 {
1629 if (line1 == &uni_blank)
1630- {
1631+ {
1632 line = line2;
1633 field = join_field_2;
1634 }
1635 else
1636- {
1637+ {
1638 line = line1;
1639 field = join_field_1;
1640 }
[d33f80c]1641@@ -548,7 +780,7 @@
[577dd2d]1642 o = o->next;
1643 if (o == NULL)
1644 break;
1645- putchar (output_separator);
1646+ PUT_TAB_CHAR;
1647 }
1648 putchar ('\n');
1649 }
[d33f80c]1650@@ -566,23 +798,23 @@
[577dd2d]1651 prfield (join_field_1, line1);
1652 for (i = 0; i < join_field_1 && i < line1->nfields; ++i)
1653 {
1654- putchar (output_separator);
1655+ PUT_TAB_CHAR;
1656 prfield (i, line1);
1657 }
1658 for (i = join_field_1 + 1; i < line1->nfields; ++i)
1659 {
1660- putchar (output_separator);
1661+ PUT_TAB_CHAR;
1662 prfield (i, line1);
1663 }
1664
1665 for (i = 0; i < join_field_2 && i < line2->nfields; ++i)
1666 {
1667- putchar (output_separator);
1668+ PUT_TAB_CHAR;
1669 prfield (i, line2);
1670 }
1671 for (i = join_field_2 + 1; i < line2->nfields; ++i)
1672 {
1673- putchar (output_separator);
1674+ PUT_TAB_CHAR;
1675 prfield (i, line2);
1676 }
1677 putchar ('\n');
[d33f80c]1678@@ -1016,20 +1248,41 @@
[577dd2d]1679
1680 case 't':
1681 {
1682- unsigned char newtab = optarg[0];
1683- if (! newtab)
1684+ char *newtab;
1685+ size_t newtablen;
1686+ if (! optarg[0])
1687 error (EXIT_FAILURE, 0, _("empty tab"));
1688- if (optarg[1])
1689+ newtab = xstrdup (optarg);
1690+#if HAVE_MBRTOWC
1691+ if (MB_CUR_MAX > 1)
1692+ {
1693+ mbstate_t state;
1694+
1695+ memset (&state, 0, sizeof (mbstate_t));
1696+ newtablen = mbrtowc (NULL, newtab,
1697+ strnlen (newtab, MB_LEN_MAX),
1698+ &state);
1699+ if (newtablen == (size_t) 0
1700+ || newtablen == (size_t) -1
1701+ || newtablen == (size_t) -2)
1702+ newtablen = 1;
1703+ }
1704+ else
1705+#endif
1706+ newtablen = 1;
1707+
1708+ if (newtablen == 1 && newtab[1])
1709+ {
1710+ if (STREQ (newtab, "\\0"))
1711+ newtab[0] = '\0';
1712+ }
1713+ if (tab != NULL && strcmp (tab, newtab))
1714 {
1715- if (STREQ (optarg, "\\0"))
1716- newtab = '\0';
1717- else
1718- error (EXIT_FAILURE, 0, _("multi-character tab %s"),
1719- quote (optarg));
1720+ free (newtab);
1721+ error (EXIT_FAILURE, 0, _("incompatible tabs"));
1722 }
1723- if (0 <= tab && tab != newtab)
1724- error (EXIT_FAILURE, 0, _("incompatible tabs"));
1725 tab = newtab;
1726+ tablen = newtablen;
1727 }
1728 break;
1729
1730diff -Naur coreutils-6.12.orig/src/pr.c coreutils-6.12/src/pr.c
1731--- coreutils-6.12.orig/src/pr.c 2008-05-25 23:40:32.000000000 -0700
[d33f80c]1732+++ coreutils-6.12/src/pr.c 2009-01-08 20:14:37.000000000 -0800
[577dd2d]1733@@ -312,6 +312,32 @@
1734
1735 #include <getopt.h>
1736 #include <sys/types.h>
1737+
1738+/* Get MB_LEN_MAX. */
1739+#include <limits.h>
1740+/* MB_LEN_MAX is incorrectly defined to be 1 in at least one GCC
1741+ installation; work around this configuration error. */
1742+#if !defined MB_LEN_MAX || MB_LEN_MAX == 1
1743+# define MB_LEN_MAX 16
1744+#endif
1745+
1746+/* Get MB_CUR_MAX. */
1747+#include <stdlib.h>
1748+
1749+/* Solaris 2.5 has a bug: <wchar.h> must be included before <wctype.h>. */
1750+/* Get mbstate_t, mbrtowc(), wcwidth(). */
1751+#if HAVE_WCHAR_H
1752+# include <wchar.h>
1753+#endif
1754+
1755+/* Get iswprint(). -- for wcwidth(). */
1756+#if HAVE_WCTYPE_H
1757+# include <wctype.h>
1758+#endif
1759+#if !defined iswprint && !HAVE_ISWPRINT
1760+# define iswprint(wc) 1
1761+#endif
1762+
1763 #include "system.h"
1764 #include "error.h"
1765 #include "hard-locale.h"
1766@@ -322,6 +348,18 @@
1767 #include "strftime.h"
1768 #include "xstrtol.h"
1769
1770+/* Some systems, like BeOS, have multibyte encodings but lack mbstate_t. */
1771+#if HAVE_MBRTOWC && defined mbstate_t
1772+# define mbrtowc(pwc, s, n, ps) (mbrtowc) (pwc, s, n, 0)
1773+#endif
1774+
1775+#ifndef HAVE_DECL_WCWIDTH
1776+"this configure-time declaration test was not run"
1777+#endif
1778+#if !HAVE_DECL_WCWIDTH
1779+extern int wcwidth ();
1780+#endif
1781+
1782 /* The official name of this program (e.g., no `g' prefix). */
1783 #define PROGRAM_NAME "pr"
1784
1785@@ -416,7 +454,20 @@
1786
1787 #define NULLCOL (COLUMN *)0
1788
1789-static int char_to_clump (char c);
1790+/* Funtion pointers to switch functions for single byte locale or for
1791+ multibyte locale. If multibyte functions do not exist in your sysytem,
1792+ these pointers always point the function for single byte locale. */
1793+static void (*print_char) (char c);
1794+static int (*char_to_clump) (char c);
1795+
1796+/* Functions for single byte locale. */
1797+static void print_char_single (char c);
1798+static int char_to_clump_single (char c);
1799+
1800+/* Functions for multibyte locale. */
1801+static void print_char_multi (char c);
1802+static int char_to_clump_multi (char c);
1803+
1804 static bool read_line (COLUMN *p);
1805 static bool print_page (void);
1806 static bool print_stored (COLUMN *p);
1807@@ -426,6 +477,7 @@
1808 static void pad_across_to (int position);
1809 static void add_line_number (COLUMN *p);
1810 static void getoptarg (char *arg, char switch_char, char *character,
1811+ int *character_length, int *character_width,
1812 int *number);
1813 void usage (int status);
1814 static void print_files (int number_of_files, char **av);
1815@@ -440,7 +492,6 @@
1816 static void pad_down (int lines);
1817 static void read_rest_of_line (COLUMN *p);
1818 static void skip_read (COLUMN *p, int column_number);
1819-static void print_char (char c);
1820 static void cleanup (void);
1821 static void print_sep_string (void);
1822 static void separator_string (const char *optarg_S);
1823@@ -455,7 +506,7 @@
1824 we store the leftmost columns contiguously in buff.
1825 To print a line from buff, get the index of the first character
1826 from line_vector[i], and print up to line_vector[i + 1]. */
1827-static char *buff;
1828+static unsigned char *buff;
1829
1830 /* Index of the position in buff where the next character
1831 will be stored. */
1832@@ -559,7 +610,7 @@
1833 static bool untabify_input = false;
1834
1835 /* (-e) The input tab character. */
1836-static char input_tab_char = '\t';
1837+static char input_tab_char[MB_LEN_MAX] = "\t";
1838
1839 /* (-e) Tabstops are at chars_per_tab, 2*chars_per_tab, 3*chars_per_tab, ...
1840 where the leftmost column is 1. */
1841@@ -569,7 +620,10 @@
1842 static bool tabify_output = false;
1843
1844 /* (-i) The output tab character. */
1845-static char output_tab_char = '\t';
1846+static char output_tab_char[MB_LEN_MAX] = "\t";
1847+
1848+/* (-i) The byte length of output tab character. */
1849+static int output_tab_char_length = 1;
1850
1851 /* (-i) The width of the output tab. */
1852 static int chars_per_output_tab = 8;
1853@@ -643,7 +697,13 @@
1854 static bool numbered_lines = false;
1855
1856 /* (-n) Character which follows each line number. */
1857-static char number_separator = '\t';
1858+static char number_separator[MB_LEN_MAX] = "\t";
1859+
1860+/* (-n) The byte length of the character which follows each line number. */
1861+static int number_separator_length = 1;
1862+
1863+/* (-n) The character width of the character which follows each line number. */
1864+static int number_separator_width = 0;
1865
1866 /* (-n) line counting starts with 1st line of input file (not with 1st
1867 line of 1st page printed). */
1868@@ -696,6 +756,7 @@
1869 -a|COLUMN|-m is a `space' and with the -J option a `tab'. */
1870 static char *col_sep_string = "";
1871 static int col_sep_length = 0;
1872+static int col_sep_width = 0;
1873 static char *column_separator = " ";
1874 static char *line_separator = "\t";
1875
1876@@ -852,6 +913,13 @@
1877 col_sep_length = (int) strlen (optarg_S);
1878 col_sep_string = xmalloc (col_sep_length + 1);
1879 strcpy (col_sep_string, optarg_S);
1880+
1881+#if HAVE_MBRTOWC
1882+ if (MB_CUR_MAX > 1)
1883+ col_sep_width = mbswidth (col_sep_string, 0);
1884+ else
1885+#endif
1886+ col_sep_width = col_sep_length;
1887 }
1888
1889 int
1890@@ -876,6 +944,21 @@
1891
1892 atexit (close_stdout);
1893
1894+/* Define which functions are used, the ones for single byte locale or the ones
1895+ for multibyte locale. */
1896+#if HAVE_MBRTOWC
1897+ if (MB_CUR_MAX > 1)
1898+ {
1899+ print_char = print_char_multi;
1900+ char_to_clump = char_to_clump_multi;
1901+ }
1902+ else
1903+#endif
1904+ {
1905+ print_char = print_char_single;
1906+ char_to_clump = char_to_clump_single;
1907+ }
1908+
1909 n_files = 0;
1910 file_names = (argc > 1
1911 ? xmalloc ((argc - 1) * sizeof (char *))
1912@@ -952,8 +1035,12 @@
1913 break;
1914 case 'e':
1915 if (optarg)
1916- getoptarg (optarg, 'e', &input_tab_char,
1917- &chars_per_input_tab);
1918+ {
1919+ int dummy_length, dummy_width;
1920+
1921+ getoptarg (optarg, 'e', input_tab_char, &dummy_length,
1922+ &dummy_width, &chars_per_input_tab);
1923+ }
1924 /* Could check tab width > 0. */
1925 untabify_input = true;
1926 break;
1927@@ -966,8 +1053,12 @@
1928 break;
1929 case 'i':
1930 if (optarg)
1931- getoptarg (optarg, 'i', &output_tab_char,
1932- &chars_per_output_tab);
1933+ {
1934+ int dummy_width;
1935+
1936+ getoptarg (optarg, 'i', output_tab_char, &output_tab_char_length,
1937+ &dummy_width, &chars_per_output_tab);
1938+ }
1939 /* Could check tab width > 0. */
1940 tabify_output = true;
1941 break;
1942@@ -994,8 +1085,8 @@
1943 case 'n':
1944 numbered_lines = true;
1945 if (optarg)
1946- getoptarg (optarg, 'n', &number_separator,
1947- &chars_per_number);
1948+ getoptarg (optarg, 'n', number_separator, &number_separator_length,
1949+ &number_separator_width, &chars_per_number);
1950 break;
1951 case 'N':
1952 skip_count = false;
1953@@ -1034,7 +1125,7 @@
1954 old_s = false;
1955 /* Reset an additional input of -s, -S dominates -s */
1956 col_sep_string = "";
1957- col_sep_length = 0;
1958+ col_sep_length = col_sep_width = 0;
1959 use_col_separator = true;
1960 if (optarg)
1961 separator_string (optarg);
1962@@ -1191,10 +1282,45 @@
1963 a number. */
1964
1965 static void
1966-getoptarg (char *arg, char switch_char, char *character, int *number)
1967+getoptarg (char *arg, char switch_char, char *character, int *character_length,
1968+ int *character_width, int *number)
1969 {
1970 if (!ISDIGIT (*arg))
1971- *character = *arg++;
1972+ {
1973+#ifdef HAVE_MBRTOWC
1974+ if (MB_CUR_MAX > 1) /* for multibyte locale. */
1975+ {
1976+ wchar_t wc;
1977+ size_t mblength;
1978+ int width;
1979+ mbstate_t state = {'\0'};
1980+
1981+ mblength = mbrtowc (&wc, arg, strnlen(arg, MB_LEN_MAX), &state);
1982+
1983+ if (mblength == (size_t)-1 || mblength == (size_t)-2)
1984+ {
1985+ *character_length = 1;
1986+ *character_width = 1;
1987+ }
1988+ else
1989+ {
1990+ *character_length = (mblength < 1) ? 1 : mblength;
1991+ width = wcwidth (wc);
1992+ *character_width = (width < 0) ? 0 : width;
1993+ }
1994+
1995+ strncpy (character, arg, *character_length);
1996+ arg += *character_length;
1997+ }
1998+ else /* for single byte locale. */
1999+#endif
2000+ {
2001+ *character = *arg++;
2002+ *character_length = 1;
2003+ *character_width = 1;
2004+ }
2005+ }
2006+
2007 if (*arg)
2008 {
2009 long int tmp_long;
2010@@ -1253,7 +1379,7 @@
2011 else
2012 col_sep_string = column_separator;
2013
2014- col_sep_length = 1;
2015+ col_sep_length = col_sep_width = 1;
2016 use_col_separator = true;
2017 }
2018 /* It's rather pointless to define a TAB separator with column
2019@@ -1284,11 +1410,11 @@
2020 TAB_WIDTH (chars_per_input_tab, chars_per_number); */
2021
2022 /* Estimate chars_per_text without any margin and keep it constant. */
2023- if (number_separator == '\t')
2024+ if (number_separator[0] == '\t')
2025 number_width = chars_per_number +
2026 TAB_WIDTH (chars_per_default_tab, chars_per_number);
2027 else
2028- number_width = chars_per_number + 1;
2029+ number_width = chars_per_number + number_separator_width;
2030
2031 /* The number is part of the column width unless we are
2032 printing files in parallel. */
2033@@ -1303,7 +1429,7 @@
2034 }
2035
2036 chars_per_column = (chars_per_line - chars_used_by_number -
2037- (columns - 1) * col_sep_length) / columns;
2038+ (columns - 1) * col_sep_width) / columns;
2039
2040 if (chars_per_column < 1)
2041 error (EXIT_FAILURE, 0, _("page width too narrow"));
2042@@ -1428,7 +1554,7 @@
2043
2044 /* Enlarge p->start_position of first column to use the same form of
2045 padding_not_printed with all columns. */
2046- h = h + col_sep_length;
2047+ h = h + col_sep_width;
2048
2049 /* This loop takes care of all but the rightmost column. */
2050
2051@@ -1462,7 +1588,7 @@
2052 }
2053 else
2054 {
2055- h = h_next + col_sep_length;
2056+ h = h_next + col_sep_width;
2057 h_next = h + chars_per_column;
2058 }
2059 }
2060@@ -1752,9 +1878,9 @@
2061 align_column (COLUMN *p)
2062 {
2063 padding_not_printed = p->start_position;
2064- if (padding_not_printed - col_sep_length > 0)
2065+ if (padding_not_printed - col_sep_width > 0)
2066 {
2067- pad_across_to (padding_not_printed - col_sep_length);
2068+ pad_across_to (padding_not_printed - col_sep_width);
2069 padding_not_printed = ANYWHERE;
2070 }
2071
2072@@ -2025,13 +2151,13 @@
2073 /* May be too generous. */
2074 buff = X2REALLOC (buff, &buff_allocated);
2075 }
2076- buff[buff_current++] = c;
2077+ buff[buff_current++] = (unsigned char) c;
2078 }
2079
2080 static void
2081 add_line_number (COLUMN *p)
2082 {
2083- int i;
2084+ int i, j;
2085 char *s;
2086 int left_cut;
2087
2088@@ -2054,22 +2180,24 @@
2089 /* Tabification is assumed for multiple columns, also for n-separators,
2090 but `default n-separator = TAB' hasn't been given priority over
2091 equal column_width also specified by POSIX. */
2092- if (number_separator == '\t')
2093+ if (number_separator[0] == '\t')
2094 {
2095 i = number_width - chars_per_number;
2096 while (i-- > 0)
2097 (p->char_func) (' ');
2098 }
2099 else
2100- (p->char_func) (number_separator);
2101+ for (j = 0; j < number_separator_length; j++)
2102+ (p->char_func) (number_separator[j]);
2103 }
2104 else
2105 /* To comply with POSIX, we avoid any expansion of default TAB
2106 separator with a single column output. No column_width requirement
2107 has to be considered. */
2108 {
2109- (p->char_func) (number_separator);
2110- if (number_separator == '\t')
2111+ for (j = 0; j < number_separator_length; j++)
2112+ (p->char_func) (number_separator[j]);
2113+ if (number_separator[0] == '\t')
2114 output_position = POS_AFTER_TAB (chars_per_output_tab,
2115 output_position);
2116 }
2117@@ -2230,7 +2358,7 @@
2118 while (goal - h_old > 1
2119 && (h_new = POS_AFTER_TAB (chars_per_output_tab, h_old)) <= goal)
2120 {
2121- putchar (output_tab_char);
2122+ fwrite (output_tab_char, sizeof(char), output_tab_char_length, stdout);
2123 h_old = h_new;
2124 }
2125 while (++h_old <= goal)
2126@@ -2250,6 +2378,7 @@
2127 {
2128 char *s;
2129 int l = col_sep_length;
2130+ int not_space_flag;
2131
2132 s = col_sep_string;
2133
2134@@ -2263,6 +2392,7 @@
2135 {
2136 for (; separators_not_printed > 0; --separators_not_printed)
2137 {
2138+ not_space_flag = 0;
2139 while (l-- > 0)
2140 {
2141 /* 3 types of sep_strings: spaces only, spaces and chars,
2142@@ -2276,12 +2406,15 @@
2143 }
2144 else
2145 {
2146+ not_space_flag = 1;
2147 if (spaces_not_printed > 0)
2148 print_white_space ();
2149 putchar (*s++);
2150- ++output_position;
2151 }
2152 }
2153+ if (not_space_flag)
2154+ output_position += col_sep_width;
2155+
2156 /* sep_string ends with some spaces */
2157 if (spaces_not_printed > 0)
2158 print_white_space ();
2159@@ -2309,7 +2442,7 @@
2160 required number of tabs and spaces. */
2161
2162 static void
2163-print_char (char c)
2164+print_char_single (char c)
2165 {
2166 if (tabify_output)
2167 {
2168@@ -2333,6 +2466,74 @@
2169 putchar (c);
2170 }
2171
2172+#ifdef HAVE_MBRTOWC
2173+static void
2174+print_char_multi (char c)
2175+{
2176+ static size_t mbc_pos = 0;
2177+ static char mbc[MB_LEN_MAX] = {'\0'};
2178+ static mbstate_t state = {'\0'};
2179+ mbstate_t state_bak;
2180+ wchar_t wc;
2181+ size_t mblength;
2182+ int width;
2183+
2184+ if (tabify_output)
2185+ {
2186+ state_bak = state;
2187+ mbc[mbc_pos++] = c;
2188+ mblength = mbrtowc (&wc, mbc, mbc_pos, &state);
2189+
2190+ while (mbc_pos > 0)
2191+ {
2192+ switch (mblength)
2193+ {
2194+ case (size_t)-2:
2195+ state = state_bak;
2196+ return;
2197+
2198+ case (size_t)-1:
2199+ state = state_bak;
2200+ ++output_position;
2201+ putchar (mbc[0]);
2202+ memmove (mbc, mbc + 1, MB_CUR_MAX - 1);
2203+ --mbc_pos;
2204+ break;
2205+
2206+ case 0:
2207+ mblength = 1;
2208+
2209+ default:
2210+ if (wc == L' ')
2211+ {
2212+ memmove (mbc, mbc + mblength, MB_CUR_MAX - mblength);
2213+ --mbc_pos;
2214+ ++spaces_not_printed;
2215+ return;
2216+ }
2217+ else if (spaces_not_printed > 0)
2218+ print_white_space ();
2219+
2220+ /* Nonprintables are assumed to have width 0, except L'\b'. */
2221+ if ((width = wcwidth (wc)) < 1)
2222+ {
2223+ if (wc == L'\b')
2224+ --output_position;
2225+ }
2226+ else
2227+ output_position += width;
2228+
2229+ fwrite (mbc, sizeof(char), mblength, stdout);
2230+ memmove (mbc, mbc + mblength, MB_CUR_MAX - mblength);
2231+ mbc_pos -= mblength;
2232+ }
2233+ }
2234+ return;
2235+ }
2236+ putchar (c);
2237+}
2238+#endif
2239+
2240 /* Skip to page PAGE before printing.
2241 PAGE may be larger than total number of pages. */
2242
2243@@ -2510,9 +2711,9 @@
2244 align_empty_cols = false;
2245 }
2246
2247- if (padding_not_printed - col_sep_length > 0)
2248+ if (padding_not_printed - col_sep_width > 0)
2249 {
2250- pad_across_to (padding_not_printed - col_sep_length);
2251+ pad_across_to (padding_not_printed - col_sep_width);
2252 padding_not_printed = ANYWHERE;
2253 }
2254
2255@@ -2613,9 +2814,9 @@
2256 }
2257 }
2258
2259- if (padding_not_printed - col_sep_length > 0)
2260+ if (padding_not_printed - col_sep_width > 0)
2261 {
2262- pad_across_to (padding_not_printed - col_sep_length);
2263+ pad_across_to (padding_not_printed - col_sep_width);
2264 padding_not_printed = ANYWHERE;
2265 }
2266
2267@@ -2628,8 +2829,8 @@
2268 if (spaces_not_printed == 0)
2269 {
2270 output_position = p->start_position + end_vector[line];
2271- if (p->start_position - col_sep_length == chars_per_margin)
2272- output_position -= col_sep_length;
2273+ if (p->start_position - col_sep_width == chars_per_margin)
2274+ output_position -= col_sep_width;
2275 }
2276
2277 return true;
2278@@ -2648,7 +2849,7 @@
2279 number of characters is 1.) */
2280
2281 static int
2282-char_to_clump (char c)
2283+char_to_clump_single (char c)
2284 {
2285 unsigned char uc = c;
2286 char *s = clump_buff;
2287@@ -2658,10 +2859,10 @@
2288 int chars;
2289 int chars_per_c = 8;
2290
2291- if (c == input_tab_char)
2292+ if (c == input_tab_char[0])
2293 chars_per_c = chars_per_input_tab;
2294
2295- if (c == input_tab_char || c == '\t')
2296+ if (c == input_tab_char[0] || c == '\t')
2297 {
2298 width = TAB_WIDTH (chars_per_c, input_position);
2299
2300@@ -2742,6 +2943,154 @@
2301 return chars;
2302 }
2303
2304+#ifdef HAVE_MBRTOWC
2305+static int
2306+char_to_clump_multi (char c)
2307+{
2308+ static size_t mbc_pos = 0;
2309+ static char mbc[MB_LEN_MAX] = {'\0'};
2310+ static mbstate_t state = {'\0'};
2311+ mbstate_t state_bak;
2312+ wchar_t wc;
2313+ size_t mblength;
2314+ int wc_width;
2315+ register char *s = clump_buff;
2316+ register int i, j;
2317+ char esc_buff[4];
2318+ int width;
2319+ int chars;
2320+ int chars_per_c = 8;
2321+
2322+ state_bak = state;
2323+ mbc[mbc_pos++] = c;
2324+ mblength = mbrtowc (&wc, mbc, mbc_pos, &state);
2325+
2326+ width = 0;
2327+ chars = 0;
2328+ while (mbc_pos > 0)
2329+ {
2330+ switch (mblength)
2331+ {
2332+ case (size_t)-2:
2333+ state = state_bak;
2334+ return 0;
2335+
2336+ case (size_t)-1:
2337+ state = state_bak;
2338+ mblength = 1;
2339+
2340+ if (use_esc_sequence || use_cntrl_prefix)
2341+ {
2342+ width = +4;
2343+ chars = +4;
2344+ *s++ = '\\';
2345+ sprintf (esc_buff, "%03o", mbc[0]);
2346+ for (i = 0; i <= 2; ++i)
2347+ *s++ = (int) esc_buff[i];
2348+ }
2349+ else
2350+ {
2351+ width += 1;
2352+ chars += 1;
2353+ *s++ = mbc[0];
2354+ }
2355+ break;
2356+
2357+ case 0:
2358+ mblength = 1;
2359+ /* Fall through */
2360+
2361+ default:
2362+ if (memcmp (mbc, input_tab_char, mblength) == 0)
2363+ chars_per_c = chars_per_input_tab;
2364+
2365+ if (memcmp (mbc, input_tab_char, mblength) == 0 || c == '\t')
2366+ {
2367+ int width_inc;
2368+
2369+ width_inc = TAB_WIDTH (chars_per_c, input_position);
2370+ width += width_inc;
2371+
2372+ if (untabify_input)
2373+ {
2374+ for (i = width_inc; i; --i)
2375+ *s++ = ' ';
2376+ chars += width_inc;
2377+ }
2378+ else
2379+ {
2380+ for (i = 0; i < mblength; i++)
2381+ *s++ = mbc[i];
2382+ chars += mblength;
2383+ }
2384+ }
2385+ else if ((wc_width = wcwidth (wc)) < 1)
2386+ {
2387+ if (use_esc_sequence)
2388+ {
2389+ for (i = 0; i < mblength; i++)
2390+ {
2391+ width += 4;
2392+ chars += 4;
2393+ *s++ = '\\';
2394+ sprintf (esc_buff, "%03o", c);
2395+ for (j = 0; j <= 2; ++j)
2396+ *s++ = (int) esc_buff[j];
2397+ }
2398+ }
2399+ else if (use_cntrl_prefix)
2400+ {
2401+ if (wc < 0200)
2402+ {
2403+ width += 2;
2404+ chars += 2;
2405+ *s++ = '^';
2406+ *s++ = wc ^ 0100;
2407+ }
2408+ else
2409+ {
2410+ for (i = 0; i < mblength; i++)
2411+ {
2412+ width += 4;
2413+ chars += 4;
2414+ *s++ = '\\';
2415+ sprintf (esc_buff, "%03o", c);
2416+ for (j = 0; j <= 2; ++j)
2417+ *s++ = (int) esc_buff[j];
2418+ }
2419+ }
2420+ }
2421+ else if (wc == L'\b')
2422+ {
2423+ width += -1;
2424+ chars += 1;
2425+ *s++ = c;
2426+ }
2427+ else
2428+ {
2429+ width += 0;
2430+ chars += mblength;
2431+ for (i = 0; i < mblength; i++)
2432+ *s++ = mbc[i];
2433+ }
2434+ }
2435+ else
2436+ {
2437+ width += wc_width;
2438+ chars += mblength;
2439+ for (i = 0; i < mblength; i++)
2440+ *s++ = mbc[i];
2441+ }
2442+ }
2443+ memmove (mbc, mbc + mblength, MB_CUR_MAX - mblength);
2444+ mbc_pos -= mblength;
2445+ }
2446+
2447+ input_position += width;
2448+ return chars;
2449+}
2450+#endif
2451+
2452 /* We've just printed some files and need to clean up things before
2453 looking for more options and printing the next batch of files.
2454
2455diff -Naur coreutils-6.12.orig/src/sort.c coreutils-6.12/src/sort.c
2456--- coreutils-6.12.orig/src/sort.c 2008-05-25 23:40:32.000000000 -0700
[d33f80c]2457+++ coreutils-6.12/src/sort.c 2009-01-08 20:14:37.000000000 -0800
[577dd2d]2458@@ -22,10 +22,19 @@
2459
2460 #include <config.h>
2461
2462+#include <assert.h>
2463 #include <getopt.h>
2464 #include <sys/types.h>
2465 #include <sys/wait.h>
2466 #include <signal.h>
2467+#if HAVE_WCHAR_H
2468+# include <wchar.h>
2469+#endif
2470+/* Get isw* functions. */
2471+#if HAVE_WCTYPE_H
2472+# include <wctype.h>
2473+#endif
2474+
2475 #include "system.h"
2476 #include "argmatch.h"
2477 #include "error.h"
2478@@ -117,14 +126,38 @@
2479 /* Thousands separator; if -1, then there isn't one. */
2480 static int thousands_sep;
2481
2482+static int force_general_numcompare = 0;
2483+
2484 /* Nonzero if the corresponding locales are hard. */
2485 static bool hard_LC_COLLATE;
2486-#if HAVE_NL_LANGINFO
2487+#if HAVE_LANGINFO_CODESET
2488 static bool hard_LC_TIME;
2489 #endif
2490
2491 #define NONZERO(x) ((x) != 0)
2492
2493+/* get a multibyte character's byte length. */
2494+#define GET_BYTELEN_OF_CHAR(LIM, PTR, MBLENGTH, STATE) \
2495+ do \
2496+ { \
2497+ wchar_t wc; \
2498+ mbstate_t state_bak; \
2499+ \
2500+ state_bak = STATE; \
2501+ mblength = mbrtowc (&wc, PTR, LIM - PTR, &STATE); \
2502+ \
2503+ switch (MBLENGTH) \
2504+ { \
2505+ case (size_t)-1: \
2506+ case (size_t)-2: \
2507+ STATE = state_bak; \
2508+ /* Fall through. */ \
2509+ case 0: \
2510+ MBLENGTH = 1; \
2511+ } \
2512+ } \
2513+ while (0)
2514+
2515 /* The kind of blanks for '-b' to skip in various options. */
2516 enum blanktype { bl_start, bl_end, bl_both };
2517
2518@@ -262,13 +295,11 @@
2519 they were read if all keys compare equal. */
2520 static bool stable;
2521
2522-/* If TAB has this value, blanks separate fields. */
2523-enum { TAB_DEFAULT = CHAR_MAX + 1 };
2524-
2525-/* Tab character separating fields. If TAB_DEFAULT, then fields are
2526+/* Tab character separating fields. If tab_length is 0, then fields are
2527 separated by the empty string between a non-blank character and a blank
2528 character. */
2529-static int tab = TAB_DEFAULT;
2530+static char tab[MB_LEN_MAX + 1];
2531+static size_t tab_length = 0;
2532
2533 /* Flag to remove consecutive duplicate lines from the output.
2534 Only the last of a sequence of equal lines will be output. */
2535@@ -655,6 +686,44 @@
2536 update_proc (pid);
2537 }
2538
2539+/* Function pointers. */
2540+static void
2541+(*inittables) (void);
2542+static char *
2543+(*begfield) (const struct line*, const struct keyfield *);
2544+static char *
2545+(*limfield) (const struct line*, const struct keyfield *);
2546+static int
2547+(*getmonth) (char const *, size_t);
2548+static int
2549+(*keycompare) (const struct line *, const struct line *);
2550+static int
2551+(*numcompare) (const char *, const char *);
2552+
2553+/* Test for white space multibyte character.
2554+ Set LENGTH the byte length of investigated multibyte character. */
2555+#if HAVE_MBRTOWC
2556+static int
2557+ismbblank (const char *str, size_t len, size_t *length)
2558+{
2559+ size_t mblength;
2560+ wchar_t wc;
2561+ mbstate_t state;
2562+
2563+ memset (&state, '\0', sizeof(mbstate_t));
2564+ mblength = mbrtowc (&wc, str, len, &state);
2565+
2566+ if (mblength == (size_t)-1 || mblength == (size_t)-2)
2567+ {
2568+ *length = 1;
2569+ return 0;
2570+ }
2571+
2572+ *length = (mblength < 1) ? 1 : mblength;
2573+ return iswblank (wc);
2574+}
2575+#endif
2576+
2577 /* Clean up any remaining temporary files. */
2578
2579 static void
2580@@ -994,7 +1063,7 @@
2581 free (node);
2582 }
2583
2584-#if HAVE_NL_LANGINFO
2585+#if HAVE_LANGINFO_CODESET
2586
2587 static int
2588 struct_month_cmp (const void *m1, const void *m2)
2589@@ -1009,7 +1078,7 @@
2590 /* Initialize the character class tables. */
2591
2592 static void
2593-inittables (void)
2594+inittables_uni (void)
2595 {
2596 size_t i;
2597
2598@@ -1021,7 +1090,7 @@
2599 fold_toupper[i] = toupper (i);
2600 }
2601
2602-#if HAVE_NL_LANGINFO
2603+#if HAVE_LANGINFO_CODESET
2604 /* If we're not in the "C" locale, read different names for months. */
2605 if (hard_LC_TIME)
2606 {
2607@@ -1047,6 +1116,64 @@
2608 #endif
2609 }
2610
2611+#if HAVE_MBRTOWC
2612+static void
2613+inittables_mb (void)
2614+{
2615+ int i, j, k, l;
2616+ char *name, *s;
2617+ size_t s_len, mblength;
2618+ char mbc[MB_LEN_MAX];
2619+ wchar_t wc, pwc;
2620+ mbstate_t state_mb, state_wc;
2621+
2622+ for (i = 0; i < MONTHS_PER_YEAR; i++)
2623+ {
2624+ s = (char *) nl_langinfo (ABMON_1 + i);
2625+ s_len = strlen (s);
2626+ monthtab[i].name = name = (char *) xmalloc (s_len + 1);
2627+ monthtab[i].val = i + 1;
2628+
2629+ memset (&state_mb, '\0', sizeof (mbstate_t));
2630+ memset (&state_wc, '\0', sizeof (mbstate_t));
2631+
2632+ for (j = 0; j < s_len;)
2633+ {
2634+ if (!ismbblank (s + j, s_len - j, &mblength))
2635+ break;
2636+ j += mblength;
2637+ }
2638+
2639+ for (k = 0; j < s_len;)
2640+ {
2641+ mblength = mbrtowc (&wc, (s + j), (s_len - j), &state_mb);
2642+ assert (mblength != (size_t)-1 && mblength != (size_t)-2);
2643+ if (mblength == 0)
2644+ break;
2645+
2646+ pwc = towupper (wc);
2647+ if (pwc == wc)
2648+ {
2649+ memcpy (mbc, s + j, mblength);
2650+ j += mblength;
2651+ }
2652+ else
2653+ {
2654+ j += mblength;
2655+ mblength = wcrtomb (mbc, pwc, &state_wc);
2656+ assert (mblength != (size_t)0 && mblength != (size_t)-1);
2657+ }
2658+
2659+ for (l = 0; l < mblength; l++)
2660+ name[k++] = mbc[l];
2661+ }
2662+ name[k] = '\0';
2663+ }
2664+ qsort ((void *) monthtab, MONTHS_PER_YEAR,
2665+ sizeof (struct month), struct_month_cmp);
2666+}
2667+#endif
2668+
2669 /* Specify the amount of main memory to use when sorting. */
2670 static void
2671 specify_sort_size (int oi, char c, char const *s)
2672@@ -1257,7 +1384,7 @@
2673 by KEY in LINE. */
2674
2675 static char *
2676-begfield (const struct line *line, const struct keyfield *key)
2677+begfield_uni (const struct line *line, const struct keyfield *key)
2678 {
2679 char *ptr = line->text, *lim = ptr + line->length - 1;
2680 size_t sword = key->sword;
2681@@ -1267,10 +1394,10 @@
2682 /* The leading field separator itself is included in a field when -t
2683 is absent. */
2684
2685- if (tab != TAB_DEFAULT)
2686+ if (tab_length)
2687 while (ptr < lim && sword--)
2688 {
2689- while (ptr < lim && *ptr != tab)
2690+ while (ptr < lim && *ptr != tab[0])
2691 ++ptr;
2692 if (ptr < lim)
2693 ++ptr;
2694@@ -1298,11 +1425,70 @@
2695 return ptr;
2696 }
2697
2698+#if HAVE_MBRTOWC
2699+static char *
2700+begfield_mb (const struct line *line, const struct keyfield *key)
2701+{
2702+ int i;
2703+ char *ptr = line->text, *lim = ptr + line->length - 1;
2704+ size_t sword = key->sword;
2705+ size_t schar = key->schar;
2706+ size_t mblength;
2707+ mbstate_t state;
2708+
2709+ memset (&state, '\0', sizeof(mbstate_t));
2710+
2711+ if (tab_length)
2712+ while (ptr < lim && sword--)
2713+ {
2714+ while (ptr < lim && memcmp (ptr, tab, tab_length) != 0)
2715+ {
2716+ GET_BYTELEN_OF_CHAR (lim, ptr, mblength, state);
2717+ ptr += mblength;
2718+ }
2719+ if (ptr < lim)
2720+ {
2721+ GET_BYTELEN_OF_CHAR (lim, ptr, mblength, state);
2722+ ptr += mblength;
2723+ }
2724+ }
2725+ else
2726+ while (ptr < lim && sword--)
2727+ {
2728+ while (ptr < lim && ismbblank (ptr, lim - ptr, &mblength))
2729+ ptr += mblength;
2730+ if (ptr < lim)
2731+ {
2732+ GET_BYTELEN_OF_CHAR (lim, ptr, mblength, state);
2733+ ptr += mblength;
2734+ }
2735+ while (ptr < lim && !ismbblank (ptr, lim - ptr, &mblength))
2736+ ptr += mblength;
2737+ }
2738+
2739+ if (key->skipsblanks)
2740+ while (ptr < lim && ismbblank (ptr, lim - ptr, &mblength))
2741+ ptr += mblength;
2742+
2743+ for (i = 0; i < schar; i++)
2744+ {
2745+ GET_BYTELEN_OF_CHAR (lim, ptr, mblength, state);
2746+
2747+ if (ptr + mblength > lim)
2748+ break;
2749+ else
2750+ ptr += mblength;
2751+ }
2752+
2753+ return ptr;
2754+}
2755+#endif
2756+
2757 /* Return the limit of (a pointer to the first character after) the field
2758 in LINE specified by KEY. */
2759
2760 static char *
2761-limfield (const struct line *line, const struct keyfield *key)
2762+limfield_uni (const struct line *line, const struct keyfield *key)
2763 {
2764 char *ptr = line->text, *lim = ptr + line->length - 1;
2765 size_t eword = key->eword, echar = key->echar;
2766@@ -1315,10 +1501,10 @@
2767 `beginning' is the first character following the delimiting TAB.
2768 Otherwise, leave PTR pointing at the first `blank' character after
2769 the preceding field. */
2770- if (tab != TAB_DEFAULT)
2771+ if (tab_length)
2772 while (ptr < lim && eword--)
2773 {
2774- while (ptr < lim && *ptr != tab)
2775+ while (ptr < lim && *ptr != tab[0])
2776 ++ptr;
2777 if (ptr < lim && (eword | echar))
2778 ++ptr;
2779@@ -1364,10 +1550,10 @@
2780 */
2781
2782 /* Make LIM point to the end of (one byte past) the current field. */
2783- if (tab != TAB_DEFAULT)
2784+ if (tab_length)
2785 {
2786 char *newlim;
2787- newlim = memchr (ptr, tab, lim - ptr);
2788+ newlim = memchr (ptr, tab[0], lim - ptr);
2789 if (newlim)
2790 lim = newlim;
2791 }
2792@@ -1400,6 +1586,107 @@
2793 return ptr;
2794 }
2795
2796+#if HAVE_MBRTOWC
2797+static char *
2798+limfield_mb (const struct line *line, const struct keyfield *key)
2799+{
2800+ char *ptr = line->text, *lim = ptr + line->length - 1;
2801+ size_t eword = key->eword, echar = key->echar;
2802+ int i;
2803+ size_t mblength;
2804+ mbstate_t state;
2805+
2806+ memset (&state, '\0', sizeof(mbstate_t));
2807+
2808+ if (tab_length)
2809+ while (ptr < lim && eword--)
2810+ {
2811+ while (ptr < lim && memcmp (ptr, tab, tab_length) != 0)
2812+ {
2813+ GET_BYTELEN_OF_CHAR (lim, ptr, mblength, state);
2814+ ptr += mblength;
2815+ }
2816+ if (ptr < lim && (eword | echar))
2817+ {
2818+ GET_BYTELEN_OF_CHAR (lim, ptr, mblength, state);
2819+ ptr += mblength;
2820+ }
2821+ }
2822+ else
2823+ while (ptr < lim && eword--)
2824+ {
2825+ while (ptr < lim && ismbblank (ptr, lim - ptr, &mblength))
2826+ ptr += mblength;
2827+ if (ptr < lim)
2828+ {
2829+ GET_BYTELEN_OF_CHAR (lim, ptr, mblength, state);
2830+ ptr += mblength;
2831+ }
2832+ while (ptr < lim && !ismbblank (ptr, lim - ptr, &mblength))
2833+ ptr += mblength;
2834+ }
2835+
2836+
2837+# ifdef POSIX_UNSPECIFIED
2838+ /* Make LIM point to the end of (one byte past) the current field. */
2839+ if (tab_length)
2840+ {
2841+ char *newlim, *p;
2842+
2843+ newlim = NULL;
2844+ for (p = ptr; p < lim;)
2845+ {
2846+ if (memcmp (p, tab, tab_length) == 0)
2847+ {
2848+ newlim = p;
2849+ break;
2850+ }
2851+
2852+ GET_BYTELEN_OF_CHAR (lim, ptr, mblength, state);
2853+ p += mblength;
2854+ }
2855+ }
2856+ else
2857+ {
2858+ char *newlim;
2859+ newlim = ptr;
2860+
2861+ while (newlim < lim && ismbblank (newlim, lim - newlim, &mblength))
2862+ newlim += mblength;
2863+ if (ptr < lim)
2864+ {
2865+ GET_BYTELEN_OF_CHAR (lim, ptr, mblength, state);
2866+ ptr += mblength;
2867+ }
2868+ while (newlim < lim && !ismbblank (newlim, lim - newlim, &mblength))
2869+ newlim += mblength;
2870+ lim = newlim;
2871+ }
2872+# endif
2873+
2874+ /* If we're skipping leading blanks, don't start counting characters
2875+ * until after skipping past any leading blanks. */
2876+ if (key->skipsblanks)
2877+ while (ptr < lim && ismbblank (ptr, lim - ptr, &mblength))
2878+ ptr += mblength;
2879+
2880+ memset (&state, '\0', sizeof(mbstate_t));
2881+
2882+ /* Advance PTR by ECHAR (if possible), but no further than LIM. */
2883+ for (i = 0; i < echar; i++)
2884+ {
2885+ GET_BYTELEN_OF_CHAR (lim, ptr, mblength, state);
2886+
2887+ if (ptr + mblength > lim)
2888+ break;
2889+ else
2890+ ptr += mblength;
2891+ }
2892+
2893+ return ptr;
2894+}
2895+#endif
2896+
2897 /* Fill BUF reading from FP, moving buf->left bytes from the end
2898 of buf->buf to the beginning first. If EOF is reached and the
2899 file wasn't terminated by a newline, supply one. Set up BUF's line
2900@@ -1482,8 +1769,24 @@
2901 else
2902 {
2903 if (key->skipsblanks)
2904- while (blanks[to_uchar (*line_start)])
2905- line_start++;
2906+ {
2907+#if HAVE_MBRTOWC
2908+ if (MB_CUR_MAX > 1)
2909+ {
2910+ size_t mblength;
2911+ mbstate_t state;
2912+ memset (&state, '\0', sizeof(mbstate_t));
2913+ while (line_start < line->keylim &&
2914+ ismbblank (line_start,
2915+ line->keylim - line_start,
2916+ &mblength))
2917+ line_start += mblength;
2918+ }
2919+ else
2920+#endif
2921+ while (blanks[to_uchar (*line_start)])
2922+ line_start++;
2923+ }
2924 line->keybeg = line_start;
2925 }
2926 }
2927@@ -1521,7 +1824,7 @@
2928 hideously fast. */
2929
2930 static int
2931-numcompare (const char *a, const char *b)
2932+numcompare_uni (const char *a, const char *b)
2933 {
2934 while (blanks[to_uchar (*a)])
2935 a++;
2936@@ -1531,6 +1834,25 @@
2937 return strnumcmp (a, b, decimal_point, thousands_sep);
2938 }
2939
2940+#if HAVE_MBRTOWC
2941+static int
2942+numcompare_mb (const char *a, const char *b)
2943+{
2944+ size_t mblength, len;
2945+ len = strlen (a); /* okay for UTF-8 */
2946+ while (*a && ismbblank (a, len > MB_CUR_MAX ? MB_CUR_MAX : len, &mblength))
2947+ {
2948+ a += mblength;
2949+ len -= mblength;
2950+ }
2951+ len = strlen (b); /* okay for UTF-8 */
2952+ while (*b && ismbblank (b, len > MB_CUR_MAX ? MB_CUR_MAX : len, &mblength))
2953+ b += mblength;
2954+
2955+ return strnumcmp (a, b, decimal_point, thousands_sep);
2956+}
2957+#endif /* HAV_EMBRTOWC */
2958+
2959 static int
2960 general_numcompare (const char *sa, const char *sb)
2961 {
2962@@ -1564,7 +1886,7 @@
2963 Return 0 if the name in S is not recognized. */
2964
2965 static int
2966-getmonth (char const *month, size_t len)
2967+getmonth_uni (char const *month, size_t len)
2968 {
2969 size_t lo = 0;
2970 size_t hi = MONTHS_PER_YEAR;
2971@@ -1719,11 +2041,79 @@
2972 return diff;
2973 }
2974
2975+#if HAVE_MBRTOWC
2976+static int
2977+getmonth_mb (const char *s, size_t len)
2978+{
2979+ char *month;
2980+ register size_t i;
2981+ register int lo = 0, hi = MONTHS_PER_YEAR, result;
2982+ char *tmp;
2983+ size_t wclength, mblength;
2984+ const char **pp;
2985+ const wchar_t **wpp;
2986+ wchar_t *month_wcs;
2987+ mbstate_t state;
2988+
2989+ while (len > 0 && ismbblank (s, len, &mblength))
2990+ {
2991+ s += mblength;
2992+ len -= mblength;
2993+ }
2994+
2995+ if (len == 0)
2996+ return 0;
2997+
2998+ month = (char *) alloca (len + 1);
2999+
3000+ tmp = (char *) alloca (len + 1);
3001+ memcpy (tmp, s, len);
3002+ tmp[len] = '\0';
3003+ pp = (const char **)&tmp;
3004+ month_wcs = (wchar_t *) alloca ((len + 1) * sizeof (wchar_t));
3005+ memset (&state, '\0', sizeof(mbstate_t));
3006+
3007+ wclength = mbsrtowcs (month_wcs, pp, len + 1, &state);
3008+ assert (wclength != (size_t)-1 && *pp == NULL);
3009+
3010+ for (i = 0; i < wclength; i++)
3011+ {
3012+ month_wcs[i] = towupper(month_wcs[i]);
3013+ if (iswblank (month_wcs[i]))
3014+ {
3015+ month_wcs[i] = L'\0';
3016+ break;
3017+ }
3018+ }
3019+
3020+ wpp = (const wchar_t **)&month_wcs;
3021+
3022+ mblength = wcsrtombs (month, wpp, len + 1, &state);
3023+ assert (mblength != (-1) && *wpp == NULL);
3024+
3025+ do
3026+ {
3027+ int ix = (lo + hi) / 2;
3028+
3029+ if (strncmp (month, monthtab[ix].name, strlen (monthtab[ix].name)) < 0)
3030+ hi = ix;
3031+ else
3032+ lo = ix;
3033+ }
3034+ while (hi - lo > 1);
3035+
3036+ result = (!strncmp (month, monthtab[lo].name, strlen (monthtab[lo].name))
3037+ ? monthtab[lo].val : 0);
3038+
3039+ return result;
3040+}
3041+#endif
3042+
3043 /* Compare two lines A and B trying every key in sequence until there
3044 are no more keys or a difference is found. */
3045
3046 static int
3047-keycompare (const struct line *a, const struct line *b)
3048+keycompare_uni (const struct line *a, const struct line *b)
3049 {
3050 struct keyfield const *key = keylist;
3051
3052@@ -1896,6 +2286,179 @@
3053 return key->reverse ? -diff : diff;
3054 }
3055
3056+#if HAVE_MBRTOWC
3057+static int
3058+keycompare_mb (const struct line *a, const struct line *b)
3059+{
3060+ struct keyfield *key = keylist;
3061+
3062+ /* For the first iteration only, the key positions have been
3063+ precomputed for us. */
3064+ char *texta = a->keybeg;
3065+ char *textb = b->keybeg;
3066+ char *lima = a->keylim;
3067+ char *limb = b->keylim;
3068+
3069+ size_t mblength_a, mblength_b;
3070+ wchar_t wc_a, wc_b;
3071+ mbstate_t state_a, state_b;
3072+
3073+ int diff;
3074+
3075+ memset (&state_a, '\0', sizeof(mbstate_t));
3076+ memset (&state_b, '\0', sizeof(mbstate_t));
3077+
3078+ for (;;)
3079+ {
3080+ unsigned char *translate = (unsigned char *) key->translate;
3081+ bool const *ignore = key->ignore;
3082+
3083+ /* Find the lengths. */
3084+ size_t lena = lima <= texta ? 0 : lima - texta;
3085+ size_t lenb = limb <= textb ? 0 : limb - textb;
3086+
3087+ /* Actually compare the fields. */
3088+ if (key->random)
3089+ diff = compare_random (texta, lena, textb, lenb);
3090+ else if (key->numeric | key->general_numeric)
3091+ {
3092+ char savea = *lima, saveb = *limb;
3093+
3094+ *lima = *limb = '\0';
3095+ if (force_general_numcompare)
3096+ diff = general_numcompare (texta, textb);
3097+ else
3098+ diff = ((key->numeric ? numcompare : general_numcompare)
3099+ (texta, textb));
3100+ *lima = savea, *limb = saveb;
3101+ }
3102+ else if (key->month)
3103+ diff = getmonth (texta, lena) - getmonth (textb, lenb);
3104+ else
3105+ {
3106+ if (ignore || translate)
3107+ {
3108+ char *copy_a = (char *) alloca (lena + 1 + lenb + 1);
3109+ char *copy_b = copy_a + lena + 1;
3110+ size_t new_len_a, new_len_b;
3111+ size_t i, j;
3112+
3113+ /* Ignore and/or translate chars before comparing. */
3114+# define IGNORE_CHARS(NEW_LEN, LEN, TEXT, COPY, WC, MBLENGTH, STATE) \
3115+ do \
3116+ { \
3117+ wchar_t uwc; \
3118+ char mbc[MB_LEN_MAX]; \
3119+ mbstate_t state_wc; \
3120+ \
3121+ for (NEW_LEN = i = 0; i < LEN;) \
3122+ { \
3123+ mbstate_t state_bak; \
3124+ \
3125+ state_bak = STATE; \
3126+ MBLENGTH = mbrtowc (&WC, TEXT + i, LEN - i, &STATE); \
3127+ \
3128+ if (MBLENGTH == (size_t)-2 || MBLENGTH == (size_t)-1 \
3129+ || MBLENGTH == 0) \
3130+ { \
3131+ if (MBLENGTH == (size_t)-2 || MBLENGTH == (size_t)-1) \
3132+ STATE = state_bak; \
3133+ if (!ignore) \
3134+ COPY[NEW_LEN++] = TEXT[i++]; \
3135+ continue; \
3136+ } \
3137+ \
3138+ if (ignore) \
3139+ { \
3140+ if ((ignore == nonprinting && !iswprint (WC)) \
3141+ || (ignore == nondictionary \
3142+ && !iswalnum (WC) && !iswblank (WC))) \
3143+ { \
3144+ i += MBLENGTH; \
3145+ continue; \
3146+ } \
3147+ } \
3148+ \
3149+ if (translate) \
3150+ { \
3151+ \
3152+ uwc = towupper(WC); \
3153+ if (WC == uwc) \
3154+ { \
3155+ memcpy (mbc, TEXT + i, MBLENGTH); \
3156+ i += MBLENGTH; \
3157+ } \
3158+ else \
3159+ { \
3160+ i += MBLENGTH; \
3161+ WC = uwc; \
3162+ memset (&state_wc, '\0', sizeof (mbstate_t)); \
3163+ \
3164+ MBLENGTH = wcrtomb (mbc, WC, &state_wc); \
3165+ assert (MBLENGTH != (size_t)-1 && MBLENGTH != 0); \
3166+ } \
3167+ \
3168+ for (j = 0; j < MBLENGTH; j++) \
3169+ COPY[NEW_LEN++] = mbc[j]; \
3170+ } \
3171+ else \
3172+ for (j = 0; j < MBLENGTH; j++) \
3173+ COPY[NEW_LEN++] = TEXT[i++]; \
3174+ } \
3175+ COPY[NEW_LEN] = '\0'; \
3176+ } \
3177+ while (0)
3178+ IGNORE_CHARS (new_len_a, lena, texta, copy_a,
3179+ wc_a, mblength_a, state_a);
3180+ IGNORE_CHARS (new_len_b, lenb, textb, copy_b,
3181+ wc_b, mblength_b, state_b);
3182+ diff = xmemcoll (copy_a, new_len_a, copy_b, new_len_b);
3183+ }
3184+ else if (lena == 0)
3185+ diff = - NONZERO (lenb);
3186+ else if (lenb == 0)
3187+ goto greater;
3188+ else
3189+ diff = xmemcoll (texta, lena, textb, lenb);
3190+ }
3191+
3192+ if (diff)
3193+ goto not_equal;
3194+
3195+ key = key->next;
3196+ if (! key)
3197+ break;
3198+
3199+ /* Find the beginning and limit of the next field. */
3200+ if (key->eword != -1)
3201+ lima = limfield (a, key), limb = limfield (b, key);
3202+ else
3203+ lima = a->text + a->length - 1, limb = b->text + b->length - 1;
3204+
3205+ if (key->sword != -1)
3206+ texta = begfield (a, key), textb = begfield (b, key);
3207+ else
3208+ {
3209+ texta = a->text, textb = b->text;
3210+ if (key->skipsblanks)
3211+ {
3212+ while (texta < lima && ismbblank (texta, lima - texta, &mblength_a))
3213+ texta += mblength_a;
3214+ while (textb < limb && ismbblank (textb, limb - textb, &mblength_b))
3215+ textb += mblength_b;
3216+ }
3217+ }
3218+ }
3219+
3220+ return 0;
3221+
3222+greater:
3223+ diff = 1;
3224+not_equal:
3225+ return key->reverse ? -diff : diff;
3226+}
3227+#endif
3228+
3229 /* Compare two lines A and B, returning negative, zero, or positive
3230 depending on whether A compares less than, equal to, or greater than B. */
3231
3232@@ -2765,7 +3328,7 @@
3233 initialize_exit_failure (SORT_FAILURE);
3234
3235 hard_LC_COLLATE = hard_locale (LC_COLLATE);
3236-#if HAVE_NL_LANGINFO
3237+#if HAVE_LANGINFO_CODESET
3238 hard_LC_TIME = hard_locale (LC_TIME);
3239 #endif
3240
3241@@ -2786,6 +3349,27 @@
3242 thousands_sep = -1;
3243 }
3244
3245+#if HAVE_MBRTOWC
3246+ if (MB_CUR_MAX > 1)
3247+ {
3248+ inittables = inittables_mb;
3249+ begfield = begfield_mb;
3250+ limfield = limfield_mb;
3251+ getmonth = getmonth_mb;
3252+ keycompare = keycompare_mb;
3253+ numcompare = numcompare_mb;
3254+ }
3255+ else
3256+#endif
3257+ {
3258+ inittables = inittables_uni;
3259+ begfield = begfield_uni;
3260+ limfield = limfield_uni;
3261+ getmonth = getmonth_uni;
3262+ keycompare = keycompare_uni;
3263+ numcompare = numcompare_uni;
3264+ }
3265+
3266 have_read_stdin = false;
3267 inittables ();
3268
3269@@ -3037,13 +3621,35 @@
3270
3271 case 't':
3272 {
3273- char newtab = optarg[0];
3274- if (! newtab)
3275+ char newtab[MB_LEN_MAX + 1];
3276+ size_t newtab_length = 1;
3277+ strncpy (newtab, optarg, MB_LEN_MAX);
3278+ if (! newtab[0])
3279 error (SORT_FAILURE, 0, _("empty tab"));
3280- if (optarg[1])
3281+#if HAVE_MBRTOWC
3282+ if (MB_CUR_MAX > 1)
3283+ {
3284+ wchar_t wc;
3285+ mbstate_t state;
3286+ size_t i;
3287+
3288+ memset (&state, '\0', sizeof (mbstate_t));
3289+ newtab_length = mbrtowc (&wc, newtab, strnlen (newtab,
3290+ MB_LEN_MAX),
3291+ &state);
3292+ switch (newtab_length)
3293+ {
3294+ case (size_t) -1:
3295+ case (size_t) -2:
3296+ case 0:
3297+ newtab_length = 1;
3298+ }
3299+ }
3300+#endif
3301+ if (newtab_length == 1 && optarg[1])
3302 {
3303 if (STREQ (optarg, "\\0"))
3304- newtab = '\0';
3305+ newtab[0] = '\0';
3306 else
3307 {
3308 /* Provoke with `sort -txx'. Complain about
3309@@ -3054,9 +3660,12 @@
3310 quote (optarg));
3311 }
3312 }
3313- if (tab != TAB_DEFAULT && tab != newtab)
3314+ if (tab_length
3315+ && (tab_length != newtab_length
3316+ || memcmp (tab, newtab, tab_length) != 0))
3317 error (SORT_FAILURE, 0, _("incompatible tabs"));
3318- tab = newtab;
3319+ memcpy (tab, newtab, newtab_length);
3320+ tab_length = newtab_length;
3321 }
3322 break;
3323
3324diff -Naur coreutils-6.12.orig/src/unexpand.c coreutils-6.12/src/unexpand.c
3325--- coreutils-6.12.orig/src/unexpand.c 2008-05-25 23:40:33.000000000 -0700
[d33f80c]3326+++ coreutils-6.12/src/unexpand.c 2009-01-08 20:14:37.000000000 -0800
[577dd2d]3327@@ -38,11 +38,28 @@
3328 #include <stdio.h>
3329 #include <getopt.h>
3330 #include <sys/types.h>
3331+
3332+/* Get mbstate_t, mbrtowc(), wcwidth(). */
3333+#if HAVE_WCHAR_H
3334+# include <wchar.h>
3335+#endif
3336+
3337 #include "system.h"
3338 #include "error.h"
3339 #include "quote.h"
3340 #include "xstrndup.h"
3341
3342+/* MB_LEN_MAX is incorrectly defined to be 1 in at least one GCC
3343+ installation; work around this configuration error. */
3344+#if !defined MB_LEN_MAX || MB_LEN_MAX < 2
3345+# define MB_LEN_MAX 16
3346+#endif
3347+
3348+/* Some systems, like BeOS, have multibyte encodings but lack mbstate_t. */
3349+#if HAVE_MBRTOWC && defined mbstate_t
3350+# define mbrtowc(pwc, s, n, ps) (mbrtowc) (pwc, s, n, 0)
3351+#endif
3352+
3353 /* The official name of this program (e.g., no `g' prefix). */
3354 #define PROGRAM_NAME "unexpand"
3355
3356@@ -109,6 +126,208 @@
3357 {NULL, 0, NULL, 0}
3358 };
3359
3360+static FILE *next_file (FILE *fp);
3361+
3362+#if HAVE_MBRTOWC
3363+static void
3364+unexpand_multibyte (void)
3365+{
3366+ FILE *fp; /* Input stream. */
3367+ mbstate_t i_state; /* Current shift state of the input stream. */
3368+ mbstate_t i_state_bak; /* Back up the I_STATE. */
3369+ mbstate_t o_state; /* Current shift state of the output stream. */
3370+ char buf[MB_LEN_MAX + BUFSIZ]; /* For spooling a read byte sequence. */
3371+ char *bufpos; /* Next read position of BUF. */
3372+ size_t buflen = 0; /* The length of the byte sequence in buf. */
3373+ wint_t wc; /* A gotten wide character. */
3374+ size_t mblength; /* The byte size of a multibyte character
3375+ which shows as same character as WC. */
3376+
3377+ /* Index in `tab_list' of next tabstop: */
3378+ int tab_index = 0; /* For calculating width of pending tabs. */
3379+ int print_tab_index = 0; /* For printing as many tabs as possible. */
3380+ unsigned int column = 0; /* Column on screen of next char. */
3381+ int next_tab_column; /* Column the next tab stop is on. */
3382+ int convert = 1; /* If nonzero, perform translations. */
3383+ unsigned int pending = 0; /* Pending columns of blanks. */
3384+
3385+ fp = next_file ((FILE *) NULL);
3386+ if (fp == NULL)
3387+ return;
3388+
3389+ memset (&o_state, '\0', sizeof(mbstate_t));
3390+ memset (&i_state, '\0', sizeof(mbstate_t));
3391+
3392+ for (;;)
3393+ {
3394+ if (buflen < MB_LEN_MAX && !feof(fp) && !ferror(fp))
3395+ {
3396+ memmove (buf, bufpos, buflen);
3397+ buflen += fread (buf + buflen, sizeof(char), BUFSIZ, fp);
3398+ bufpos = buf;
3399+ }
3400+
3401+ /* Get a wide character. */
3402+ if (buflen < 1)
3403+ {
3404+ mblength = 1;
3405+ wc = WEOF;
3406+ }
3407+ else
3408+ {
3409+ i_state_bak = i_state;
3410+ mblength = mbrtowc ((wchar_t *)&wc, bufpos, buflen, &i_state);
3411+ }
3412+
3413+ if (mblength == (size_t)-1 || mblength == (size_t)-2)
3414+ {
3415+ i_state = i_state_bak;
3416+ wc = L'\0';
3417+ }
3418+
3419+ if (wc == L' ' && convert && column < INT_MAX)
3420+ {
3421+ ++pending;
3422+ ++column;
3423+ }
3424+ else if (wc == L'\t' && convert)
3425+ {
3426+ if (tab_size == 0)
3427+ {
3428+ /* Do not let tab_index == first_free_tab;
3429+ stop when it is 1 less. */
3430+ while (tab_index < first_free_tab - 1
3431+ && column >= tab_list[tab_index])
3432+ tab_index++;
3433+ next_tab_column = tab_list[tab_index];
3434+ if (tab_index < first_free_tab - 1)
3435+ tab_index++;
3436+ if (column >= next_tab_column)
3437+ {
3438+ convert = 0; /* Ran out of tab stops. */
3439+ goto flush_pend_mb;
3440+ }
3441+ }
3442+ else
3443+ {
3444+ next_tab_column = column + tab_size - column % tab_size;
3445+ }
3446+ pending += next_tab_column - column;
3447+ column = next_tab_column;
3448+ }
3449+ else
3450+ {
3451+flush_pend_mb:
3452+ /* Flush pending spaces. Print as many tabs as possible,
3453+ then print the rest as spaces. */
3454+ if (pending == 1)
3455+ {
3456+ putchar (' ');
3457+ pending = 0;
3458+ }
3459+ column -= pending;
3460+ while (pending > 0)
3461+ {
3462+ if (tab_size == 0)
3463+ {
3464+ /* Do not let print_tab_index == first_free_tab;
3465+ stop when it is 1 less. */
3466+ while (print_tab_index < first_free_tab - 1
3467+ && column >= tab_list[print_tab_index])
3468+ print_tab_index++;
3469+ next_tab_column = tab_list[print_tab_index];
3470+ if (print_tab_index < first_free_tab - 1)
3471+ print_tab_index++;
3472+ }
3473+ else
3474+ {
3475+ next_tab_column =
3476+ column + tab_size - column % tab_size;
3477+ }
3478+ if (next_tab_column - column <= pending)
3479+ {
3480+ putchar ('\t');
3481+ pending -= next_tab_column - column;
3482+ column = next_tab_column;
3483+ }
3484+ else
3485+ {
3486+ --print_tab_index;
3487+ column += pending;
3488+ while (pending != 0)
3489+ {
3490+ putchar (' ');
3491+ pending--;
3492+ }
3493+ }
3494+ }
3495+
3496+ if (wc == WEOF)
3497+ {
3498+ fp = next_file (fp);
3499+ if (fp == NULL)
3500+ break; /* No more files. */
3501+ else
3502+ {
3503+ memset (&i_state, '\0', sizeof(mbstate_t));
3504+ continue;
3505+ }
3506+ }
3507+
3508+ if (mblength == (size_t)-1 || mblength == (size_t)-2)
3509+ {
3510+ if (convert)
3511+ {
3512+ ++column;
3513+ if (convert_entire_line == 0)
3514+ convert = 0;
3515+ }
3516+ mblength = 1;
3517+ putchar (buf[0]);
3518+ }
3519+ else if (mblength == 0)
3520+ {
3521+ if (convert && convert_entire_line == 0)
3522+ convert = 0;
3523+ mblength = 1;
3524+ putchar ('\0');
3525+ }
3526+ else
3527+ {
3528+ if (convert)
3529+ {
3530+ if (wc == L'\b')
3531+ {
3532+ if (column > 0)
3533+ --column;
3534+ }
3535+ else
3536+ {
3537+ int width; /* The width of WC. */
3538+
3539+ width = wcwidth (wc);
3540+ column += (width > 0) ? width : 0;
3541+ if (convert_entire_line == 0)
3542+ convert = 0;
3543+ }
3544+ }
3545+
3546+ if (wc == L'\n')
3547+ {
3548+ tab_index = print_tab_index = 0;
3549+ column = pending = 0;
3550+ convert = 1;
3551+ }
3552+ fwrite (bufpos, sizeof(char), mblength, stdout);
3553+ }
3554+ }
3555+ buflen -= mblength;
3556+ bufpos += mblength;
3557+ }
3558+}
3559+#endif
3560+
3561+
3562 void
3563 usage (int status)
3564 {
3565@@ -530,7 +749,12 @@
3566
3567 file_list = (optind < argc ? &argv[optind] : stdin_argv);
3568
3569- unexpand ();
3570+#if HAVE_MBRTOWC
3571+ if (MB_CUR_MAX > 1)
3572+ unexpand_multibyte ();
3573+ else
3574+#endif
3575+ unexpand ();
3576
3577 if (have_read_stdin && fclose (stdin) != 0)
3578 error (EXIT_FAILURE, errno, "-");
3579diff -Naur coreutils-6.12.orig/src/uniq.c coreutils-6.12/src/uniq.c
3580--- coreutils-6.12.orig/src/uniq.c 2008-05-25 23:40:32.000000000 -0700
[d33f80c]3581+++ coreutils-6.12/src/uniq.c 2009-01-08 20:14:37.000000000 -0800
[577dd2d]3582@@ -22,6 +22,16 @@
3583 #include <getopt.h>
3584 #include <sys/types.h>
3585
3586+/* Get mbstate_t, mbrtowc(). */
3587+#if HAVE_WCHAR_H
3588+# include <wchar.h>
3589+#endif
3590+
3591+/* Get isw* functions. */
3592+#if HAVE_WCTYPE_H
3593+# include <wctype.h>
3594+#endif
3595+
3596 #include "system.h"
3597 #include "argmatch.h"
3598 #include "linebuffer.h"
3599@@ -31,7 +41,19 @@
3600 #include "quote.h"
3601 #include "xmemcoll.h"
3602 #include "xstrtol.h"
3603-#include "memcasecmp.h"
3604+#include "xmemcoll.h"
3605+
3606+/* MB_LEN_MAX is incorrectly defined to be 1 in at least one GCC
3607+ installation; work around this configuration error. */
3608+#if !defined MB_LEN_MAX || MB_LEN_MAX < 2
3609+# define MB_LEN_MAX 16
3610+#endif
3611+
3612+/* Some systems, like BeOS, have multibyte encodings but lack mbstate_t. */
3613+#if HAVE_MBRTOWC && defined mbstate_t
3614+# define mbrtowc(pwc, s, n, ps) (mbrtowc) (pwc, s, n, 0)
3615+#endif
3616+
3617
3618 /* The official name of this program (e.g., no `g' prefix). */
3619 #define PROGRAM_NAME "uniq"
3620@@ -110,6 +132,10 @@
3621 /* Select whether/how to delimit groups of duplicate lines. */
3622 static enum delimit_method delimit_groups;
3623
3624+/* Function pointers. */
3625+static char *
3626+(*find_field) (struct linebuffer *line);
3627+
3628 static struct option const longopts[] =
3629 {
3630 {"count", no_argument, NULL, 'c'},
3631@@ -206,7 +232,7 @@
3632 return a pointer to the beginning of the line's field to be compared. */
3633
3634 static char *
3635-find_field (const struct linebuffer *line)
3636+find_field_uni (struct linebuffer *line)
3637 {
3638 size_t count;
3639 char *lp = line->buffer;
3640@@ -227,6 +253,83 @@
3641 return lp + i;
3642 }
3643
3644+#if HAVE_MBRTOWC
3645+
3646+# define MBCHAR_TO_WCHAR(WC, MBLENGTH, LP, POS, SIZE, STATEP, CONVFAIL) \
3647+ do \
3648+ { \
3649+ mbstate_t state_bak; \
3650+ \
3651+ CONVFAIL = 0; \
3652+ state_bak = *STATEP; \
3653+ \
3654+ MBLENGTH = mbrtowc (&WC, LP + POS, SIZE - POS, STATEP); \
3655+ \
3656+ switch (MBLENGTH) \
3657+ { \
3658+ case (size_t)-2: \
3659+ case (size_t)-1: \
3660+ *STATEP = state_bak; \
3661+ CONVFAIL++; \
3662+ /* Fall through */ \
3663+ case 0: \
3664+ MBLENGTH = 1; \
3665+ } \
3666+ } \
3667+ while (0)
3668+
3669+static char *
3670+find_field_multi (struct linebuffer *line)
3671+{
3672+ size_t count;
3673+ char *lp = line->buffer;
3674+ size_t size = line->length - 1;
3675+ size_t pos;
3676+ size_t mblength;
3677+ wchar_t wc;
3678+ mbstate_t *statep;
3679+ int convfail;
3680+
3681+ pos = 0;
3682+ statep = &(line->state);
3683+
3684+ /* skip fields. */
3685+ for (count = 0; count < skip_fields && pos < size; count++)
3686+ {
3687+ while (pos < size)
3688+ {
3689+ MBCHAR_TO_WCHAR (wc, mblength, lp, pos, size, statep, convfail);
3690+
3691+ if (convfail || !iswblank (wc))
3692+ {
3693+ pos += mblength;
3694+ break;
3695+ }
3696+ pos += mblength;
3697+ }
3698+
3699+ while (pos < size)
3700+ {
3701+ MBCHAR_TO_WCHAR (wc, mblength, lp, pos, size, statep, convfail);
3702+
3703+ if (!convfail && iswblank (wc))
3704+ break;
3705+
3706+ pos += mblength;
3707+ }
3708+ }
3709+
3710+ /* skip fields. */
3711+ for (count = 0; count < skip_chars && pos < size; count++)
3712+ {
3713+ MBCHAR_TO_WCHAR (wc, mblength, lp, pos, size, statep, convfail);
3714+ pos += mblength;
3715+ }
3716+
3717+ return lp + pos;
3718+}
3719+#endif
3720+
3721 /* Return false if two strings OLD and NEW match, true if not.
3722 OLD and NEW point not to the beginnings of the lines
3723 but rather to the beginnings of the fields to compare.
3724@@ -235,6 +338,8 @@
3725 static bool
3726 different (char *old, char *new, size_t oldlen, size_t newlen)
3727 {
3728+ char *copy_old, *copy_new;
3729+
3730 if (check_chars < oldlen)
3731 oldlen = check_chars;
3732 if (check_chars < newlen)
3733@@ -242,14 +347,92 @@
3734
3735 if (ignore_case)
3736 {
3737- /* FIXME: This should invoke strcoll somehow. */
3738- return oldlen != newlen || memcasecmp (old, new, oldlen);
3739+ size_t i;
3740+
3741+ copy_old = alloca (oldlen + 1);
3742+ copy_new = alloca (oldlen + 1);
3743+
3744+ for (i = 0; i < oldlen; i++)
3745+ {
3746+ copy_old[i] = toupper (old[i]);
3747+ copy_new[i] = toupper (new[i]);
3748+ }
3749 }
3750- else if (hard_LC_COLLATE)
3751- return xmemcoll (old, oldlen, new, newlen) != 0;
3752 else
3753- return oldlen != newlen || memcmp (old, new, oldlen);
3754+ {
3755+ copy_old = (char *)old;
3756+ copy_new = (char *)new;
3757+ }
3758+
3759+ return xmemcoll (copy_old, oldlen, copy_new, newlen);
3760+}
3761+
3762+#if HAVE_MBRTOWC
3763+static int
3764+different_multi (const char *old, const char *new, size_t oldlen, size_t newlen, mbstate_t oldstate, mbstate_t newstate)
3765+{
3766+ size_t i, j, chars;
3767+ const char *str[2];
3768+ char *copy[2];
3769+ size_t len[2];
3770+ mbstate_t state[2];
3771+ size_t mblength;
3772+ wchar_t wc, uwc;
3773+ mbstate_t state_bak;
3774+
3775+ str[0] = old;
3776+ str[1] = new;
3777+ len[0] = oldlen;
3778+ len[1] = newlen;
3779+ state[0] = oldstate;
3780+ state[1] = newstate;
3781+
3782+ for (i = 0; i < 2; i++)
3783+ {
3784+ copy[i] = alloca (len[i] + 1);
3785+
3786+ for (j = 0, chars = 0; j < len[i] && chars < check_chars; chars++)
3787+ {
3788+ state_bak = state[i];
3789+ mblength = mbrtowc (&wc, str[i] + j, len[i] - j, &(state[i]));
3790+
3791+ switch (mblength)
3792+ {
3793+ case (size_t)-1:
3794+ case (size_t)-2:
3795+ state[i] = state_bak;
3796+ /* Fall through */
3797+ case 0:
3798+ mblength = 1;
3799+ break;
3800+
3801+ default:
3802+ if (ignore_case)
3803+ {
3804+ uwc = towupper (wc);
3805+
3806+ if (uwc != wc)
3807+ {
3808+ mbstate_t state_wc;
3809+
3810+ memset (&state_wc, '\0', sizeof(mbstate_t));
3811+ wcrtomb (copy[i] + j, uwc, &state_wc);
3812+ }
3813+ else
3814+ memcpy (copy[i] + j, str[i] + j, mblength);
3815+ }
3816+ else
3817+ memcpy (copy[i] + j, str[i] + j, mblength);
3818+ }
3819+ j += mblength;
3820+ }
3821+ copy[i][j] = '\0';
3822+ len[i] = j;
3823+ }
3824+
3825+ return xmemcoll (copy[0], len[0], copy[1], len[1]);
3826 }
3827+#endif
3828
3829 /* Output the line in linebuffer LINE to standard output
3830 provided that the switches say it should be output.
3831@@ -303,15 +486,43 @@
3832 {
3833 char *prevfield IF_LINT (= NULL);
3834 size_t prevlen IF_LINT (= 0);
3835+#if HAVE_MBRTOWC
3836+ mbstate_t prevstate;
3837+
3838+ memset (&prevstate, '\0', sizeof (mbstate_t));
3839+#endif
3840
3841 while (!feof (stdin))
3842 {
3843 char *thisfield;
3844 size_t thislen;
3845+#if HAVE_MBRTOWC
3846+ mbstate_t thisstate;
3847+#endif
3848+
3849 if (readlinebuffer_delim (thisline, stdin, delimiter) == 0)
3850 break;
3851 thisfield = find_field (thisline);
3852 thislen = thisline->length - 1 - (thisfield - thisline->buffer);
3853+#if HAVE_MBRTOWC
3854+ if (MB_CUR_MAX > 1)
3855+ {
3856+ thisstate = thisline->state;
3857+
3858+ if (prevline->length == 0 || different_multi
3859+ (thisfield, prevfield, thislen, prevlen, thisstate, prevstate))
3860+ {
3861+ fwrite (thisline->buffer, sizeof (char),
3862+ thisline->length, stdout);
3863+
3864+ SWAP_LINES (prevline, thisline);
3865+ prevfield = thisfield;
3866+ prevlen = thislen;
3867+ prevstate = thisstate;
3868+ }
3869+ }
3870+ else
3871+#endif
3872 if (prevline->length == 0
3873 || different (thisfield, prevfield, thislen, prevlen))
3874 {
3875@@ -330,17 +541,26 @@
3876 size_t prevlen;
3877 uintmax_t match_count = 0;
3878 bool first_delimiter = true;
3879+#if HAVE_MBRTOWC
3880+ mbstate_t prevstate;
3881+#endif
3882
3883 if (readlinebuffer_delim (prevline, stdin, delimiter) == 0)
3884 goto closefiles;
3885 prevfield = find_field (prevline);
3886 prevlen = prevline->length - 1 - (prevfield - prevline->buffer);
3887+#if HAVE_MBRTOWC
3888+ prevstate = prevline->state;
3889+#endif
3890
3891 while (!feof (stdin))
3892 {
3893 bool match;
3894 char *thisfield;
3895 size_t thislen;
3896+#if HAVE_MBRTOWC
3897+ mbstate_t thisstate;
3898+#endif
3899 if (readlinebuffer_delim (thisline, stdin, delimiter) == 0)
3900 {
3901 if (ferror (stdin))
3902@@ -349,6 +569,15 @@
3903 }
3904 thisfield = find_field (thisline);
3905 thislen = thisline->length - 1 - (thisfield - thisline->buffer);
3906+#if HAVE_MBRTOWC
3907+ if (MB_CUR_MAX > 1)
3908+ {
3909+ thisstate = thisline->state;
3910+ match = !different_multi (thisfield, prevfield,
3911+ thislen, prevlen, thisstate, prevstate);
3912+ }
3913+ else
3914+#endif
3915 match = !different (thisfield, prevfield, thislen, prevlen);
3916 match_count += match;
3917
3918@@ -381,6 +610,9 @@
3919 SWAP_LINES (prevline, thisline);
3920 prevfield = thisfield;
3921 prevlen = thislen;
3922+#if HAVE_MBRTOWC
3923+ prevstate = thisstate;
3924+#endif
3925 if (!match)
3926 match_count = 0;
3927 }
3928@@ -426,6 +658,19 @@
3929
3930 atexit (close_stdout);
3931
3932+#if HAVE_MBRTOWC
3933+ if (MB_CUR_MAX > 1)
3934+ {
3935+ find_field = find_field_multi;
3936+ }
3937+ else
3938+#endif
3939+ {
3940+ find_field = find_field_uni;
3941+ }
3942+
3943+
3944+
3945 skip_chars = 0;
3946 skip_fields = 0;
3947 check_chars = SIZE_MAX;
3948diff -Naur coreutils-6.12.orig/tests/Makefile.am coreutils-6.12/tests/Makefile.am
3949--- coreutils-6.12.orig/tests/Makefile.am 2008-05-27 04:47:53.000000000 -0700
[d33f80c]3950+++ coreutils-6.12/tests/Makefile.am 2009-01-08 20:14:37.000000000 -0800
[577dd2d]3951@@ -191,6 +191,7 @@
3952 misc/shuf \
3953 misc/sort \
3954 misc/sort-compress \
3955+ misc/sort-mb-tests \
3956 misc/sort-merge \
3957 misc/sort-rand \
3958 misc/split-a \
3959@@ -391,6 +392,10 @@
3960 $(root_tests)
3961
3962 pr_data = \
3963+ misc/mb1.X \
3964+ misc/mb1.I \
3965+ misc/mb2.X \
3966+ misc/mb2.I \
3967 pr/0F \
3968 pr/0FF \
3969 pr/0FFnt \
3970diff -Naur coreutils-6.12.orig/tests/misc/mb1.I coreutils-6.12/tests/misc/mb1.I
3971--- coreutils-6.12.orig/tests/misc/mb1.I 1969-12-31 16:00:00.000000000 -0800
[d33f80c]3972+++ coreutils-6.12/tests/misc/mb1.I 2009-01-08 20:14:37.000000000 -0800
[577dd2d]3973@@ -0,0 +1,4 @@
[d33f80c]3974+AppleïŒ 10
3975+BananaïŒ 5
3976+CitrusïŒ 20
3977+CherryïŒ 30
[577dd2d]3978diff -Naur coreutils-6.12.orig/tests/misc/mb1.X coreutils-6.12/tests/misc/mb1.X
3979--- coreutils-6.12.orig/tests/misc/mb1.X 1969-12-31 16:00:00.000000000 -0800
[d33f80c]3980+++ coreutils-6.12/tests/misc/mb1.X 2009-01-08 20:14:37.000000000 -0800
[577dd2d]3981@@ -0,0 +1,4 @@
[d33f80c]3982+BananaïŒ 5
3983+AppleïŒ 10
3984+CitrusïŒ 20
3985+CherryïŒ 30
[577dd2d]3986diff -Naur coreutils-6.12.orig/tests/misc/mb2.I coreutils-6.12/tests/misc/mb2.I
3987--- coreutils-6.12.orig/tests/misc/mb2.I 1969-12-31 16:00:00.000000000 -0800
[d33f80c]3988+++ coreutils-6.12/tests/misc/mb2.I 2009-01-08 20:14:37.000000000 -0800
[577dd2d]3989@@ -0,0 +1,4 @@
[d33f80c]3990+AppleïŒ ïŒ¡ïŒ¡10ïŒ ïŒ 20
3991+BananaïŒ ïŒ¡ïŒ¡5ïŒ ïŒ 30
3992+CitrusïŒ ïŒ¡ïŒ¡20ïŒ ïŒ 5
3993+CherryïŒ ïŒ¡ïŒ¡30ïŒ ïŒ 10
[577dd2d]3994diff -Naur coreutils-6.12.orig/tests/misc/mb2.X coreutils-6.12/tests/misc/mb2.X
3995--- coreutils-6.12.orig/tests/misc/mb2.X 1969-12-31 16:00:00.000000000 -0800
[d33f80c]3996+++ coreutils-6.12/tests/misc/mb2.X 2009-01-08 20:14:37.000000000 -0800
[577dd2d]3997@@ -0,0 +1,4 @@
[d33f80c]3998+CitrusïŒ ïŒ¡ïŒ¡20ïŒ ïŒ 5
3999+CherryïŒ ïŒ¡ïŒ¡30ïŒ ïŒ 10
4000+AppleïŒ ïŒ¡ïŒ¡10ïŒ ïŒ 20
4001+BananaïŒ ïŒ¡ïŒ¡5ïŒ ïŒ 30
[577dd2d]4002diff -Naur coreutils-6.12.orig/tests/misc/sort-mb-tests coreutils-6.12/tests/misc/sort-mb-tests
4003--- coreutils-6.12.orig/tests/misc/sort-mb-tests 1969-12-31 16:00:00.000000000 -0800
[d33f80c]4004+++ coreutils-6.12/tests/misc/sort-mb-tests 2009-01-08 20:14:37.000000000 -0800
[577dd2d]4005@@ -0,0 +1,58 @@
4006+#! /bin/sh
4007+case $# in
4008+ 0) xx='../src/sort';;
4009+ *) xx="$1";;
4010+esac
4011+test "$VERBOSE" && echo=echo || echo=:
4012+$echo testing program: $xx
4013+errors=0
4014+test "$srcdir" || srcdir=.
4015+test "$VERBOSE" && $xx --version 2> /dev/null
4016+
4017+export LC_ALL=en_US.UTF-8
4018+locale -k LC_CTYPE 2>&1 | grep -q charmap.*UTF-8 || exit 77
4019+errors=0
4020+
[d33f80c]4021+$xx -t ïŒ -k2 -n misc/mb1.I > misc/mb1.O
[577dd2d]4022+code=$?
4023+if test $code != 0; then
4024+ $echo "Test mb1 failed: $xx return code $code differs from expected value 0" 1>&2
4025+ errors=`expr $errors + 1`
4026+else
4027+ cmp misc/mb1.O $srcdir/misc/mb1.X > /dev/null 2>&1
4028+ case $? in
4029+ 0) if test "$VERBOSE"; then $echo "passed mb1"; fi;;
4030+ 1) $echo "Test mb1 failed: files misc/mb1.O and $srcdir/misc/mb1.X differ" 1>&2
4031+ (diff -c misc/mb1.O $srcdir/misc/mb1.X) 2> /dev/null
4032+ errors=`expr $errors + 1`;;
4033+ 2) $echo "Test mb1 may have failed." 1>&2
4034+ $echo The command "cmp misc/mb1.O $srcdir/misc/mb1.X" failed. 1>&2
4035+ errors=`expr $errors + 1`;;
4036+ esac
4037+fi
4038+
[d33f80c]4039+$xx -t ïŒ -k4 -n misc/mb2.I > misc/mb2.O
[577dd2d]4040+code=$?
4041+if test $code != 0; then
4042+ $echo "Test mb2 failed: $xx return code $code differs from expected value 0" 1>&2
4043+ errors=`expr $errors + 1`
4044+else
4045+ cmp misc/mb2.O $srcdir/misc/mb2.X > /dev/null 2>&1
4046+ case $? in
4047+ 0) if test "$VERBOSE"; then $echo "passed mb2"; fi;;
4048+ 1) $echo "Test mb2 failed: files misc/mb2.O and $srcdir/misc/mb2.X differ" 1>&2
4049+ (diff -c misc/mb2.O $srcdir/misc/mb2.X) 2> /dev/null
4050+ errors=`expr $errors + 1`;;
4051+ 2) $echo "Test mb2 may have failed." 1>&2
4052+ $echo The command "cmp misc/mb2.O $srcdir/misc/mb2.X" failed. 1>&2
4053+ errors=`expr $errors + 1`;;
4054+ esac
4055+fi
4056+
4057+if test $errors = 0; then
4058+ $echo Passed all 113 tests. 1>&2
4059+else
4060+ $echo Failed $errors tests. 1>&2
4061+fi
4062+test $errors = 0 || errors=1
4063+exit $errors
4064
4065
Note: See TracBrowser for help on using the repository browser.