--- /dev/null
+/* Word breaks in UTF-8/UTF-16/UTF-32 strings.
+ Copyright (C) 2009 Free Software Foundation, Inc.
+ Written by Bruno Haible <bruno@clisp.org>, 2009.
+
+ This program is free software: you can redistribute it and/or modify it
+ under the terms of the GNU Lesser General Public License as published
+ by the Free Software Foundation; either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>. */
+
+void
+FUNC (const UNIT *s, size_t n, char *p)
+{
+ if (n > 0)
+ {
+ const UNIT *s_end = s + n;
+
+ /* Word break property of the last character.
+ -1 at the very beginning of the string. */
+ int last_char_prop = -1;
+
+ /* Format and Extend characters are ignored; this means, the mostly used
+ unit is the complex character (= character with subsequent ignored
+ characters).
+ Word break property of the last complex character.
+ -1 at the very beginning of the string. */
+ int last_compchar_prop = -1;
+ char *last_compchar_ptr = NULL;
+
+ /* For recognizing rules involving 3 complex characters:
+ Word break property of the second-to-last complex character.
+ -1 at the very beginning of the string. */
+ int secondlast_compchar_prop = -1;
+
+ /* Don't break inside multibyte characters. */
+ memset (p, 0, n);
+
+ while (s < s_end)
+ {
+ ucs4_t uc;
+ int count = U_MBTOUC_UNSAFE (&uc, s, s_end - s);
+ int prop = uc_wordbreak_property (uc);
+
+ /* No break at the start of the string. */
+ if (last_char_prop >= 0)
+ {
+ /* No break between CR and LF. */
+ if (last_char_prop == WBP_CR && prop == WBP_LF)
+ /* *p = 0 */;
+ /* Break before and after newlines. */
+ else if (last_char_prop >= WBP_NEWLINE
+ /* same as:
+ last_char_prop == WBP_CR
+ || last_char_prop == WBP_LF
+ || last_char_prop == WBP_NEWLINE */
+ || prop >= WBP_NEWLINE
+ /* same as:
+ prop == WBP_CR
+ || prop == WBP_LF
+ || prop == WBP_NEWLINE */)
+ *p = 1;
+ /* Ignore Format and Extend characters. */
+ else if (!(prop == WBP_EXTEND || prop == WBP_FORMAT))
+ {
+ /* No break in these situations (see UAX #29):
+
+ secondlast last current
+
+ ALetter (MidLetter | MidNumLet) × ALetter (WB7)
+ ALetter × (MidLetter | MidNumLet) ALetter (WB6)
+ Numeric (MidNum | MidNumLet) × Numeric (WB11)
+ Numeric × (MidNum | MidNumLet) Numeric (WB12)
+ ALetter × ALetter (WB5)
+ ALetter × Numeric (WB9)
+ Numeric × ALetter (WB10)
+ Numeric × Numeric (WB8)
+ Katakana × Katakana (WB13)
+ (ALetter | Numeric | Katakana) × ExtendNumLet (WB13a)
+ ExtendNumLet × ExtendNumLet (WB13a)
+ ExtendNumLet × (ALetter | Numeric | Katakana) (WB13b)
+ */
+ /* No break across certain punctuation. Also, disable word
+ breaks that were recognized earlier (due to lookahead of
+ only one complex character). */
+ if ((prop == WBP_ALETTER
+ && (last_compchar_prop == WBP_MIDLETTER
+ || last_compchar_prop == WBP_MIDNUMLET)
+ && secondlast_compchar_prop == WBP_ALETTER)
+ || (prop == WBP_NUMERIC
+ && (last_compchar_prop == WBP_MIDNUM
+ || last_compchar_prop == WBP_MIDNUMLET)
+ && secondlast_compchar_prop == WBP_NUMERIC))
+ {
+ *last_compchar_ptr = 0;
+ /* *p = 0; */
+ }
+ else
+ {
+ /* Perform a single table lookup. */
+ if (uniwbrk_table[last_compchar_prop][prop])
+ *p = 1;
+ /* else *p = 0; */
+ }
+ }
+ }
+
+ last_char_prop = prop;
+ /* Ignore Format and Extend characters, except at the start of the string. */
+ if (last_compchar_prop < 0 || !(prop == WBP_EXTEND || prop == WBP_FORMAT))
+ {
+ secondlast_compchar_prop = last_compchar_prop;
+ last_compchar_prop = prop;
+ last_compchar_ptr = p;
+ }
+
+ s += count;
+ p += count;
+ }
+ }
+}
--- /dev/null
+/* Word breaks in UTF-8 strings.
+ Copyright (C) 2009 Free Software Foundation, Inc.
+ Written by Bruno Haible <bruno@clisp.org>, 2009.
+
+ This program is free software: you can redistribute it and/or modify it
+ under the terms of the GNU Lesser General Public License as published
+ by the Free Software Foundation; either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>. */
+
+#include <config.h>
+
+/* Specification. */
+#include "uniwbrk.h"
+
+#include <string.h>
+
+#include "unistr.h"
+#include "uniwbrk/wbrktable.h"
+
+#define FUNC u8_wordbreaks
+#define UNIT uint8_t
+#define U_MBTOUC_UNSAFE u8_mbtouc_unsafe
+#include "u-wordbreaks.h"
+
+
+#ifdef TEST
+
+#include <stdio.h>
+#include <stdlib.h>
+
+/* Read the contents of an input stream, and return it, terminated with a NUL
+ byte. */
+char *
+read_file (FILE *stream)
+{
+#define BUFSIZE 4096
+ char *buf = NULL;
+ int alloc = 0;
+ int size = 0;
+ int count;
+
+ while (! feof (stream))
+ {
+ if (size + BUFSIZE > alloc)
+ {
+ alloc = alloc + alloc / 2;
+ if (alloc < size + BUFSIZE)
+ alloc = size + BUFSIZE;
+ buf = realloc (buf, alloc);
+ if (buf == NULL)
+ {
+ fprintf (stderr, "out of memory\n");
+ exit (1);
+ }
+ }
+ count = fread (buf + size, 1, BUFSIZE, stream);
+ if (count == 0)
+ {
+ if (ferror (stream))
+ {
+ perror ("fread");
+ exit (1);
+ }
+ }
+ else
+ size += count;
+ }
+ buf = realloc (buf, size + 1);
+ if (buf == NULL)
+ {
+ fprintf (stderr, "out of memory\n");
+ exit (1);
+ }
+ buf[size] = '\0';
+ return buf;
+#undef BUFSIZE
+}
+
+int
+main (int argc, char * argv[])
+{
+ if (argc == 1)
+ {
+ /* Display all the word breaks in the input string. */
+ char *input = read_file (stdin);
+ int length = strlen (input);
+ char *breaks = malloc (length);
+ int i;
+
+ u8_wordbreaks ((uint8_t *) input, length, breaks);
+
+ for (i = 0; i < length; i++)
+ {
+ switch (breaks[i])
+ {
+ case 1:
+ /* U+2027 in UTF-8 encoding */
+ putc (0xe2, stdout); putc (0x80, stdout); putc (0xa7, stdout);
+ break;
+ case 0:
+ break;
+ default:
+ abort ();
+ }
+ putc (input[i], stdout);
+ }
+
+ free (breaks);
+
+ return 0;
+ }
+ else
+ return 1;
+}
+
+#endif /* TEST */
projects / gnulib.git / commitdiff