diff --git a/tools/anntoconll.py b/tools/anntoconll.py index adcd334c3..2cd95a8b7 100644 --- a/tools/anntoconll.py +++ b/tools/anntoconll.py @@ -114,7 +114,7 @@ def attach_labels(labels, lines): # NERsuite tokenization: any alnum sequence is preserved as a single # token, while any non-alnum character is separated into a # single-character token. TODO: non-ASCII alnum. -TOKENIZATION_REGEX = re.compile(r'([0-9a-zA-Z]+|[^0-9a-zA-Z])') +TOKENIZATION_REGEX = re.compile(r'([0-9a-zA-ZäüöÄÜÖ]+|[^0-9a-zA-ZäüöÄÜÖ])') NEWLINE_TERM_REGEX = re.compile(r'(.*?\n)')