diff --git a/wikiedits/__init__.py b/wikiedits/__init__.py index 7190ac1..9f38d8d 100644 --- a/wikiedits/__init__.py +++ b/wikiedits/__init__.py @@ -4,4 +4,6 @@ 'english', 'polish', 'german', + 'slovak', + 'ukrainian' ] diff --git a/wikiedits/edit_filter.py b/wikiedits/edit_filter.py index 4e89235..173516d 100644 --- a/wikiedits/edit_filter.py +++ b/wikiedits/edit_filter.py @@ -20,7 +20,10 @@ def __init__(self, edit_ratio=0.3, min_chars=10): - self.segmenter = nltk.data.load('tokenizers/punkt/%s.pickle' % lang) + if lang=='slovak': + self.segmenter = nltk.data.load('tokenizers/punkt/czech.pickle') + else: + self.segmenter = nltk.data.load('tokenizers/punkt/%s.pickle' % lang) self.LEVENSHTEIN_RATIO_LOG_BASE = 20 diff --git a/wikiedits/wiki/__init__.py b/wikiedits/wiki/__init__.py index 641d69b..d17bcac 100644 --- a/wikiedits/wiki/__init__.py +++ b/wikiedits/wiki/__init__.py @@ -1,7 +1,10 @@ # -*- coding: utf-8 -*- VANDALISM_REGEXES = { - 'english': "vandal|stupid|revert", - 'polish': "anulowan|wycofan|cofnię|cofnie|przywróc|przywroc|revert|rewert", - 'german': "revert|vandal|rückgängig", + 'english': u"vandal|stupid|revert", + 'polish': u"anulowan|wycofan|cofnię|cofnie|przywróc|przywroc|revert|rewert", + 'german': u"revert|vandal|rückgängig", + 'slovak': u"revert|vráten|vraten|vandal|obnov|revízia|revizia", + 'ukrainian': u"відкинут|відміна редагування|скасування редагування|вандал", + } diff --git a/wikiedits/wiki/revision_iterator.py b/wikiedits/wiki/revision_iterator.py index 2b2b3ee..86857c7 100644 --- a/wikiedits/wiki/revision_iterator.py +++ b/wikiedits/wiki/revision_iterator.py @@ -6,6 +6,7 @@ import WikiExtractor import re + HTML_TAG_REGEX = r'<[^>]{1,20}?>' class RevisionIterator(object): @@ -48,6 +49,8 @@ def clean_markups(self, text): return "\n".join(clean_html) if len(clean_html) > 0 else "" def __is_revert_vandalism(self, comment): - if type(comment) is str: + if comment is not None: return bool(self.vandalism_regex.search(comment)) - return False + else: + return False +