#!/usr/bin/env python
# -*- coding: utf-8 -*-
import re

import langid

from coding_transformer import CodingTransformer


class LanguageFilter(object):
    def __init__(self):
        self._identifier = langid.langid.LanguageIdentifier.from_modelstring(
            langid.langid.model, norm_probs=True)
        self.en_chars_range_list = [
            range(0x0000, 0x007f),  # C0 Controls and Basic Latin
            range(0x0080, 0x00ff),  # C1 Controls and Latin-1 Supplement
            range(0x0100, 0x017f),  # Latin Extended-A
            range(0x0180, 0x024f),  # Latin Extended-B
            range(0x0250, 0x02af),  # IPA Extension
            range(0x02b0, 0x02ff),  # Spacing Modifier Letters
        ]


    def parse_english_chars(self, text):
        text = CodingTransformer.decode(text)
        regular_expression = "[" + "|".join(
            [unichr(range_field[0]) + "-" + unichr(range_field[len(
                range_field) - 1]) for range_field in (
                 self.en_chars_range_list)]) + "]+"
        return re.findall(regular_expression, text)


    def _filter_en(self, text):
        en_texts = self.parse_english_chars(text)
        filtered_text = []
        for text in en_texts:
            lang, prob = self._identifier.classify(text.lower())
            if lang == "en":
                filtered_text.append(text)
        return filtered_text

    # lang_set is left for further multilingual extension
    def filter(self, text, lang_set=None):
        return self._filter_en(text)


if __name__ == '__main__':
    print(LanguageFilter().filter("test测试 Tag"))
