【机器学习】朴素贝叶斯对文本分类--对人名国别分类

朴素贝叶斯

1. 基于先验概率、条件概率得到联合概率，进而得到后验概率；
2. 满足所有特征独立性假设，而且同等重要；
3. 是一种生成式模型；
4. 可用于小数据集的多分类问题。

代码

import re

from sklearn.feature_extraction.text import CountVectorizer # 文本表示，一直处理到n-gram
from sklearn.model_selection import train_test_split # 划分数据集
from sklearn.naive_bayes import MultinomialNB # 引入贝叶斯公式


class LanguageDetector():

    def __init__(self, classifier=MultinomialNB()):
        self.classifier = classifier
        # ngram_range=(1,2) N-GRAM为1，2
        self.vectorizer = CountVectorizer(ngram_range=(1,2), max_features=1000, preprocessor=self._remove_noise)
    
    # 去噪
    def _remove_noise(self, document):
        noise_pattern = re.compile("|".join(["http\S+", "\@\w+", "\#\w+"]))
        clean_text = re.sub(noise_pattern, "", document)
        return clean_text
    
    # 特征提取
    def features(self, X):
        return self.vectorizer.transform(X)

    # 训练
    def fit(self, X, y):
        self.vectorizer.fit(X)
        self.classifier.fit(self.features(X), y)
   
    # 预测
    def predict(self, x):
        return self.classifier.predict(self.features([x]))
    
    # 准确率
    def score(self, X, y):
        return self.classifier.score(self.features(X), y)

if __name__ == "__main__":
    in_f = open('data.csv')
    lines = in_f.readlines()
    in_f.close()
    dataset = [(line.strip()[:-3], line.strip()[-2:]) for line in lines]
    x, y = zip(*dataset)
    x_train, x_test, y_train, y_test = train_test_split(x, y, random_state=1)

    language_detector = LanguageDetector()
    language_detector.fit(x_train, y_train)
    print(language_detector.predict('This is an English sentence')) # 测试语句
    print(language_detector.score(x_test, y_test))

运行结果

['en']
0.9779444199382443

本文的数据展示