natural language processing nltk

对自然语言处理,Converting human expressions to characters that computers can recognize is a very difficult task.There are various expressions for the same meaning,语法,句型, Synonyms all affect the computer's judgment.nltk Modules provide some functionality,Important information can be grabbed from the document,然后分析.

installation

​​pip install nltk​​

tokenize(分割)

登录后复制

      
      

       
       import 
       
       nltk
       
       

       
       from 
       
       nltk 
       
       import 
       
       word_tokenize
       
       

       
       

       
       nltk.
       
       download(
       
       'punkt')
       
       

       
       text 
       
       = 
       
       'Rock is a smart guy.'
       
       

       
       words 
       
       = 
       
       word_tokenize(
       
       text)
       
       
[
       
       'Rock', 
       
       'is', 
       
       'a', 
       
       'smart', 
       
       'guy', 
       
       '.']
      
      
      
      

       
       
1.
       
       
2.
       
       
3.
       
       
4.
       
       
5.
       
       
6.
       
       
7.
      
      

过滤标点符号

登录后复制

      
      

       
       [
       
       word 
       
       for 
       
       word 
       
       in 
       
       words 
       
       if 
       
       word.
       
       isalpha()]
      
      
      
      

       
       
1.
      
      

Filter common but meaningless words

登录后复制

      
      

       
       from 
       
       nltk.
       
       corpus 
       
       import 
       
       stopwords
       
       

       
       stops 
       
       = 
       
       set(
       
       stopwords.
       
       words(
       
       'english'))
      
      
      
      

       
       
1.
       
       
2.
      
      

登录后复制

      
      

       
       stops 
       
       = 
       
       set(
       
       stopwords.
       
       words(
       
       'chinese'))
      
      
      
      

       
       
1.
      
      

登录后复制

      
      

       
       clean_words 
       
       = [
       
       word 
       
       for 
       
       word 
       
       in 
       
       words 
       
       if 
       
       word 
       
       not 
       
       in 
       
       stops]
       
       
[
       
       'rock', 
       
       'smart', 
       
       'guy']
      
      
      
      

       
       
1.
       
       
2.
      
      

convert to stem

登录后复制

      
      

       
       text 
       
       = 
       
       'Rock is a smart guy. he likes playing cards.'
       
       

       
       words 
       
       = 
       
       word_tokenize(
       
       text)
       
       

       
       clean_words 
       
       = [
       
       word 
       
       for 
       
       word 
       
       in 
       
       words 
       
       if 
       
       word.
       
       isalpha() 
       
       and 
       
       word 
       
       not 
       
       in 
       
       stops]
       
       

       
       

       
       from 
       
       nltk.
       
       stem 
       
       import 
       
       PorterStemmer
       
       

       
       porter 
       
       = 
       
       PorterStemmer()
       
       

       
       stem_words 
       
       = [
       
       porter.
       
       stem(
       
       word) 
       
       for 
       
       word 
       
       in 
       
       clean_words]
       
       
[
       
       'rock', 
       
       'smart', 
       
       'guy', 
       
       'like', 
       
       'play', 
       
       'card']
      
      
      
      

       
       
1.
       
       
2.
       
       
3.
       
       
4.
       
       
5.
       
       
6.
       
       
7.
       
       
8.