Python NLTK库全面解析及代码示例(NLP核心库)

pip install nltk
# 下载nltk数据包（首次使用时需运行）
import nltk
nltk.download('punkt')      # 分词模型
nltk.download('averaged_perceptron_tagger')  # 词性标注模型
nltk.download('wordnet')    # 词汇数据库
nltk.download('stopwords')  # 停用词

from nltk.tokenize import sent_tokenize
text = "hello world! this is nltk. let's learn nlp."
sentences = sent_tokenize(text)  # ['hello world!', 'this is nltk.', "let's learn nlp."]

from nltk.tokenize import word_tokenize
words = word_tokenize("hello, world!")  # ['hello', ',', 'world', '!']

from nltk import pos_tag
tokens = word_tokenize("i love nlp.")
tags = pos_tag(tokens)  # [('i', 'prp'), ('love', 'vbp'), ('nlp', 'nnp'), ('.', '.')]

from nltk.stem import porterstemmer
stemmer = porterstemmer()
stemmed = stemmer.stem("running")  # 'run'

from nltk.stem import wordnetlemmatizer
lemmatizer = wordnetlemmatizer()
lemma = lemmatizer.lemmatize("better", pos='a')  # 'good'（需指定词性）

from nltk import regexpparser
grammar = r"np: {<dt>?<jj>*<nn>}"  # 定义名词短语规则
parser = regexpparser(grammar)
tree = parser.parse(tags)  # 生成语法树
tree.draw()  # 可视化树结构

from nltk import ne_chunk
text = "apple is headquartered in cupertino."
tags = pos_tag(word_tokenize(text))
entities = ne_chunk(tags)
# 输出： (gpe apple/nnp) is/vbz headquartered/vbn in/in (gpe cupertino/nnp)

from nltk.corpus import stopwords
stop_words = set(stopwords.words('english'))
filtered_words = [w for w in word_tokenize(text) if w.lower() not in stop_words]

from nltk import edit_distance
distance = edit_distance("apple", "appel")  # 2

from nltk.sentiment import sentimentintensityanalyzer
sia = sentimentintensityanalyzer()
score = sia.polarity_scores("i love this movie!")  # {'compound': 0.8316, 'pos': 0.624, ...}

from nltk.corpus import gutenberg
print(gutenberg.fileids())  # 查看内置语料库
emma = gutenberg.words('austen-emma.txt')  # 加载文本

from nltk.text import textcollection
corpus = textcollection([text1, text2, text3])
tfidf = corpus.tf_idf(word, text)

from nltk.util import ngrams
bigrams = list(ngrams(tokens, 2))  # 生成二元组

# 示例：使用 jieba 分词
import jieba
words = jieba.lcut("自然语言处理很有趣")  # ['自然语言', '处理', '很', '有趣']

from nltk.corpus import movie_reviews
import random
# 加载数据（正面和负面评论）
documents = [(list(movie_reviews.words(fileid)), category)
             for category in movie_reviews.categories()
             for fileid in movie_reviews.fileids(category)]
random.shuffle(documents)  # 打乱顺序
# 提取所有单词并构建特征集
all_words = [word.lower() for word in movie_reviews.words()]
all_words = nltk.freqdist(all_words)
word_features = list(all_words.keys())[:3000]  # 选择前3000个高频词作为特征
# 定义特征提取函数
def document_features(document):
    document_words = set(document)
    features = {}
    for word in word_features:
        features[f'contains({word})'] = (word in document_words)
    return features
featuresets = [(document_features(doc), category) for (doc, category) in documents]
train_set, test_set = featuresets[100:], featuresets[:100]  # 划分训练集和测试集

classifier = nltk.naivebayesclassifier.train(train_set)
# 评估模型
accuracy = nltk.classify.accuracy(classifier, test_set)
print(f"accuracy: {accuracy:.2f}")  # 输出约 0.7-0.8
# 查看重要特征
classifier.show_most_informative_features(10)
# 示例输出：
# most informative features
#     contains(outstanding) = true              pos : neg    =     12.4 : 1.0
#       contains(seagal) = true              neg : pos    =     10.6 : 1.0

from nltk.corpus import plaintextcorpusreader
corpus_root = './my_corpus'  # 本地文件夹路径
file_pattern = r'.*\.txt'    # 匹配所有txt文件
my_corpus = plaintextcorpusreader(corpus_root, file_pattern)
# 访问语料库内容
print(my_corpus.fileids())          # 查看文件列表
print(my_corpus.words('doc1.txt'))  # 获取特定文档的单词

from nltk.probability import freqdist
import matplotlib.pyplot as plt
custom_text = nltk.text(my_corpus.words())
fdist = freqdist(custom_text)
# 绘制高频词分布
plt.figure(figsize=(12,5))
fdist.plot(30, cumulative=false)
plt.show()
# 查找特定词的上下文
custom_text.concordance("人工智能", width=100, lines=10)

from functools import lru_cache
from nltk.stem import wordnetlemmatizer
lemmatizer = wordnetlemmatizer()
@lru_cache(maxsize=10000)  # 缓存最近10000次调用
def cached_lemmatize(word, pos='n'):
    return lemmatizer.lemmatize(word, pos)
# 使用缓存版本处理大规模文本
lemmas = [cached_lemmatize(word) for word in huge_word_list]

from joblib import parallel, delayed
from nltk.tokenize import word_tokenize
# 并行分词加速
texts = [...]  # 大规模文本列表
results = parallel(n_jobs=4)(delayed(word_tokenize)(text) for text in texts)

from nltk.corpus import stopwords
from nltk.stem import wordnetlemmatizer
from gensim import models, corpora
# 预处理
stop_words = stopwords.words('english')
lemmatizer = wordnetlemmatizer()
processed_docs = [
    [lemmatizer.lemmatize(word) for word in doc.lower().split() 
     if word not in stop_words and word.isalpha()]
    for doc in text_corpus
]
# 创建词典和文档-词矩阵
dictionary = corpora.dictionary(processed_docs)
doc_term_matrix = [dictionary.doc2bow(doc) for doc in processed_docs]
# 训练lda模型
lda_model = models.ldamodel(
    doc_term_matrix,
    num_topics=5,
    id2word=dictionary,
    passes=10
)
# 查看主题
print(lda_model.print_topics())

import networkx as nx
from nltk import bigrams
# 生成共现网络
cooc_network = nx.graph()
for doc in documents:
    doc_bigrams = list(bigrams(doc))
    for (w1, w2) in doc_bigrams:
        if cooc_network.has_edge(w1, w2):
            cooc_network[w1][w2]['weight'] += 1
        else:
            cooc_network.add_edge(w1, w2, weight=1)
# 可视化重要连接
plt.figure(figsize=(15,10))
pos = nx.spring_layout(cooc_network)
nx.draw_networkx_nodes(cooc_network, pos, node_size=50)
nx.draw_networkx_edges(cooc_network, pos, alpha=0.2)
nx.draw_networkx_labels(cooc_network, pos, font_size=8)
plt.show()

# 指定下载镜像源
import nltk
nltk.download('punkt', download_dir='/path/to/nltk_data', 
             quiet=true, halt_on_error=false)

# 使用生成器处理大文件
def stream_docs(path):
    with open(path, 'r', encoding='utf-8') as f:
        for line in f:
            yield line.strip()
# 分块处理
for chunk in nltk.chunk(stream_docs('big_file.txt'), 10000):
    process(chunk)

from nltk import data
data.path.append('/path/to/unicode/corpora')  # 添加自定义编码语料路径

import pandas as pd
from nltk.sentiment import sentimentintensityanalyzer
df = pd.read_csv('reviews.csv')
sia = sentimentintensityanalyzer()
# 为每条评论添加情感分数
df['sentiment'] = df['text'].apply(
    lambda x: sia.polarity_scores(x)['compound']
)
# 分析结果分布
df['sentiment'].hist(bins=20)

from sklearn.pipeline import pipeline
from sklearn.feature_extraction.text import tfidftransformer
from sklearn.naive_bayes import multinomialnb
from nltk.tokenize import treebankwordtokenizer
# 自定义分词器
nltk_tokenizer = treebankwordtokenizer().tokenize
pipeline = pipeline([
    ('vect', countvectorizer(tokenizer=nltk_tokenizer)),
    ('tfidf', tfidftransformer()),
    ('clf', multinomialnb()),
])
pipeline.fit(x_train, y_train)

from nltk.sentiment.util import mark_negation
from nltk import freqdist
# 自定义情感词典
positive_words = {'excellent', 'brilliant', 'superb'}
negative_words = {'terrible', 'awful', 'horrible'}
def custom_sentiment_analyzer(text):
    tokens = mark_negation(word_tokenize(text.lower()))  # 处理否定词
    score = 0
    for word in tokens:
        if word in positive_words:
            score += 1
        elif word in negative_words:
            score -= 1
        elif word.endswith("_neg"):  # 处理否定修饰
            base_word = word[:-4]
            if base_word in positive_words:
                score -= 1
            elif base_word in negative_words:
                score += 1
    return score
# 测试示例
text = "the service was not excellent but the food was superb."
print(custom_sentiment_analyzer(text))  # 输出：0 (因为"excellent_neg"扣分，但"superb"加分)

from sklearn.svm import svc
from nltk.classify.scikitlearn import sklearnclassifier
from nltk.sentiment import sentimentanalyzer
# 使用scikit-learn的svm算法
sentiment_analyzer = sentimentanalyzer()
svm_classifier = sklearnclassifier(svc(kernel='linear'))
# 添加自定义特征
all_words = [word.lower() for word in movie_reviews.words()]
unigram_feats = sentiment_analyzer.unigram_word_feats(all_words, min_freq=10)
sentiment_analyzer.add_feat_extractor(
    nltk.sentiment.util.extract_unigram_feats, unigrams=unigram_feats[:2000]
)
# 转换特征格式
training_set = sentiment_analyzer.apply_features(movie_reviews.sents(categories='pos')[:500] + \
               sentiment_analyzer.apply_features(movie_reviews.sents(categories='neg')[:500]
# 训练并评估模型
svm_classifier.train(training_set)
accuracy = nltk.classify.accuracy(svm_classifier, training_set)
print(f"svm分类器准确率: {accuracy:.2%}")

import pandas as pd
from nltk.sentiment import sentimentintensityanalyzer
# 加载带时间戳的新闻数据
news_data = [
    ("2023-01-01", "company a launched revolutionary new product"),
    ("2023-02-15", "company a faces regulatory investigation"),
    ("2023-03-30", "company a reports record profits")
]
df = pd.dataframe(news_data, columns=['date', 'text'])
df['date'] = pd.to_datetime(df['date'])
# 计算每日情感分数
sia = sentimentintensityanalyzer()
df['sentiment'] = df['text'].apply(lambda x: sia.polarity_scores(x)['compound'])
# 可视化趋势
df.set_index('date')['sentiment'].plot(
    title='公司a新闻情感趋势分析',
    ylabel='情感分数',
    figsize=(10,6),
    grid=true
)

from nltk.tokenize import regexptokenizer
# 自定义多语言分词器
multilingual_tokenizer = regexptokenizer(r'''\w+@\w+\.\w+ |  # 保留电子邮件
                                           [a-za-z]+(?:'\w+)? |  # 英文单词
                                           [\u4e00-\u9fff]+ |  # 中文字符
                                           \d+''')  # 数字
text = "hello 你好！contact me at example@email.com 或拨打400-123456"
tokens = multilingual_tokenizer.tokenize(text)
# 输出：['hello', '你好', 'contact', 'me', 'at', 'example@email.com', '或', '拨打', '400', '123456']

from gensim.models import keyedvectors
from nltk.corpus import wordnet as wn
# 加载预训练跨语言词向量（需提前下载）
# 示例使用facebook的muse词向量
zh_model = keyedvectors.load_word2vec_format('wiki.multi.zh.vec')
en_model = keyedvectors.load_word2vec_format('wiki.multi.en.vec')
def cross_lingual_similarity(word_en, word_zh):
    try:
        return en_model.similarity(word_en, zh_model[word_zh])
    except keyerror:
        return none
print(f"apple 与 苹果 的相似度: {cross_lingual_similarity('apple', '苹果'):.2f}")
# 输出：约0.65-0.75

from nltk.metrics import confusionmatrix, precision, recall, f_measure
ref_set = ['pos', 'neg', 'pos', 'pos']
test_set = ['pos', 'pos', 'neg', 'pos']
# 创建混淆矩阵
cm = confusionmatrix(ref_set, test_set)
print(cm)
# 计算指标
print(f"precision: {precision(set(ref_set), set(test_set)):.2f}")
print(f"recall: {recall(set(ref_set), set(test_set)):.2f}")
print(f"f1-score: {f_measure(set(ref_set), set(test_set)):.2f}")

from nltk.translate.bleu_score import sentence_bleu
reference = [['this', 'is', 'a', 'test']]
candidate = ['this', 'is', 'a', 'test']
print(f"bleu-4 score: {sentence_bleu(reference, candidate):.2f}")
# 输出：1.0
candidate = ['this', 'is', 'test']
print(f"bleu-4 score: {sentence_bleu(reference, candidate):.2f}")
# 输出：约0.59

from tweepy import stream
from nltk import freqdist
import json
class tweetanalyzer(stream):
    def __init__(self, consumer_key, consumer_secret):
        super().__init__(consumer_key, consumer_secret)
        self.keywords_fd = freqdist()
    def on_data(self, data):
        tweet = json.loads(data)
        text = tweet.get('text', '')
        tokens = [word.lower() for word in word_tokenize(text) 
                 if word.isalpha() and len(word) > 2]
        for word in tokens:
            self.keywords_fd[word] += 1
        return true
# 使用示例（需申请twitter api密钥）
analyzer = tweetanalyzer('your_key', 'your_secret')
analyzer.filter(track=['python', 'ai'], languages=['en'])

from dash import dash, dcc, html
import plotly.express as px
from collections import deque
# 创建实时更新队列
sentiment_history = deque(maxlen=100)
timestamps = deque(maxlen=100)
app = dash(__name__)
app.layout = html.div([
    dcc.graph(id='live-graph'),
    dcc.interval(id='interval', interval=5000)
])
@app.callback(output('live-graph', 'figure'),
              input('interval', 'n_intervals'))
def update_graph(n):
    # 这里添加实时获取数据的逻辑
    return px.line(x=list(timestamps), 
                  y=list(sentiment_history),
                  title="实时情感趋势")
if __name__ == '__main__':
    app.run_server(debug=true)

from nltk.tag import unigramtagger
from nltk.corpus import treebank
# 训练自定义标注器
train_sents = treebank.tagged_sents()[:3000]
tagger = unigramtagger(train_sents)
# 查看内部概率分布
word = 'run'
prob_dist = tagger._model[word]
print(f"{word} 的标注概率分布:")
for tag, prob in prob_dist.items():
    print(f"{tag}: {prob:.2%}")
# 输出示例：
# vb: 45.32%
# nn: 32.15%
# ...其他词性概率

from nltk.parse import recursivedescentparser
from nltk.grammar import cfg
# 定义简单语法
grammar = cfg.fromstring("""
    s -> np vp
    vp -> v np | v np pp
    pp -> p np
    np -> det n | det n pp
    det -> 'a' | 'the'
    n -> 'man' | 'park' | 'dog'
    v -> 'saw' | 'walked'
    p -> 'in' | 'with'
""")
# 创建解析器
parser = recursivedescentparser(grammar)
sentence = "the man saw a dog in the park".split()
for tree in parser.parse(sentence):
    tree.pretty_print()

from ipython.display import display
import ipywidgets as widgets
# 创建交互式词性标注器
text_input = widgets.textarea(value='enter text here')
output = widgets.output()
def tag_text(b):
    with output:
        output.clear_output()
        text = text_input.value
        tokens = word_tokenize(text)
        tags = pos_tag(tokens)
        print("标注结果:")
        for word, tag in tags:
            print(f"{word:15}{tag}")
button = widgets.button(description="标注文本")
button.on_click(tag_text)
display(widgets.vbox([text_input, button, output]))

from nltk import ngrams
from nltk.corpus import brown
# 构建语言模型
brown_ngrams = list(ngrams(brown.words(), 3))
freq_dist = freqdist(brown_ngrams)
def detect_errors(sentence):
    tokens = word_tokenize(sentence)
    trigrams = list(ngrams(tokens, 3))
    for i, trigram in enumerate(trigrams):
        if freq_dist[trigram] < 5:  # 出现频率过低的组合
            print(f"潜在错误位置 {i+1}-{i+3}: {' '.join(trigram)}")
detect_errors("he don't knows the answer.")
# 输出：潜在错误位置 2-4: don't knows the

from transformers import pipeline
from nltk import word_tokenize
# 结合huggingface模型
class advancednltkanalyzer:
    def __init__(self):
        self.sentiment = pipeline('sentiment-analysis')
        self.ner = pipeline('ner')
    def enhanced_analysis(self, text):
        return {
            'sentiment': self.sentiment(text),
            'entities': self.ner(text),
            'tokens': word_tokenize(text)
        }
# 使用示例
analyzer = advancednltkanalyzer()
result = analyzer.enhanced_analysis("apple inc. is looking to buy u.k. startup for $1 billion")
print(result['entities'])  # 识别组织、地点、货币等实体

from numba import jit
from nltk import edit_distance
# 使用gpu加速编辑距离计算
@jit(nopython=true, parallel=true)
def gpu_edit_distance(s1, s2):
    # 实现动态规划算法
    m, n = len(s1), len(s2)
    dp = [[0]*(n+1) for _ in range(m+1)]
    for i in range(m+1): dp[i][0] = i
    for j in range(n+1): dp[0][j] = j
    for i in range(1, m+1):
        for j in range(1, n+1):
            cost = 0 if s1[i-1] == s2[j-1] else 1
            dp[i][j] = min(dp[i-1][j]+1, 
                          dp[i][j-1]+1, 
                          dp[i-1][j-1]+cost)
    return dp[m][n]
print(gpu_edit_distance("kitten", "sitting"))  # 输出：3