从基础到高级详解Python文本分词的完全指南

def basic_tokenize(text):
    """基础空格分词"""
    return text.split()

# 测试
text = "Python is an interpreted programming language"
tokens = basic_tokenize(text)
# ['Python', 'is', 'an', 'interpreted', 'programming', 'language']

import re

def regex_tokenize(text):
    """正则表达式分词"""
    # 匹配单词和基本标点
    pattern = r'\w+|[^\w\s]'
    return re.findall(pattern, text)

# 测试
text = "Hello, world! How are you?"
tokens = regex_tokenize(text)
# ['Hello', ',', 'world', '!', 'How', 'are', 'you', '?']

def advanced_regex_tokenize(text):
    """处理复杂文本的分词"""
    # 匹配：单词、连字符词、缩写、货币、表情符号
    pattern = r"""
        \d+\.\d+          | # 浮点数
        \d+,\d+           | # 千位分隔数字
        \d+               | # 整数
        \w+(?:-\w+)+      | # 连字符词
        [A-Z]+\.[A-Z]+\.?| # 缩写 (U.S.A.)
        \$\d+             | # 货币
        [\U0001F600-\U0001F64F] | # 表情符号
        \w+               | # 单词
        [^\w\s]             # 标点符号
    """
    return re.findall(pattern, text, re.VERBOSE | re.UNICODE)

# 测试
text = "I paid $99.99 for this item in the U.S.A. 😊"
tokens = advanced_regex_tokenize(text)
# ['I', 'paid', '$99.99', 'for', 'this', 'item', 'in', 'the', 'U.S.A.', '😊']

import nltk
nltk.download('punkt')

from nltk.tokenize import word_tokenize, sent_tokenize

# 句子分词
sentences = sent_tokenize("First sentence. Second sentence!")
# ['First sentence.', 'Second sentence!']

# 单词分词
tokens = word_tokenize("Python's nltk module is powerful!")
# ['Python', "'s", 'nltk', 'module', 'is', 'powerful', '!']

from nltk.tokenize import TweetTokenizer, MWETokenizer

# 推特分词器（处理表情符号和@提及）
tweet_tokenizer = TweetTokenizer()
tokens = tweet_tokenizer.tokenize("OMG! This is so cool 😍 #NLP @nlp_news")
# ['OMG', '!', 'This', 'is', 'so', 'cool', '😍', '#NLP', '@nlp_news']

# 多词表达分词器
mwe_tokenizer = MWETokenizer([('New', 'York'), ('machine', 'learning')])
tokens = mwe_tokenizer.tokenize("I live in New York and study machine learning".split())
# ['I', 'live', 'in', 'New_York', 'and', 'study', 'machine_learning']

import spacy

# 加载模型
nlp = spacy.load("en_core_web_sm")

# 分词处理
doc = nlp("Apple's stock price rose $5.45 to $126.33 in pre-market trading.")
tokens = [token.text for token in doc]
# ['Apple', "'s", 'stock', 'price', 'rose', '$', '5.45', 'to', '$', '126.33', 'in', 'pre', '-', 'market', 'trading', '.']

def analyze_tokens(doc):
    """获取分词详细信息"""
    token_data = []
    for token in doc:
        token_data.append({
            "text": token.text,
            "lemma": token.lemma_,
            "pos": token.pos_,
            "tag": token.tag_,
            "dep": token.dep_,
            "is_stop": token.is_stop,
            "is_alpha": token.is_alpha,
            "is_digit": token.is_digit
        })
    return token_data

# 测试
doc = nlp("The quick brown fox jumps over the lazy dog.")
token_info = analyze_tokens(doc)

from spacy.tokenizer import Tokenizer
from spacy.util import compile_prefix_regex, compile_suffix_regex

def create_custom_tokenizer(nlp):
    """创建自定义分词器"""
    # 自定义前缀规则（处理$等特殊前缀）
    prefixes = nlp.Defaults.prefixes + [r'\$']
    prefix_regex = compile_prefix_regex(prefixes)
    
    # 自定义后缀规则
    suffixes = nlp.Defaults.suffixes + [r'\%']
    suffix_regex = compile_suffix_regex(suffixes)
    
    # 自定义分词规则
    rules = nlp.Defaults.tokenizer_exceptions
    rules.update({
        "dont": [{"ORTH": "dont"}],  # 不分词
        "can't": [{"ORTH": "can"}, {"ORTH": "'t"}]  # 特殊分词
    })
    
    return Tokenizer(
        nlp.vocab,
        rules=rules,
        prefix_search=prefix_regex.search,
        suffix_search=suffix_regex.search,
        infix_finditer=nlp.Defaults.infix_finditer,
        token_match=nlp.Defaults.token_match
    )

# 使用自定义分词器
nlp = spacy.load("en_core_web_sm")
nlp.tokenizer = create_custom_tokenizer(nlp)

doc = nlp("I don't like $100% increases.")
tokens = [token.text for token in doc]
# ['I', 'dont', 'like', '$100%', 'increases', '.']

import jieba
import jieba.posseg as pseg

# 基础分词
text = "自然语言处理是人工智能的重要方向"
words = jieba.cut(text)
print("/".join(words))  # "自然语言/处理/是/人工智能/的/重要/方向"

# 全模式
words_full = jieba.cut(text, cut_all=True)
# "自然/自然语言/语言/处理/是/人工/人工智能/智能/重要/方向"

# 搜索引擎模式
words_search = jieba.cut_for_search(text)
# "自然/语言/自然语言/处理/是/人工/智能/人工智能/重要/方向"

# 添加自定义词典
jieba.add_word("自然语言处理")
jieba.add_word("人工智能")

# 加载自定义词典文件
jieba.load_userdict("custom_dict.txt")

# 词性标注
words = pseg.cut(text)
for word, flag in words:
    print(f"{word} ({flag})")
    
# 自然语言处理 (n)
# 是 (v)
# 人工智能 (n)
# 的 (uj)
# 重要 (a)
# 方向 (n)

def medical_tokenizer(text):
    """医学文本分词器"""
    # 加载基础模型
    nlp = spacy.load("en_core_sci_sm")
    
    # 添加医学词典
    with open("medical_terms.txt") as f:
        for term in f:
            nlp.tokenizer.add_special_case(term, [{"ORTH": term}])
    
    # 处理医学缩写
    abbreviations = {
        "CVD": "cardiovascular disease",
        "MI": "myocardial infarction"
    }
    
    # 处理医学复合词
    compound_rules = [
        ("blood", "pressure"),
        ("heart", "rate"),
        ("red", "blood", "cell")
    ]
    
    for terms in compound_rules:
        nlp.tokenizer.add_special_case(
            " ".join(terms),
            [{"ORTH": "_".join(terms)}]
        )
    
    return nlp(text)

# 使用
text = "Patient with CVD and high blood pressure. History of MI."
doc = medical_tokenizer(text)
tokens = [token.text for token in doc]
# ['Patient', 'with', 'CVD', 'and', 'high_blood_pressure', '.', 'History', 'of', 'MI', '.']

def legal_tokenizer(text):
    """法律文本分词器"""
    # 基础分词
    nlp = spacy.load("en_core_web_sm")
    
    # 添加法律术语
    legal_terms = [
        "force majeure",
        "prima facie",
        "pro bono",
        "voir dire"
    ]
    
    for term in legal_terms:
        nlp.tokenizer.add_special_case(
            term,
            [{"ORTH": term.replace(" ", "_")}]
        )
    
    # 处理法律引用
    pattern = r"(\d+)\s+(U\.S\.C\.|U\.S\.)\s+§\s+(\d+)"
    text = re.sub(pattern, r"\1_\2_§_\3", text)
    
    return nlp(text)

# 使用
text = "As per 42 U.S.C. § 1983, the plaintiff..."
doc = legal_tokenizer(text)
tokens = [token.text for token in doc]
# ['As', 'per', '42_U.S.C._§_1983', ',', 'the', 'plaintiff', '...']

class StreamTokenizer:
    """流式分词处理器"""
    def __init__(self, tokenizer_func, buffer_size=4096):
        self.tokenizer = tokenizer_func
        self.buffer = ""
        self.buffer_size = buffer_size
    
    def process(self, text_chunk):
        """处理文本块"""
        self.buffer += text_chunk
        tokens = []
        
        # 处理完整句子
        while '.' in self.buffer or '!' in self.buffer or '?' in self.buffer:
            # 查找最近的句子结束符
            end_pos = min(
                self.buffer.find('.'),
                self.buffer.find('!'),
                self.buffer.find('?')
            )
            if end_pos == -1:
                break
                
            # 提取句子
            sentence = self.buffer[:end_pos+1]
            self.buffer = self.buffer[end_pos+1:]
            
            # 分词
            tokens.extend(self.tokenizer(sentence))
        
        return tokens
    
    def finalize(self):
        """处理剩余文本"""
        if self.buffer:
            tokens = self.tokenizer(self.buffer)
            self.buffer = ""
            return tokens
        return []

# 使用示例
tokenizer = StreamTokenizer(word_tokenize)
with open("large_text.txt") as f:
    while chunk := f.read(1024):
        tokens = tokenizer.process(chunk)
        process_tokens(tokens)  # 处理分词结果
    
# 处理剩余内容
final_tokens = tokenizer.finalize()
process_tokens(final_tokens)

from flask import Flask, request, jsonify
import threading
import spacy

app = Flask(__name__)

# 预加载模型
nlp = spacy.load("en_core_web_sm")

# 请求队列
request_queue = []
result_dict = {}
lock = threading.Lock()

def tokenizer_worker():
    """分词工作线程"""
    while True:
        if request_queue:
            with lock:
                req_id, text = request_queue.pop(0)
            
            # 处理分词
            doc = nlp(text)
            tokens = [token.text for token in doc]
            
            with lock:
                result_dict[req_id] = tokens

# 启动工作线程
threading.Thread(target=tokenizer_worker, daemon=True).start()

@app.route('/tokenize', methods=['POST'])
def tokenize_endpoint():
    """分词API端点"""
    data = request.json
    text = data.get('text', '')
    req_id = id(text)
    
    with lock:
        request_queue.append((req_id, text))
    
    # 等待结果
    while req_id not in result_dict:
        time.sleep(0.01)
    
    with lock:
        tokens = result_dict.pop(req_id)
    
    return jsonify({"tokens": tokens})

# 启动服务
if __name__ == '__main__':
    app.run(threaded=True, port=5000)

from collections import Counter
from string import punctuation

def extract_keywords(text, top_n=10):
    """提取关键词"""
    # 分词
    doc = nlp(text)
    
    # 过滤停用词和标点
    words = [
        token.text.lower() 
        for token in doc 
        if not token.is_stop and not token.is_punct and token.is_alpha
    ]
    
    # 计算词频
    word_freq = Counter(words)
    return word_freq.most_common(top_n)

# 测试
text = "Python is an interpreted high-level programming language. Python is widely used in data science."
keywords = extract_keywords(text)
# [('python', 2), ('interpreted', 1), ('high', 1), ('level', 1), ('programming', 1), ('language', 1), ('widely', 1), ('used', 1), ('data', 1), ('science', 1)]

def preprocess_for_sentiment(text):
    """情感分析预处理"""
    # 分词
    doc = nlp(text)
    
    # 预处理步骤
    tokens = []
    for token in doc:
        # 小写化
        token_text = token.text.lower()
        
        # 移除停用词
        if token.is_stop:
            continue
        
        # 词形还原
        lemma = token.lemma_
        
        # 移除标点
        if lemma in punctuation:
            continue
        
        tokens.append(lemma)
    
    return tokens

# 测试
text = "I really love this product! It's amazing."
processed = preprocess_for_sentiment(text)
# ['really', 'love', 'product', 'amazing']

import timeit

text = "Natural language processing is a subfield of linguistics, computer science, and artificial intelligence."

# 测试函数
def test_regex():
    return regex_tokenize(text)

def test_nltk():
    return word_tokenize(text)

def test_spacy():
    doc = nlp(text)
    return [token.text for token in doc]

def test_jieba():
    return list(jieba.cut(text))

# 性能测试
methods = {
    "Regex": test_regex,
    "NLTK": test_nltk,
    "spaCy": test_spacy,
    "jieba": test_jieba
}

results = {}
for name, func in methods.items():
    time = timeit.timeit(func, number=1000)
    results[name] = time

print("1000次分词操作耗时:")
for name, time in sorted(results.items(), key=lambda x: x[1]):
    print(f"{name}: {time:.4f}秒")

def preprocess(text):
    # 小写化
    text = text.lower()
    # 移除特殊字符
    text = re.sub(r'[^\w\s]', '', text)
    # 分词
    return word_tokenize(text)

from nltk.corpus import stopwords
stop_words = set(stopwords.words('english'))

def remove_stopwords(tokens):
    return [t for t in tokens if t not in stop_words]

from nltk.stem import WordNetLemmatizer
lemmatizer = WordNetLemmatizer()

def lemmatize(tokens):
    return [lemmatizer.lemmatize(t) for t in tokens]

# 预加载模型
nlp = spacy.load("en_core_web_sm")

# 批量处理
texts = ["text1", "text2", "text3"]
docs = list(nlp.pipe(texts))

# 添加领域术语
nlp.tokenizer.add_special_case("machine_learning", [{"ORTH": "machine_learning"}])

try:
    tokens = tokenize(text)
except TokenizationError as e:
    logger.error(f"Tokenization failed: {str(e)}")
    tokens = fallback_tokenize(text)

class TestTokenization(unittest.TestCase):
    def test_basic_tokenization(self):
        tokens = tokenize("Hello, world!")
        self.assertEqual(tokens, ["Hello", ",", "world", "!"])
    
    def test_domain_term(self):
        tokens = tokenize("machine learning")
        self.assertEqual(tokens, ["machine_learning"])