当使用Tokenizers的时候，内部发生了什么？

from tokenizers import normalizers
from tokenizers.normalizers import NFD, StripAccents, Lowercase

normalizer = normalizers.Sequence([
    NFD(),             # Unicode正规化
    StripAccents(),    # 去除音调符号
    Lowercase()])      # 转小写

normalizer.normalize_str("Héllò hõw are ù?")
tokenizer.normalizer = normalizer

from tokenizers.pre_tokenizers import Whitespace, Digits

# 简单的空格分词器
pre_tokenizer = Whitespace()
pre_tokenizer.pre_tokenize_str("Hello! How are you? I'm fine, thank you.")
# 会按空格分割文本

# 或者组合多个预分词器
pre_tokenizer = pre_tokenizers.Sequence([
    Whitespace(),
    Digits(individual_digits=True)
])

pre_tokenizer.pre_tokenize_str("Call 911! How are you? I'm fine thank you") # 空格分词+数字单独处理

tokenizer.pre_tokenizer = pre_tokenizer

tokenizer = Tokenizer(BPE(unk_token="[UNK]"))

from tokenizers import Tokenizer
from tokenizers.models import BPE
from tokenizers.normalizers import NFD, StripAccents, Lowercase, Sequence as NormSequence
from tokenizers.pre_tokenizers import Whitespace, Digits, Sequence as PreSequence
from tokenizers.processors import TemplateProcessing

# 1. 创建一个基本的分词器
tokenizer = Tokenizer(BPE(unk_token="[UNK]"))

# 2. 设置归一化组件
normalizer = NormSequence([
    NFD(),             # Unicode标准化
    StripAccents(),    # 去除音调符号
    Lowercase()        # 转小写
])
tokenizer.normalizer = normalizer

# 3. 设置预分词组件
pre_tokenizer = PreSequence([
    Whitespace(),      # 按空格分词
    Digits(individual_digits=True)  # 数字单独处理
])
tokenizer.pre_tokenizer = pre_tokenizer

# 4. 设置后处理组件
tokenizer.post_processor = TemplateProcessing(
    single="[CLS] $A [SEP]",
    pair="[CLS] $A [SEP] $B [SEP]",
    special_tokens=[
        ("[CLS]", 1),
        ("[SEP]", 2),
    ],
)

# 假设已经训练好了这个分词器
# 对文本进行分词处理
text = "Hello! 你好123 Cómo estás?"

# 实际处理流程：
# 1. 归一化: "hello! 你好123 como estas?"
# 2. 预分词: ["hello!", "你好", "1", "2", "3", "como", "estas?"]
# 3. 模型处理: 将预分词结果转换为token IDs
# 4. 后处理: 添加特殊标记 "[CLS] ... [SEP]"

encoded = tokenizer.encode(text)
print(f"Tokens: {encoded.tokens}")
print(f"IDs: {encoded.ids}")
# tokens = ["[CLS]", "hello", "!", "你好", "1", "2", "3", "como", "estas", "?", "[SEP]"]
# ids = [1, 25, 16, 8930, 13, 14, 15, 345, 789, 19, 2]

from tokenizers import Tokenizer
from tokenizers.models import BPE
from tokenizers.trainers import BpeTrainer
from tokenizers.pre_tokenizers import Whitespace
from tokenizers.normalizers import NFD, Lowercase, StripAccents
from tokenizers.processors import TemplateProcessing
import os

# 步骤1: 准备训练数据
# 这里假设我们有一些文本文件用于训练
def get_training_corpus(folder_path, batch_size=1000):
    # 这个函数会批量读取文本文件
    files = [os.path.join(folder_path, f) for f in os.listdir(folder_path) if f.endswith('.txt')]
    for file_path in files:
        with open(file_path, 'r', encoding='utf-8') as f:
            lines = []
            for line in f:
                line = line.strip()
                if line:  # 跳过空行
                    lines.append(line)
                    if len(lines) >= batch_size:
                        yield lines
                        lines = []
            if lines:  # 不要忘记最后一批
                yield lines

# 步骤2: 初始化tokenizer
# 创建一个使用BPE算法的tokenizer
tokenizer = Tokenizer(BPE(unk_token="[UNK]"))

# 步骤3: 配置normalizer（归一化处理）
tokenizer.normalizer = NFD() + Lowercase() + StripAccents()

# 步骤4: 配置pre-tokenizer（预分词）
tokenizer.pre_tokenizer = Whitespace()  # 基于空格的预分词

# 步骤5: 配置训练器
# 我们需要指定特殊token和词汇表大小
trainer = BpeTrainer(
    vocab_size=20000,  # 词汇表大小
    min_frequency=2,   # 最小词频
    special_tokens=["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"]  # 特殊token
)

# 步骤6: 训练tokenizer
# 假设我们的训练文本在'corpus'文件夹中
corpus_folder = "corpus"

# 训练过程
tokenizer.train_from_iterator(get_training_corpus(corpus_folder), trainer)

# 步骤7: 添加后处理
# 例如，自动在序列前后添加[CLS]和[SEP]
tokenizer.post_processor = TemplateProcessing(
    single="[CLS] $A [SEP]",
    pair="[CLS] $A [SEP] $B [SEP]",
    special_tokens=[
        ("[CLS]", tokenizer.token_to_id("[CLS]")),
        ("[SEP]", tokenizer.token_to_id("[SEP]")),
    ],
)

# 步骤8: 保存训练好的tokenizer
tokenizer.save("custom_tokenizer.json")

# 测试我们的tokenizer
test_sentence = "这是一个用于测试的中文句子。It also contains some English words!"
encoded = tokenizer.encode(test_sentence)
print(f"Tokens: {encoded.tokens}")
print(f"IDs: {encoded.ids}")

# 测试解码
decoded = tokenizer.decode(encoded.ids)
print(f"Decoded: {decoded}")

from transformers import AutoTokenizer
import json
import os

# 方法1：加载预训练模型的tokenizer并查看合并规则
def inspect_tokenizer_merge_rules(model_name):
    # 加载tokenizer
    tokenizer = AutoTokenizer.from_pretrained(model_name)
    
    # 检查是否是BPE类型的tokenizer
    if hasattr(tokenizer, 'bpe_ranks'):
        print(f"BPE合并规则 (前20个):")
        for i, (pair, rank) in enumerate(sorted(tokenizer.bpe_ranks.items(), key=lambda x: x[1])):
            if i >= 20:
                break
            print(f"规则 {i}: {pair[0]}+{pair[1]} -> {''.join(pair)} (优先级: {rank})")
    else:
        # 尝试另一种方式访问 - 对于较新的Tokenizers
        vocab_file = tokenizer.vocab_file if hasattr(tokenizer, 'vocab_file') else None
        merges_file = tokenizer.merges_file if hasattr(tokenizer, 'merges_file') else None
        
        print(f"Tokenizer类型: {type(tokenizer).__name__}")
        print(f"词汇表文件: {vocab_file}")
        print(f"合并规则文件: {merges_file}")
        
        # 如果有merges文件，尝试读取
        if merges_file and os.path.exists(merges_file):
            with open(merges_file, 'r', encoding='utf-8') as f:
                merges = f.readlines()[1:]  # 跳过表头
            
            print(f"\n前20个合并规则:")
            for i, merge in enumerate(merges[:20]):
                print(f"规则 {i}: {merge.strip()}")

# 方法2：从保存的tokenizer.json文件中提取合并规则
def inspect_tokenizer_json(tokenizer_json_path):
    with open(tokenizer_json_path, 'r', encoding='utf-8') as f:
        tokenizer_data = json.load(f)
    
    # 检查是否包含merges
    if 'model' in tokenizer_data and 'merges' in tokenizer_data['model']:
        merges = tokenizer_data['model']['merges']
        print(f"合并规则总数: {len(merges)}")
        print(f"前20个合并规则:")
        for i, merge in enumerate(merges[:20]):
            print(f"规则 {i}: {merge}")
    else:
        print("该tokenizer.json不包含明确的合并规则")
        # 打印tokenizer的模型类型
        if 'model' in tokenizer_data and 'type' in tokenizer_data['model']:
            print(f"Tokenizer模型类型: {tokenizer_data['model']['type']}")

# 示例用法
print("查看BERT tokenizer:")
inspect_tokenizer_merge_rules('bert-base-uncased')

print("\n查看GPT-2 tokenizer (使用BPE):")
inspect_tokenizer_merge_rules('gpt2')

# 如果有保存的tokenizer.json文件，也可以直接查看
# inspect_tokenizer_json('path/to/tokenizer.json')

"天津工业大学相当于中等985的水平" → "天津工业大学相当于中等985的水平"

["天", "津", "工", "业", "大", "学", "相", "当", "于", "中", "等", "9", "8", "5", "的", "水", "平"]

"天津工业大学" → 8426
"相当" → 2145
"于" → 21
"中等" → 3762
"985" → 5329
"的" → 8
"水平" → 4218

[101, 8426, 2145, 21, 3762, 5329, 8, 4218, 102]