baby-llama2-chinese/
├── model.py            # LLaMA2 模型结构与推理实现
├── pretrain.py         # 预训练脚本（百度百科 / 维基语料）
├── sft.py              # 有监督微调（SFT）脚本
├── eval.py             # 使用微调后模型的推理 / 简单评测脚本
├── eval_pretrain.py    # 预训练阶段模型的评估脚本
├── dataset.py          # 预训练数据集定义与加载
├── dataset_sft.py      # SFT 数据集定义与加载
├── data_process.py     # 预训练语料预处理脚本
├── sft_data_process.py # SFT 数据预处理脚本
├── chatglm_tokenizer/  # ChatGLM 分词器相关文件
├── data/               # 预训练语料与中间数据
├── data_clean/         # 数据清洗、日志等辅助工具
├── sft_data/           # SFT 微调所用的指令/对话数据
├── out/                # 训练输出目录（checkpoint、日志等）
├── requirements.txt    # Python 依赖列表
├── README.md           # 项目说明文档

# clear.py
def remove_duplicate_punctuation(sentence: str) -> str:
    '''
    删除句子中重复的标点符号、重复的空格，同时将换行变为特殊字符'\n'
    '''
    # 将空格（全角空格）替换为逗号, 可能会有重复的空格，下面删除重复标点会删除
    sentence = re.sub(' |　', '，', sentence) 

    ans = ''
    n = len(sentence)
    p = 0
    while p < n:
        ans += sentence[p]

        while p + 1 < n and sentence[p] in punctuation and sentence[p + 1] in punctuation:
            p += 1
        p += 1

    return ans

{
    "title": "文章标题",
    "summary": "文章简介/摘要",
    "sections": [
        {
            "title": "章节标题1",
            "content": "章节内容..."
        },
        {
            "title": "章节标题2", 
            "content": "章节内容..."
        }
    ],
    "tags": ["标签1", "标签2", ...],
    "url": "原始链接"
}

# process_baike()函数中
def process_function(line: str) -> dict:
    
    item = ujson.loads(line)
    item_title = item['title']
    item_sections = item ['sections']
    for data in item_sections:
        #print(item['completion'])
        # 数据清洗
        response = data['content'].replace('\r','')
        response = remove_duplicate_punctuation(response)
        # 剔除短数据
        if len(response) < response_less_word:
            return None
        response = data['title']+data['content']
        write_dict = {
                "response": response,
            }
        return write_dict

# clear.py
def read_and_write_template_baike(read_file: str, write_to_file: str, call_back: object, group_cnt: int=10000) -> None:
    '''
    处理数据读写模板，需要提供一个回调函数call_back，
    read_file: 原始数据文件
    write_to_file：处理后的要保存数据文件
    call_back：函数输入一个字符串，输出一个处理后的字典dict，如果输入的字符串为无效数据，请返回None
    group_cnt: parquet file分割行数
    如：
    >>> def call_back(inputs: str) -> dict:
    >>>     if check(inputs) not valid:
    >>>         return None
    ...    
    ...    do something for inputs
    ...
    >>>     my_dict = {
    >>>             'prompt': inputs['p'],
    >>>             'response': inputs['a1'] + inputs['a2'],
    >>>             ...
    >>>         }
    >>>     return my_dict
    '''

    log.info('process file:{}'.format(read_file), save_to_file=True)
    start = time.time()
    
    raw_line_cnt = 0
    keep_line_cnt = 0
    with progress.open(read_file, 'r', encoding='utf-8') as f_read:
        cur_rows = []
        append = cur_rows.append
        for line in f_read:
            try:
                #print(line)
                raw_line_cnt += 1
                write_dict = call_back(line)
                if write_dict is None: continue
                keep_line_cnt += 1
                append(write_dict)
                if len(cur_rows) >= group_cnt:
                    df = pd.DataFrame(cur_rows)
                    write_single_parquet_file(write_to_file, df)
                    cur_rows = []
                    append = cur_rows.append
            except Exception as e:
                # log.error('处理文件异常：{}, content:{}'.format(str(e), line))
                print(line)
                raise e
            # end for
            # 处理末尾部分
        if len(cur_rows) > 0:
            df = pd.DataFrame(cur_rows)
            write_single_parquet_file(write_to_file, df)
            cur_rows = []
        end = time.time()
        log.info('原始文件:{}，共{}行，处理后剩余{}行，保存到文件：{}。耗时：{:.6}s'\
                    .format(read_file, raw_line_cnt, keep_line_cnt, write_to_file, end - start), save_to_file=True)

输入文件 (JSON/文本)
    │
    ▼ 逐行读取
┌─────────────────┐
│  call_back()   │ ← 回调函数处理每一行
└────────┬────────┘
         │
    ┌────┴────┐
    ▼         ▼
  None     返回 dict
 (跳过)     (收集保存)
    │         │
    └────┬────┘
         ▼
   累计达到 group_cnt 行
         │
         ▼
   写入 Parquet 文件

# data_process.py
def process_baidu():
    BATCH_SIZE = 1000000

    cnt=0
    batch_cnt=0
    token=0
    doc_ids=[]

    f1=open('./data/563w_baidubaike/563w_baidubaike.json','r',encoding='utf-8')
    
    while True:
        line = f1.readline()
        if not line:
            break
        line=json.loads(line)
        text=''
        try:
            text+=line['title']+'：'+line['summary']
        except:
            pass
        for per in line['sections']:
            text+=per['title']+'：'+per['content']+'。'
        text_id=tokenizer.encode(text,add_special_tokens=False)
        text_id.append(tokenizer.special_tokens['<eos>'])
        if len(text_id)>5:
            doc_ids+=text_id
        cnt+=1
        if cnt%BATCH_SIZE==0:
            batch_cnt+=1
            arr = np.array(doc_ids,dtype=np.uint16)
            doc_ids=[]
            print('cnt:',cnt,'arr_shape:',arr.shape)
            with open('./data/baidubaike_563w_{}.bin'.format(batch_cnt),'wb') as f2:
                f2.write(arr.tobytes())
            del arr

    if not doc_ids:
        batch_cnt+=1
        arr = np.array(doc_ids,dtype=np.uint16)
        print('cnt:',cnt,'arr_shape:',arr.shape)
        with open('./data/baidubaike_563w_{}.bin'.format(batch_cnt),'wb') as f:
            f.write(arr.tobytes())