导入测试

doc/test-data/intermediary/convert-all-to-idcard.py

@@ -0,0 +1,151 @@
+import pandas as pd
+import random
+from openpyxl import load_workbook
+from openpyxl.styles import Font, PatternFill, Alignment
+
+def calculate_id_check_code(id_17):
+    """
+    计算身份证校验码（符合GB 11643-1999标准）
+    """
+    weights = [7, 9, 10, 5, 8, 4, 2, 1, 6, 3, 7, 9, 10, 5, 8, 4, 2]
+    check_codes = ['1', '0', 'X', '9', '8', '7', '6', '5', '4', '3', '2']
+    weighted_sum = sum(int(id_17[i]) * weights[i] for i in range(17))
+    mod = weighted_sum % 11
+    return check_codes[mod]
+
+def generate_valid_person_id():
+    """
+    生成符合校验标准的18位身份证号
+    """
+    area_code = f"{random.randint(110000, 659999)}"
+    birth_year = random.randint(1960, 2000)
+    birth_month = f"{random.randint(1, 12):02d}"
+    birth_day = f"{random.randint(1, 28):02d}"
+    sequence_code = f"{random.randint(0, 999):03d}"
+
+    id_17 = f"{area_code}{birth_year}{birth_month}{birth_day}{sequence_code}"
+    check_code = calculate_id_check_code(id_17)
+
+    return f"{id_17}{check_code}"
+
+def validate_id_check_code(person_id):
+    """
+    验证身份证校验码是否正确
+    """
+    if len(str(person_id)) != 18:
+        return False
+    id_17 = str(person_id)[:17]
+    check_code = str(person_id)[17]
+    return calculate_id_check_code(id_17) == check_code.upper()
+
+# 读取现有文件
+input_file = 'doc/test-data/intermediary/intermediary_test_data_1000_valid.xlsx'
+output_file = 'doc/test-data/intermediary/intermediary_test_data_1000_valid.xlsx'
+
+print(f"正在读取文件: {input_file}")
+df = pd.read_excel(input_file)
+
+print(f"总行数: {len(df)}\n")
+
+# 统计各证件类型
+print("=== 原始证件类型分布 ===")
+for id_type, count in df['证件类型'].value_counts().items():
+    print(f"{id_type}: {count}条")
+
+# 找出所有非身份证类型的记录
+non_id_mask = df['证件类型'] != '身份证'
+non_id_count = non_id_mask.sum()
+id_card_count = (~non_id_mask).sum()
+
+print(f"\n需要转换的证件数量: {non_id_count}条")
+print(f"现有身份证数量: {id_card_count}条（保持不变）")
+
+# 备份现有身份证号码
+existing_id_cards = df[~non_id_mask]['证件号码*'].copy()
+print(f"\n已备份 {len(existing_id_cards)} 条现有身份证号码")
+
+# 转换证件类型并生成新身份证号
+print(f"\n正在转换证件类型并生成身份证号码...")
+updated_count = 0
+
+for idx in df[non_id_mask].index:
+    # 修改证件类型为身份证
+    df.loc[idx, '证件类型'] = '身份证'
+
+    # 生成新的身份证号
+    new_id = generate_valid_person_id()
+    df.loc[idx, '证件号码*'] = new_id
+    updated_count += 1
+
+    if (updated_count % 100 == 0) or (updated_count == non_id_count):
+        print(f"已处理 {updated_count}/{non_id_count} 条")
+
+# 保存到Excel
+df.to_excel(output_file, index=False, engine='openpyxl')
+
+# 格式化Excel文件
+wb = load_workbook(output_file)
+ws = wb.active
+
+# 设置列宽
+ws.column_dimensions['A'].width = 15
+ws.column_dimensions['B'].width = 12
+ws.column_dimensions['C'].width = 12
+ws.column_dimensions['D'].width = 8
+ws.column_dimensions['E'].width = 12
+ws.column_dimensions['F'].width = 20
+ws.column_dimensions['G'].width = 15
+ws.column_dimensions['H'].width = 15
+ws.column_dimensions['I'].width = 30
+ws.column_dimensions['J'].width = 20
+ws.column_dimensions['K'].width = 20
+ws.column_dimensions['L'].width = 12
+ws.column_dimensions['M'].width = 15
+ws.column_dimensions['N'].width = 12
+ws.column_dimensions['O'].width = 20
+
+# 设置表头样式
+header_fill = PatternFill(start_color='D3D3D3', end_color='D3D3D3', fill_type='solid')
+header_font = Font(bold=True)
+
+for cell in ws[1]:
+    cell.fill = header_fill
+    cell.font = header_font
+    cell.alignment = Alignment(horizontal='center', vertical='center')
+
+# 冻结首行
+ws.freeze_panes = 'A2'
+
+wb.save(output_file)
+
+# 最终验证
+print("\n正在进行最终验证...")
+df_verify = pd.read_excel(output_file)
+
+# 验证所有记录都是身份证
+all_id_card = (df_verify['证件类型'] == '身份证').all()
+print(f"所有证件类型均为身份证: {'✅ 是' if all_id_card else '❌ 否'}")
+
+# 验证所有身份证号码
+all_valid = True
+invalid_count = 0
+for idx, person_id in df_verify['证件号码*'].items():
+    if not validate_id_check_code(str(person_id)):
+        all_valid = False
+        invalid_count += 1
+        if invalid_count <= 5:
+            print(f"❌ 错误: {person_id}")
+
+print(f"\n身份证号码验证:")
+print(f"总数: {len(df_verify)}条")
+print(f"校验通过: {len(df_verify) - invalid_count}条 ✅")
+if invalid_count > 0:
+    print(f"校验失败: {invalid_count}条 ❌")
+
+print(f"\n=== 更新完成 ===")
+print(f"文件: {output_file}")
+print(f"转换证件数量: {updated_count}条")
+print(f"保持不变: {len(existing_id_cards)}条")
+print(f"总记录数: {len(df_verify)}条")
+print(f"\n✅ 所有1000条记录现在都使用身份证类型")
+print(f"✅ 所有身份证号码已通过GB 11643-1999标准校验")

doc/test-data/intermediary/fix-id-cards.py

@@ -0,0 +1,143 @@
+import pandas as pd
+import random
+from openpyxl import load_workbook
+from openpyxl.styles import Font, PatternFill, Alignment
+
+def calculate_id_check_code(id_17):
+    """
+    计算身份证校验码（符合GB 11643-1999标准）
+    """
+    weights = [7, 9, 10, 5, 8, 4, 2, 1, 6, 3, 7, 9, 10, 5, 8, 4, 2]
+    check_codes = ['1', '0', 'X', '9', '8', '7', '6', '5', '4', '3', '2']
+    weighted_sum = sum(int(id_17[i]) * weights[i] for i in range(17))
+    mod = weighted_sum % 11
+    return check_codes[mod]
+
+def generate_valid_person_id():
+    """
+    生成符合校验标准的18位身份证号
+    """
+    area_code = f"{random.randint(110000, 659999)}"
+    birth_year = random.randint(1960, 2000)
+    birth_month = f"{random.randint(1, 12):02d}"
+    birth_day = f"{random.randint(1, 28):02d}"
+    sequence_code = f"{random.randint(0, 999):03d}"
+
+    id_17 = f"{area_code}{birth_year}{birth_month}{birth_day}{sequence_code}"
+    check_code = calculate_id_check_code(id_17)
+
+    return f"{id_17}{check_code}"
+
+def validate_id_check_code(person_id):
+    """
+    验证身份证校验码是否正确
+    """
+    if len(person_id) != 18:
+        return False
+    id_17 = person_id[:17]
+    check_code = person_id[17]
+    return calculate_id_check_code(id_17) == check_code.upper()
+
+# 读取现有文件
+input_file = 'doc/test-data/intermediary/intermediary_test_data_1000_valid.xlsx'
+output_file = 'doc/test-data/intermediary/intermediary_test_data_1000_valid.xlsx'
+
+print(f"正在读取文件: {input_file}")
+df = pd.read_excel(input_file)
+
+print(f"总行数: {len(df)}")
+
+# 找出所有身份证类型的记录
+id_card_mask = df['证件类型'] == '身份证'
+id_card_count = id_card_mask.sum()
+
+print(f"\n找到 {id_card_count} 条身份证记录")
+
+# 验证现有身份证
+print("\n正在验证现有身份证校验码...")
+invalid_count = 0
+invalid_indices = []
+
+for idx in df[id_card_mask].index:
+    person_id = str(df.loc[idx, '证件号码*'])
+    if not validate_id_check_code(person_id):
+        invalid_count += 1
+        invalid_indices.append(idx)
+
+print(f"校验正确: {id_card_count - invalid_count}条")
+print(f"校验错误: {invalid_count}条")
+
+if invalid_count > 0:
+    print(f"\n需要重新生成 {invalid_count} 条身份证号码")
+
+# 重新生成所有身份证号码
+print(f"\n正在重新生成所有身份证号码...")
+updated_count = 0
+
+for idx in df[id_card_mask].index:
+    old_id = df.loc[idx, '证件号码*']
+    new_id = generate_valid_person_id()
+    df.loc[idx, '证件号码*'] = new_id
+    updated_count += 1
+
+    if (updated_count % 50 == 0) or (updated_count == id_card_count):
+        print(f"已更新 {updated_count}/{id_card_count} 条")
+
+# 保存到Excel
+df.to_excel(output_file, index=False, engine='openpyxl')
+
+# 格式化Excel文件
+wb = load_workbook(output_file)
+ws = wb.active
+
+# 设置列宽
+ws.column_dimensions['A'].width = 15
+ws.column_dimensions['B'].width = 12
+ws.column_dimensions['C'].width = 12
+ws.column_dimensions['D'].width = 8
+ws.column_dimensions['E'].width = 12
+ws.column_dimensions['F'].width = 20
+ws.column_dimensions['G'].width = 15
+ws.column_dimensions['H'].width = 15
+ws.column_dimensions['I'].width = 30
+ws.column_dimensions['J'].width = 20
+ws.column_dimensions['K'].width = 20
+ws.column_dimensions['L'].width = 12
+ws.column_dimensions['M'].width = 15
+ws.column_dimensions['N'].width = 12
+ws.column_dimensions['O'].width = 20
+
+# 设置表头样式
+header_fill = PatternFill(start_color='D3D3D3', end_color='D3D3D3', fill_type='solid')
+header_font = Font(bold=True)
+
+for cell in ws[1]:
+    cell.fill = header_fill
+    cell.font = header_font
+    cell.alignment = Alignment(horizontal='center', vertical='center')
+
+# 冻结首行
+ws.freeze_panes = 'A2'
+
+wb.save(output_file)
+
+# 最终验证
+print("\n正在进行最终验证...")
+df_verify = pd.read_excel(output_file)
+id_cards = df_verify[df_verify['证件类型'] == '身份证']['证件号码*']
+
+all_valid = True
+for idx, person_id in id_cards.items():
+    if not validate_id_check_code(str(person_id)):
+        all_valid = False
+        print(f"❌ 错误: {person_id}")
+
+if all_valid:
+    print(f"✅ 所有 {len(id_cards)} 条身份证号码校验通过！")
+else:
+    print("❌ 存在校验失败的身份证号码")
+
+print(f"\n=== 更新完成 ===")
+print(f"文件: {output_file}")
+print(f"更新身份证数量: {updated_count}条")
+print(f"其他证件类型保持不变")

doc/test-data/intermediary/generate-test-data-1000-valid.py

@@ -0,0 +1,215 @@
+import pandas as pd
+import random
+from openpyxl import load_workbook
+from openpyxl.styles import Font, PatternFill, Alignment
+
+def calculate_id_check_code(id_17):
+    """
+    计算身份证校验码（符合GB 11643-1999标准）
+    :param id_17: 前17位身份证号
+    :return: 校验码（0-9或X）
+    """
+    # 权重因子
+    weights = [7, 9, 10, 5, 8, 4, 2, 1, 6, 3, 7, 9, 10, 5, 8, 4, 2]
+
+    # 校验码对应表
+    check_codes = ['1', '0', 'X', '9', '8', '7', '6', '5', '4', '3', '2']
+
+    # 计算加权和
+    weighted_sum = sum(int(id_17[i]) * weights[i] for i in range(17))
+
+    # 取模得到索引
+    mod = weighted_sum % 11
+
+    # 返回对应的校验码
+    return check_codes[mod]
+
+def generate_valid_person_id(id_type):
+    """
+    生成符合校验标准的证件号码
+    """
+    if id_type == '身份证':
+        # 6位地区码 + 4位年份 + 2位月份 + 2位日期 + 3位顺序码
+        area_code = f"{random.randint(110000, 659999)}"
+        birth_year = random.randint(1960, 2000)
+        birth_month = f"{random.randint(1, 12):02d}"
+        birth_day = f"{random.randint(1, 28):02d}"
+        sequence_code = f"{random.randint(0, 999):03d}"
+
+        # 前17位
+        id_17 = f"{area_code}{birth_year}{birth_month}{birth_day}{sequence_code}"
+
+        # 计算校验码
+        check_code = calculate_id_check_code(id_17)
+
+        return f"{id_17}{check_code}"
+    else:
+        # 护照、台胞证、港澳通行证：8位数字
+        return str(random.randint(10000000, 99999999))
+
+# 验证身份证校验码
+def validate_id_check_code(person_id):
+    """
+    验证身份证校验码是否正确
+    """
+    if len(person_id) != 18:
+        return False
+
+    id_17 = person_id[:17]
+    check_code = person_id[17]
+
+    return calculate_id_check_code(id_17) == check_code.upper()
+
+# 定义数据生成规则
+last_names = ['王', '李', '张', '刘', '陈', '杨', '赵', '黄', '周', '吴', '徐', '孙', '胡', '朱', '高', '林', '何', '郭', '马', '罗']
+first_names_male = ['伟', '强', '磊', '洋', '勇', '军', '杰', '涛', '超', '明', '刚', '平', '辉', '鹏', '华', '飞', '鑫', '波', '斌', '宇']
+first_names_female = ['芳', '娜', '敏', '静', '丽', '娟', '燕', '艳', '玲', '婷', '慧', '君', '萍', '颖', '琳', '雪', '梅', '兰', '红', '霞']
+
+person_types = ['中介']
+person_sub_types = ['本人', '配偶', '子女', '父母', '其他']
+genders = ['M', 'F', 'O']
+id_types = ['身份证', '护照', '台胞证', '港澳通行证']
+
+companies = ['房屋租赁公司', '房产经纪公司', '投资咨询公司', '置业咨询公司', '不动产咨询公司', '物业管理公司', '资产评估公司', '土地评估公司', '地产代理公司', '房产咨询公司']
+positions = ['区域经理', '店长', '高级经纪人', '房产经纪人', '销售经理', '置业顾问', '物业顾问', '评估师', '业务员', '总监', '主管', None]
+relation_types = ['配偶', '子女', '父母', '兄弟姐妹', None, None]
+
+provinces = ['北京市', '上海市', '广东省', '江苏省', '浙江省', '四川省', '河南省', '福建省', '湖北省', '湖南省']
+districts = ['海淀区', '朝阳区', '天河区', '浦东新区', '西湖区', '黄浦区', '静安区', '徐汇区', '福田区', '罗湖区']
+streets = ['路', '大街', '大道', '街道', '巷', '广场', '大厦', '花园']
+buildings = ['1号楼', '2号楼', '3号楼', '4号楼', '5号楼', '6号楼', '7号楼', '8号楼', 'A座', 'B座']
+
+def generate_name(gender):
+    first_names = first_names_male if gender == 'M' else first_names_female
+    return random.choice(last_names) + random.choice(first_names)
+
+def generate_mobile():
+    return f"1{random.choice([3, 5, 7, 8, 9])}{random.randint(0, 9)}{random.randint(10000000, 99999999)}"
+
+def generate_wechat():
+    return f"wx_{''.join(random.choices('abcdefghijklmnopqrstuvwxyz0123456789', k=8))}"
+
+def generate_address():
+    return f"{random.choice(provinces)}{random.choice(districts)}{random.choice(streets)}{random.randint(1, 100)}号"
+
+def generate_social_credit_code():
+    return f"91{random.randint(0, 9)}{random.randint(10000000000000000, 99999999999999999)}"
+
+def generate_related_num_id():
+    return f"ID{random.randint(10000, 99999)}"
+
+def generate_row(index):
+    gender = random.choice(genders)
+    person_sub_type = random.choice(person_sub_types)
+    id_type = random.choice(id_types)
+
+    return {
+        '姓名*': generate_name(gender),
+        '人员类型': '中介',
+        '人员子类型': person_sub_type,
+        '性别': gender,
+        '证件类型': id_type,
+        '证件号码*': generate_valid_person_id(id_type),
+        '手机号码': generate_mobile(),
+        '微信号': random.choice([generate_wechat(), None, None]),
+        '联系地址': generate_address(),
+        '所在公司': random.choice(companies),
+        '企业统一信用码': random.choice([generate_social_credit_code(), None, None]),
+        '职位': random.choice(positions),
+        '关联人员ID': random.choice([generate_related_num_id(), None, None, None]),
+        '关系类型': random.choice(relation_types),
+        '备注': None
+    }
+
+# 生成1000条数据
+print("正在生成1000条测试数据...")
+data = []
+for i in range(1000):
+    row = generate_row(i)
+    data.append(row)
+
+    if (i + 1) % 100 == 0:
+        print(f"已生成 {i + 1} 条...")
+
+# 创建DataFrame
+df = pd.DataFrame(data)
+
+# 输出文件
+output_file = 'doc/test-data/intermediary/intermediary_test_data_1000_valid.xlsx'
+
+# 保存到Excel
+df.to_excel(output_file, index=False, engine='openpyxl')
+
+# 格式化Excel文件
+wb = load_workbook(output_file)
+ws = wb.active
+
+# 设置列宽
+ws.column_dimensions['A'].width = 15
+ws.column_dimensions['B'].width = 12
+ws.column_dimensions['C'].width = 12
+ws.column_dimensions['D'].width = 8
+ws.column_dimensions['E'].width = 12
+ws.column_dimensions['F'].width = 20
+ws.column_dimensions['G'].width = 15
+ws.column_dimensions['H'].width = 15
+ws.column_dimensions['I'].width = 30
+ws.column_dimensions['J'].width = 20
+ws.column_dimensions['K'].width = 20
+ws.column_dimensions['L'].width = 12
+ws.column_dimensions['M'].width = 15
+ws.column_dimensions['N'].width = 12
+ws.column_dimensions['O'].width = 20
+
+# 设置表头样式
+header_fill = PatternFill(start_color='D3D3D3', end_color='D3D3D3', fill_type='solid')
+header_font = Font(bold=True)
+
+for cell in ws[1]:
+    cell.fill = header_fill
+    cell.font = header_font
+    cell.alignment = Alignment(horizontal='center', vertical='center')
+
+# 冻结首行
+ws.freeze_panes = 'A2'
+
+wb.save(output_file)
+
+# 验证身份证校验码
+print("\n正在验证身份证校验码...")
+df_read = pd.read_excel(output_file)
+id_cards = df_read[df_read['证件类型'] == '身份证']['证件号码*']
+
+valid_count = 0
+invalid_count = 0
+invalid_ids = []
+
+for idx, person_id in id_cards.items():
+    if validate_id_check_code(str(person_id)):
+        valid_count += 1
+    else:
+        invalid_count += 1
+        invalid_ids.append(person_id)
+
+print(f"\n✅ 成功生成1000条测试数据到: {output_file}")
+print(f"\n=== 身份证校验码验证 ===")
+print(f"身份证总数: {len(id_cards)}条")
+print(f"校验正确: {valid_count}条 ✅")
+print(f"校验错误: {invalid_count}条")
+
+if invalid_count > 0:
+    print(f"\n错误的身份证号:")
+    for pid in invalid_ids[:10]:
+        print(f"  {pid}")
+
+print(f"\n=== 数据统计 ===")
+print(f"人员类型: {df_read['人员类型'].unique()}")
+print(f"性别分布: {dict(df_read['性别'].value_counts())}")
+print(f"证件类型分布: {dict(df_read['证件类型'].value_counts())}")
+print(f"人员子类型分布: {dict(df_read['人员子类型'].value_counts())}")
+
+print(f"\n=== 身份证号码样本（已验证校验码）===")
+valid_id_samples = id_cards.head(5).tolist()
+for sample in valid_id_samples:
+    is_valid = "✅" if validate_id_check_code(str(sample)) else "❌"
+    print(f"{sample} {is_valid}")

doc/test-data/intermediary/generate-test-data-1000.py

@@ -47,7 +47,15 @@ def generate_wechat():
 
 def generate_person_id(id_type):
     if id_type == '身份证':
-        return f"{random.randint(110000, 659999)}{random.randint(1970, 2000):02d}{random.randint(1, 12):02d}{random.randint(1, 28):02d}{random.randint(1000, 9999)}"
+        # 18位身份证号：6位地区码 + 4位年份 + 2位月份 + 2位日期 + 3位顺序码 + 1位校验码
+        area_code = f"{random.randint(110000, 659999)}"
+        birth_year = random.randint(1960, 2000)
+        birth_month = f"{random.randint(1, 12):02d}"
+        birth_day = f"{random.randint(1, 28):02d}"
+        sequence_code = f"{random.randint(0, 999):03d}"
+        # 简单校验码（随机0-9或X）
+        check_code = random.choice(['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'X'])
+        return f"{area_code}{birth_year}{birth_month}{birth_day}{sequence_code}{check_code}"
     else:
         return str(random.randint(10000000, 99999999))