feat(ccdi-project): harden bank statement dedup import

2026-03-10 10:39:06 +08:00
parent 924605ac3a
commit 6007f3284a
5 changed files with 228 additions and 57 deletions
--- a/ccdi-project/src/main/java/com/ruoyi/ccdi/project/service/impl/CcdiFileUploadServiceImpl.java
+++ b/ccdi-project/src/main/java/com/ruoyi/ccdi/project/service/impl/CcdiFileUploadServiceImpl.java
@@ -56,7 +56,7 @@ public class CcdiFileUploadServiceImpl implements ICcdiFileUploadService {
    private static class FetchBankStatementResult {
        private boolean success;
        private int totalCount;
-        private int savedCount;
+        private int attemptedCount;
        private String errorMessage;
    }

@@ -352,6 +352,14 @@ public class CcdiFileUploadServiceImpl implements ICcdiFileUploadService {
        return errorMessage.substring(0, MAX_ERROR_MESSAGE_LENGTH);
    }

+    private String trimAccountNo(String value) {
+        return value == null ? null : value.trim();
+    }
+
+    private void normalizeDedupFields(CcdiBankStatement statement) {
+        statement.setLeAccountNo(trimAccountNo(statement.getLeAccountNo()));
+    }
+
    /**
     * 异步处理单个文件的完整流程
     * 包含:上传 → 轮询解析状态 → 获取结果 → 保存流水数据
@@ -583,14 +591,13 @@ public class CcdiFileUploadServiceImpl implements ICcdiFileUploadService {
            if (totalCount == null || totalCount <= 0) {
                log.warn("【文件上传】无流水数据需要保存: totalCount={}", totalCount);
                result.setSuccess(true);
-                result.setSavedCount(0);
                return result;
            }

            int pageSize = 1000;
            int batchSize = 1000;
            int totalPages = (int) Math.ceil((double) totalCount / pageSize);
-            int totalSaved = 0;
+            int totalAttempted = 0;
            List<CcdiBankStatement> batchList = new ArrayList<>(batchSize);

            log.info("【文件上传】获取到总数: totalCount={}", totalCount);
@@ -622,13 +629,14 @@ public class CcdiFileUploadServiceImpl implements ICcdiFileUploadService {
                        if (statement != null) {
                            statement.setBatchId(logId);
                            statement.setProjectId(projectId);
+                            normalizeDedupFields(statement);
                            batchList.add(statement);

                            if (batchList.size() >= batchSize) {
+                                int currentBatchSize = batchList.size();
                                bankStatementMapper.insertBatch(batchList);
-                                totalSaved += batchList.size();
-                                log.debug("【文件上传】批量插入流水 {}条, 累计{}条",
-                                         batchList.size(), totalSaved);
+                                totalAttempted += currentBatchSize;
+                                log.debug("【文件上传】批量写入流水 {}条", currentBatchSize);
                                batchList.clear();
                            }
                        }
@@ -643,14 +651,16 @@ public class CcdiFileUploadServiceImpl implements ICcdiFileUploadService {
            }

            if (!batchList.isEmpty()) {
+                int currentBatchSize = batchList.size();
                bankStatementMapper.insertBatch(batchList);
-                totalSaved += batchList.size();
+                totalAttempted += currentBatchSize;
                log.debug("【文件上传】批量插入剩余流水 {}条", batchList.size());
            }

-            log.info("【文件上传】流水数据保存完成: 总共保存{}条", totalSaved);
+            log.info("【文件上传】流水入库完成: fetchedCount={}, attemptedCount={}",
+                totalCount, totalAttempted);
            result.setSuccess(true);
-            result.setSavedCount(totalSaved);
+            result.setAttemptedCount(totalAttempted);
            return result;
        } catch (Exception e) {
            log.error("【文件上传】获取或保存流水数据失败: logId={}", logId, e);
--- a/ccdi-project/src/main/resources/mapper/ccdi/project/CcdiBankStatementMapper.xml
+++ b/ccdi-project/src/main/resources/mapper/ccdi/project/CcdiBankStatementMapper.xml
@@ -84,6 +84,8 @@ PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
                #{item.overrideBsId}, #{item.paymentMethod}, #{item.cretNo}
            )
        </foreach>
+        on duplicate key update
+            bank_statement_id = bank_statement_id
    </insert>

    <delete id="deleteByProjectIdAndBatchId">
--- a/ccdi-project/src/test/java/com/ruoyi/ccdi/project/service/impl/CcdiFileUploadServiceImplTest.java
+++ b/ccdi-project/src/test/java/com/ruoyi/ccdi/project/service/impl/CcdiFileUploadServiceImplTest.java
@@ -1,5 +1,8 @@
 package com.ruoyi.ccdi.project.service.impl;

+import ch.qos.logback.classic.Logger;
+import ch.qos.logback.classic.spi.ILoggingEvent;
+import ch.qos.logback.core.read.ListAppender;
 import com.ruoyi.ccdi.project.domain.entity.CcdiFileUploadRecord;
 import com.ruoyi.ccdi.project.mapper.CcdiBankStatementMapper;
 import com.ruoyi.ccdi.project.mapper.CcdiFileUploadRecordMapper;
@@ -16,14 +19,18 @@ import org.junit.jupiter.api.extension.ExtendWith;
 import org.mockito.InjectMocks;
 import org.mockito.Mock;
 import org.mockito.junit.jupiter.MockitoExtension;
+import org.slf4j.LoggerFactory;
+import org.springframework.test.util.ReflectionTestUtils;

 import java.io.IOException;
+import java.math.BigDecimal;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.atomic.AtomicInteger;

+import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.mockito.ArgumentMatchers.any;
@@ -187,6 +194,84 @@ class CcdiFileUploadServiceImplTest {
        assertTrue(failedRecord.getErrorMessage().length() <= MAX_ERROR_MESSAGE_LENGTH);
    }

+    @Test
+    void fetchAndSaveBankStatements_shouldTrimLeAccountNoBeforeInsert() throws IOException {
+        when(lsfxClient.uploadFile(eq(LSFX_PROJECT_ID), any())).thenReturn(buildUploadResponse());
+        when(lsfxClient.checkParseStatus(LSFX_PROJECT_ID, String.valueOf(LOG_ID)))
+            .thenReturn(buildCheckParseStatusResponse(false));
+        when(lsfxClient.getFileUploadStatus(any())).thenReturn(buildParsedSuccessStatusResponse());
+        when(lsfxClient.getBankStatement(any(GetBankStatementRequest.class)))
+            .thenReturn(buildBankStatementResponseWithItems(1, List.of(buildBankStatementItem(" 62220001 "))))
+            .thenReturn(buildBankStatementResponseWithItems(1, List.of(buildBankStatementItem(" 62220001 "))));
+
+        CcdiFileUploadRecord record = buildRecord();
+        Path tempFile = createTempFile();
+
+        service.processFileAsync(PROJECT_ID, LSFX_PROJECT_ID, tempFile.toString(), RECORD_ID, "batch-1", record);
+
+        verify(bankStatementMapper).insertBatch(any());
+        verify(bankStatementMapper).insertBatch(org.mockito.ArgumentMatchers.argThat(list ->
+            list.size() == 1 && "62220001".equals(list.get(0).getLeAccountNo())));
+    }
+
+    @Test
+    void fetchAndSaveBankStatements_shouldLogConservativeCountsWhenAffectedRowsAreAmbiguous() {
+        when(lsfxClient.getBankStatement(any(GetBankStatementRequest.class)))
+            .thenReturn(buildBankStatementResponseWithItems(1, List.of(buildBankStatementItem("62220001"))))
+            .thenReturn(buildBankStatementResponseWithItems(1, List.of(buildBankStatementItem("62220001"))));
+        when(bankStatementMapper.insertBatch(any())).thenReturn(1);
+
+        Logger logger = (Logger) LoggerFactory.getLogger(CcdiFileUploadServiceImpl.class);
+        ListAppender<ILoggingEvent> logAppender = new ListAppender<>();
+        logAppender.start();
+        logger.addAppender(logAppender);
+
+        try {
+            Object result = ReflectionTestUtils.invokeMethod(
+                service,
+                "fetchAndSaveBankStatements",
+                PROJECT_ID,
+                LSFX_PROJECT_ID,
+                LOG_ID
+            );
+
+            assertTrue(Boolean.TRUE.equals(ReflectionTestUtils.getField(result, "success")));
+            assertEquals(1, ReflectionTestUtils.getField(result, "totalCount"));
+            assertEquals(1, ReflectionTestUtils.getField(result, "attemptedCount"));
+            assertTrue(logAppender.list.stream().map(ILoggingEvent::getFormattedMessage)
+                .anyMatch(message -> message.contains("流水入库完成: fetchedCount=1, attemptedCount=1")));
+            assertFalse(logAppender.list.stream().map(ILoggingEvent::getFormattedMessage)
+                .anyMatch(message -> message.contains("insertedCount=")));
+        } finally {
+            logger.detachAppender(logAppender);
+        }
+    }
+
+    @Test
+    void processFileAsync_shouldMarkParsedFailedWhenInsertBatchThrowsUnexpectedSqlError() throws IOException {
+        List<String> events = new ArrayList<>();
+        AtomicInteger sequence = new AtomicInteger();
+        captureRecordStatus(events, sequence);
+
+        when(lsfxClient.uploadFile(eq(LSFX_PROJECT_ID), any())).thenReturn(buildUploadResponse());
+        when(lsfxClient.checkParseStatus(LSFX_PROJECT_ID, String.valueOf(LOG_ID)))
+            .thenReturn(buildCheckParseStatusResponse(false));
+        when(lsfxClient.getFileUploadStatus(any())).thenReturn(buildParsedSuccessStatusResponse());
+        when(lsfxClient.getBankStatement(any(GetBankStatementRequest.class)))
+            .thenReturn(buildBankStatementResponseWithItems(1, List.of(buildBankStatementItem("62220001"))))
+            .thenReturn(buildBankStatementResponseWithItems(1, List.of(buildBankStatementItem("62220001"))));
+        when(bankStatementMapper.insertBatch(any()))
+            .thenThrow(new RuntimeException("sql syntax error"));
+
+        CcdiFileUploadRecord record = buildRecord();
+        Path tempFile = createTempFile();
+
+        service.processFileAsync(PROJECT_ID, LSFX_PROJECT_ID, tempFile.toString(), RECORD_ID, "batch-1", record);
+
+        assertTrue(events.stream().anyMatch(event -> event.endsWith("record:parsed_failed")));
+        assertFalse(events.stream().anyMatch(event -> event.endsWith("record:parsed_success")));
+    }
+
    private void captureRecordStatus(List<String> events, AtomicInteger sequence) {
        doAnswer(invocation -> {
            CcdiFileUploadRecord record = invocation.getArgument(0);
@@ -279,6 +364,28 @@ class CcdiFileUploadServiceImplTest {
        return response;
    }

+    private GetBankStatementResponse buildBankStatementResponseWithItems(
+        int totalCount,
+        List<GetBankStatementResponse.BankStatementItem> items
+    ) {
+        GetBankStatementResponse.BankStatementData data = new GetBankStatementResponse.BankStatementData();
+        data.setTotalCount(totalCount);
+        data.setBankStatementList(items);
+
+        GetBankStatementResponse response = new GetBankStatementResponse();
+        response.setData(data);
+        return response;
+    }
+
+    private GetBankStatementResponse.BankStatementItem buildBankStatementItem(String accountMaskNo) {
+        GetBankStatementResponse.BankStatementItem item = new GetBankStatementResponse.BankStatementItem();
+        item.setAccountMaskNo(accountMaskNo);
+        item.setAccountingDateId(20260310);
+        item.setDrAmount(new BigDecimal("100.00"));
+        item.setCrAmount(new BigDecimal("0.00"));
+        return item;
+    }
+
    private int findEventIndex(List<String> events, String suffix) {
        for (int i = 0; i < events.size(); i++) {
            if (events.get(i).endsWith(suffix)) {