fix: preserve overlapping import deduplication

This commit is contained in:
2026-08-26 00:43:43 +03:00
parent 62519f80cd
commit 3f5681074e
3 changed files with 29 additions and 15 deletions

View File

@@ -13,6 +13,15 @@ const duplicateStatement = {
statement: { accountNumber: 'fingerprint-test', currency: 'RUB', openingBalance: 0, closingBalance: 200, exportedAt: '2026-08-20T12:00:00+03:00' },
transactions: Array.from({ length: 2 }, () => ({ operationAt: '2026-08-20T00:00:00+03:00', amountSigned: 100, commission: 0, description: 'Пополнение' })),
};
const overlapStatement = {
...duplicateStatement,
statement: { ...duplicateStatement.statement, accountNumber: 'overlap-test' },
};
const singleStatement = {
...overlapStatement,
statement: { ...overlapStatement.statement, closingBalance: 100 },
transactions: overlapStatement.transactions.slice(0, 1),
};
async function run(): Promise<void> {
try {
@@ -34,11 +43,22 @@ async function run(): Promise<void> {
duplicatesSkipped: 0,
totalInFile: 2,
});
await importStatement(singleStatement);
const overlapResult = await importStatement(overlapStatement);
if ('status' in overlapResult) throw new Error(overlapResult.message);
assert.deepEqual(overlapResult, {
accountId: overlapResult.accountId,
isNewAccount: false,
accountNumberMasked: 'overla******test',
imported: 1,
duplicatesSkipped: 1,
totalInFile: 2,
});
console.log('import metadata SQL: OK');
} finally {
await pool.query("DELETE FROM transactions WHERE account_id IN (SELECT id FROM accounts WHERE bank = 'TEST' AND account_number IN ('metadata-test', 'fingerprint-test'))");
await pool.query("DELETE FROM imports WHERE account_id IN (SELECT id FROM accounts WHERE bank = 'TEST' AND account_number IN ('metadata-test', 'fingerprint-test'))");
await pool.query("DELETE FROM accounts WHERE bank = 'TEST' AND account_number IN ('metadata-test', 'fingerprint-test')");
await pool.query("DELETE FROM transactions WHERE account_id IN (SELECT id FROM accounts WHERE bank = 'TEST' AND account_number IN ('metadata-test', 'fingerprint-test', 'overlap-test'))");
await pool.query("DELETE FROM imports WHERE account_id IN (SELECT id FROM accounts WHERE bank = 'TEST' AND account_number IN ('metadata-test', 'fingerprint-test', 'overlap-test'))");
await pool.query("DELETE FROM accounts WHERE bank = 'TEST' AND account_number IN ('metadata-test', 'fingerprint-test', 'overlap-test')");
await pool.end();
}
}

View File

@@ -167,15 +167,6 @@ export async function importStatement(
const data = body as StatementFile;
const semErr = validateSemantics(data);
if (semErr) return semErr;
const fallbackFingerprints = new Set<string>();
const duplicateFallbackFingerprints = new Set<string>();
for (const tx of data.transactions) {
if (!tx.sourceId) {
const fp = computeFingerprint(data.statement.accountNumber, tx);
if (fallbackFingerprints.has(fp)) duplicateFallbackFingerprints.add(fp);
fallbackFingerprints.add(fp);
}
}
const client = await db.connect();
try {
@@ -232,11 +223,14 @@ export async function importStatement(
// Insert transactions
const insertedIds: number[] = [];
const fallbackFingerprintOccurrences = new Map<string, number>();
for (const [sourcePosition, tx] of data.transactions.entries()) {
const fallbackFingerprint = computeFingerprint(data.statement.accountNumber, tx);
const fp = !tx.sourceId && duplicateFallbackFingerprints.has(fallbackFingerprint)
? computeFingerprint(data.statement.accountNumber, tx, sourcePosition)
const occurrence = fallbackFingerprintOccurrences.get(fallbackFingerprint) ?? 0;
fallbackFingerprintOccurrences.set(fallbackFingerprint, occurrence + 1);
const fp = !tx.sourceId && occurrence > 0
? computeFingerprint(data.statement.accountNumber, tx, occurrence)
: fallbackFingerprint;
const isCashbackCommissionImport =
tx.amountSigned === 0 &&

View File

@@ -124,7 +124,7 @@ accountNumber|operationAt|amountSigned|commission|normalizedDescription[|sourceP
```
- `normalizedDescription``description` после `trim`.
- `sourcePosition`индекс операции в массиве `transactions`; добавляется, только если одинаковые операции без `sourceId` повторяются в одном файле.
- `sourcePosition`порядковый номер повторяющейся операции в массиве `transactions`; добавляется, только если одинаковые операции без `sourceId` повторяются в одном файле.
- Суммы подставляются в том виде, в котором пришли в JSON (числовое представление).
- Разделитель `|` исключает коллизии при склейке полей разной длины.