fix: preserve overlapping import deduplication
This commit is contained in:
@@ -13,6 +13,15 @@ const duplicateStatement = {
|
||||
statement: { accountNumber: 'fingerprint-test', currency: 'RUB', openingBalance: 0, closingBalance: 200, exportedAt: '2026-08-20T12:00:00+03:00' },
|
||||
transactions: Array.from({ length: 2 }, () => ({ operationAt: '2026-08-20T00:00:00+03:00', amountSigned: 100, commission: 0, description: 'Пополнение' })),
|
||||
};
|
||||
const overlapStatement = {
|
||||
...duplicateStatement,
|
||||
statement: { ...duplicateStatement.statement, accountNumber: 'overlap-test' },
|
||||
};
|
||||
const singleStatement = {
|
||||
...overlapStatement,
|
||||
statement: { ...overlapStatement.statement, closingBalance: 100 },
|
||||
transactions: overlapStatement.transactions.slice(0, 1),
|
||||
};
|
||||
|
||||
async function run(): Promise<void> {
|
||||
try {
|
||||
@@ -34,11 +43,22 @@ async function run(): Promise<void> {
|
||||
duplicatesSkipped: 0,
|
||||
totalInFile: 2,
|
||||
});
|
||||
await importStatement(singleStatement);
|
||||
const overlapResult = await importStatement(overlapStatement);
|
||||
if ('status' in overlapResult) throw new Error(overlapResult.message);
|
||||
assert.deepEqual(overlapResult, {
|
||||
accountId: overlapResult.accountId,
|
||||
isNewAccount: false,
|
||||
accountNumberMasked: 'overla******test',
|
||||
imported: 1,
|
||||
duplicatesSkipped: 1,
|
||||
totalInFile: 2,
|
||||
});
|
||||
console.log('import metadata SQL: OK');
|
||||
} finally {
|
||||
await pool.query("DELETE FROM transactions WHERE account_id IN (SELECT id FROM accounts WHERE bank = 'TEST' AND account_number IN ('metadata-test', 'fingerprint-test'))");
|
||||
await pool.query("DELETE FROM imports WHERE account_id IN (SELECT id FROM accounts WHERE bank = 'TEST' AND account_number IN ('metadata-test', 'fingerprint-test'))");
|
||||
await pool.query("DELETE FROM accounts WHERE bank = 'TEST' AND account_number IN ('metadata-test', 'fingerprint-test')");
|
||||
await pool.query("DELETE FROM transactions WHERE account_id IN (SELECT id FROM accounts WHERE bank = 'TEST' AND account_number IN ('metadata-test', 'fingerprint-test', 'overlap-test'))");
|
||||
await pool.query("DELETE FROM imports WHERE account_id IN (SELECT id FROM accounts WHERE bank = 'TEST' AND account_number IN ('metadata-test', 'fingerprint-test', 'overlap-test'))");
|
||||
await pool.query("DELETE FROM accounts WHERE bank = 'TEST' AND account_number IN ('metadata-test', 'fingerprint-test', 'overlap-test')");
|
||||
await pool.end();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -167,15 +167,6 @@ export async function importStatement(
|
||||
const data = body as StatementFile;
|
||||
const semErr = validateSemantics(data);
|
||||
if (semErr) return semErr;
|
||||
const fallbackFingerprints = new Set<string>();
|
||||
const duplicateFallbackFingerprints = new Set<string>();
|
||||
for (const tx of data.transactions) {
|
||||
if (!tx.sourceId) {
|
||||
const fp = computeFingerprint(data.statement.accountNumber, tx);
|
||||
if (fallbackFingerprints.has(fp)) duplicateFallbackFingerprints.add(fp);
|
||||
fallbackFingerprints.add(fp);
|
||||
}
|
||||
}
|
||||
|
||||
const client = await db.connect();
|
||||
try {
|
||||
@@ -232,11 +223,14 @@ export async function importStatement(
|
||||
|
||||
// Insert transactions
|
||||
const insertedIds: number[] = [];
|
||||
const fallbackFingerprintOccurrences = new Map<string, number>();
|
||||
|
||||
for (const [sourcePosition, tx] of data.transactions.entries()) {
|
||||
const fallbackFingerprint = computeFingerprint(data.statement.accountNumber, tx);
|
||||
const fp = !tx.sourceId && duplicateFallbackFingerprints.has(fallbackFingerprint)
|
||||
? computeFingerprint(data.statement.accountNumber, tx, sourcePosition)
|
||||
const occurrence = fallbackFingerprintOccurrences.get(fallbackFingerprint) ?? 0;
|
||||
fallbackFingerprintOccurrences.set(fallbackFingerprint, occurrence + 1);
|
||||
const fp = !tx.sourceId && occurrence > 0
|
||||
? computeFingerprint(data.statement.accountNumber, tx, occurrence)
|
||||
: fallbackFingerprint;
|
||||
const isCashbackCommissionImport =
|
||||
tx.amountSigned === 0 &&
|
||||
|
||||
@@ -124,7 +124,7 @@ accountNumber|operationAt|amountSigned|commission|normalizedDescription[|sourceP
|
||||
```
|
||||
|
||||
- `normalizedDescription` — `description` после `trim`.
|
||||
- `sourcePosition` — индекс операции в массиве `transactions`; добавляется, только если одинаковые операции без `sourceId` повторяются в одном файле.
|
||||
- `sourcePosition` — порядковый номер повторяющейся операции в массиве `transactions`; добавляется, только если одинаковые операции без `sourceId` повторяются в одном файле.
|
||||
- Суммы подставляются в том виде, в котором пришли в JSON (числовое представление).
|
||||
- Разделитель `|` исключает коллизии при склейке полей разной длины.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user