diff --git a/backend/package.json b/backend/package.json index a2e634f..b401d79 100644 --- a/backend/package.json +++ b/backend/package.json @@ -1,6 +1,6 @@ { "name": "@family-budget/backend", - "version": "0.10.3", + "version": "0.10.4", "private": true, "scripts": { "dev": "tsx watch src/app.ts", diff --git a/backend/src/services/import.integration.test.ts b/backend/src/services/import.integration.test.ts index e12ae3e..862e5a7 100644 --- a/backend/src/services/import.integration.test.ts +++ b/backend/src/services/import.integration.test.ts @@ -8,6 +8,12 @@ const makeStatement = (sourceIds: string[]) => ({ transactions: sourceIds.map((sourceId) => ({ operationAt: '2026-08-20T10:00:00+03:00', amountSigned: 100, commission: 0, description: 'Пополнение', sourceId })), }); +const duplicateStatement = { + schemaVersion: '1.0', bank: 'TEST', + statement: { accountNumber: 'fingerprint-test', currency: 'RUB', openingBalance: 0, closingBalance: 200, exportedAt: '2026-08-20T12:00:00+03:00' }, + transactions: Array.from({ length: 2 }, () => ({ operationAt: '2026-08-20T00:00:00+03:00', amountSigned: 100, commission: 0, description: 'Пополнение' })), +}; + async function run(): Promise { try { await importStatement(makeStatement(['first', 'first-second'])); @@ -18,11 +24,21 @@ async function run(): Promise { await importStatement(makeStatement(['second'])); const result = await pool.query('SELECT account_type, status FROM accounts WHERE id = $1', [account.rows[0].id]); assert.deepEqual(result.rows[0], { account_type: 'savings', status: 'closed' }); + const duplicateResult = await importStatement(duplicateStatement); + if ('status' in duplicateResult) throw new Error(duplicateResult.message); + assert.deepEqual(duplicateResult, { + accountId: duplicateResult.accountId, + isNewAccount: true, + accountNumberMasked: 'fingerp******test', + imported: 2, + duplicatesSkipped: 0, + totalInFile: 2, + }); console.log('import metadata SQL: OK'); } finally { - await pool.query('DELETE FROM transactions WHERE account_id IN (SELECT id FROM accounts WHERE bank = \'TEST\' AND account_number = \'metadata-test\')'); - await pool.query("DELETE FROM imports WHERE account_id IN (SELECT id FROM accounts WHERE bank = 'TEST' AND account_number = 'metadata-test')"); - await pool.query("DELETE FROM accounts WHERE bank = 'TEST' AND account_number = 'metadata-test'"); + await pool.query("DELETE FROM transactions WHERE account_id IN (SELECT id FROM accounts WHERE bank = 'TEST' AND account_number IN ('metadata-test', 'fingerprint-test'))"); + await pool.query("DELETE FROM imports WHERE account_id IN (SELECT id FROM accounts WHERE bank = 'TEST' AND account_number IN ('metadata-test', 'fingerprint-test'))"); + await pool.query("DELETE FROM accounts WHERE bank = 'TEST' AND account_number IN ('metadata-test', 'fingerprint-test')"); await pool.end(); } } diff --git a/backend/src/services/import.test.ts b/backend/src/services/import.test.ts index 3af8b4f..29c0260 100644 --- a/backend/src/services/import.test.ts +++ b/backend/src/services/import.test.ts @@ -1,10 +1,12 @@ import assert from 'node:assert/strict'; -import { determineDirection } from './import'; +import { computeFingerprint, determineDirection } from './import'; assert.equal(determineDirection(1, 'Перечисление средств на счет N 123 со счета N 456'), 'transfer'); assert.equal(determineDirection(1, 'Перечисление средств на вклад N 123'), 'transfer'); assert.equal(determineDirection(-1, 'Перечисление суммы вклада при закрытии'), 'transfer'); assert.equal(determineDirection(-1, 'Оплата покупки'), 'expense'); assert.equal(determineDirection(1, 'Выплата процентов'), 'income'); +const duplicateTransaction = { operationAt: '2026-08-20T00:00:00+03:00', amountSigned: 100, commission: 0, description: 'Пополнение' }; +assert.notEqual(computeFingerprint('fingerprint-test', duplicateTransaction, 0), computeFingerprint('fingerprint-test', duplicateTransaction, 1)); console.log('import direction: OK'); diff --git a/backend/src/services/import.ts b/backend/src/services/import.ts index e027823..3d44e52 100644 --- a/backend/src/services/import.ts +++ b/backend/src/services/import.ts @@ -14,9 +14,10 @@ const TRANSFER_PHRASES = [ const CASHBACK_KEYWORD = 'зачисление'; const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; -function computeFingerprint( +export function computeFingerprint( accountNumber: string, tx: { operationAt: string; amountSigned: number; commission: number; description: string; sourceId?: string }, + sourcePosition?: number, ): string { if (tx.sourceId) { const raw = [accountNumber, tx.sourceId.trim()].join('|'); @@ -29,6 +30,7 @@ function computeFingerprint( String(tx.amountSigned), String(tx.commission), tx.description.trim(), + ...(sourcePosition === undefined ? [] : [String(sourcePosition)]), ].join('|'); const hash = crypto.createHash('sha256').update(raw, 'utf-8').digest('hex'); return `sha256:${hash}`; @@ -139,7 +141,7 @@ function validateSemantics(data: StatementFile): ValidationError | null { const operationIds = new Set(); for (let i = 0; i < data.transactions.length; i++) { const fp = computeFingerprint(data.statement.accountNumber, data.transactions[i]); - if (fps.has(fp)) { + if (data.transactions[i].sourceId && fps.has(fp)) { return { status: 422, error: 'VALIDATION_ERROR', message: `Duplicate fingerprint found within file at transaction index ${i}` }; } fps.add(fp); @@ -165,6 +167,15 @@ export async function importStatement( const data = body as StatementFile; const semErr = validateSemantics(data); if (semErr) return semErr; + const fallbackFingerprints = new Set(); + const duplicateFallbackFingerprints = new Set(); + for (const tx of data.transactions) { + if (!tx.sourceId) { + const fp = computeFingerprint(data.statement.accountNumber, tx); + if (fallbackFingerprints.has(fp)) duplicateFallbackFingerprints.add(fp); + fallbackFingerprints.add(fp); + } + } const client = await db.connect(); try { @@ -223,7 +234,10 @@ export async function importStatement( const insertedIds: number[] = []; for (const [sourcePosition, tx] of data.transactions.entries()) { - const fp = computeFingerprint(data.statement.accountNumber, tx); + const fallbackFingerprint = computeFingerprint(data.statement.accountNumber, tx); + const fp = !tx.sourceId && duplicateFallbackFingerprints.has(fallbackFingerprint) + ? computeFingerprint(data.statement.accountNumber, tx, sourcePosition) + : fallbackFingerprint; const isCashbackCommissionImport = tx.amountSigned === 0 && tx.commission > 0 && diff --git a/docs/backlog/api_import.md b/docs/backlog/api_import.md index 9c21536..1f60ca0 100644 --- a/docs/backlog/api_import.md +++ b/docs/backlog/api_import.md @@ -91,7 +91,7 @@ - `statement.currency` соответствует допустимому коду валюты (MVP: `"RUB"`). - `operationAt` у всех транзакций — валидная дата (парсится без ошибок). -- Отсутствуют дубликаты fingerprint внутри одного файла. +- Повторяющиеся `sourceId` внутри одного файла отклоняются; одинаковые операции без `sourceId` различаются по позиции в массиве `transactions`. Ответ при ошибке: @@ -120,10 +120,11 @@ Для каждой транзакции вычисляется SHA-256 от полей, соединённых разделителем `|`: ```text -accountNumber|operationAt|amountSigned|commission|normalizedDescription +accountNumber|operationAt|amountSigned|commission|normalizedDescription[|sourcePosition] ``` - `normalizedDescription` — `description` после `trim`. +- `sourcePosition` — индекс операции в массиве `transactions`; добавляется, только если одинаковые операции без `sourceId` повторяются в одном файле. - Суммы подставляются в том виде, в котором пришли в JSON (числовое представление). - Разделитель `|` исключает коллизии при склейке полей разной длины.