Files
family_budget/backend/src/services/import.ts
Anton 01b1f26553 feat(imports): import history and delete by import
Track imports in DB, show history in Data section, allow deleting
transactions of a specific import instead of clearing all.
2026-03-16 17:46:15 +03:00

250 lines
8.8 KiB
TypeScript

import crypto from 'crypto';
import { pool } from '../db/pool';
import { maskAccountNumber } from '../utils';
import type { StatementFile, ImportStatementResponse } from '@family-budget/shared';
const TRANSFER_PHRASES = [
'перевод между своими счетами',
'перевод средств на счет',
'внутри втб',
];
function computeFingerprint(
accountNumber: string,
tx: { operationAt: string; amountSigned: number; commission: number; description: string },
): string {
const raw = [
accountNumber,
tx.operationAt,
String(tx.amountSigned),
String(tx.commission),
tx.description.trim(),
].join('|');
const hash = crypto.createHash('sha256').update(raw, 'utf-8').digest('hex');
return `sha256:${hash}`;
}
function determineDirection(amountSigned: number, description: string): string {
const lower = description.toLowerCase();
for (const phrase of TRANSFER_PHRASES) {
if (lower.includes(phrase)) return 'transfer';
}
return amountSigned > 0 ? 'income' : 'expense';
}
interface ValidationError {
status: number;
error: string;
message: string;
}
const ISO_WITH_OFFSET = /\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}([+-]\d{2}:\d{2}|Z)/;
function validateStructure(body: unknown): ValidationError | null {
const b = body as Record<string, unknown>;
if (!b || typeof b !== 'object') {
return { status: 400, error: 'BAD_REQUEST', message: 'Body must be a JSON object' };
}
if (b.schemaVersion !== '1.0') {
return { status: 400, error: 'BAD_REQUEST', message: "schemaVersion must be '1.0'" };
}
if (typeof b.bank !== 'string' || !b.bank) {
return { status: 400, error: 'BAD_REQUEST', message: 'bank is required and must be a non-empty string' };
}
const st = b.statement as Record<string, unknown> | undefined;
if (!st || typeof st !== 'object') {
return { status: 400, error: 'BAD_REQUEST', message: 'statement is required' };
}
for (const f of ['accountNumber', 'currency'] as const) {
if (typeof st[f] !== 'string' || !(st[f] as string)) {
return { status: 400, error: 'BAD_REQUEST', message: `statement.${f} is required and must be a non-empty string` };
}
}
for (const f of ['openingBalance', 'closingBalance'] as const) {
if (typeof st[f] !== 'number' || !Number.isInteger(st[f])) {
return { status: 400, error: 'BAD_REQUEST', message: `statement.${f} must be an integer` };
}
}
if (typeof st.exportedAt !== 'string' || !ISO_WITH_OFFSET.test(st.exportedAt as string)) {
return { status: 400, error: 'BAD_REQUEST', message: 'statement.exportedAt must be ISO 8601 with offset' };
}
const txs = b.transactions;
if (!Array.isArray(txs) || txs.length === 0) {
return { status: 400, error: 'BAD_REQUEST', message: 'transactions must be a non-empty array' };
}
for (let i = 0; i < txs.length; i++) {
const t = txs[i];
if (!t || typeof t !== 'object') {
return { status: 400, error: 'BAD_REQUEST', message: `transactions[${i}] must be an object` };
}
if (typeof t.operationAt !== 'string' || !ISO_WITH_OFFSET.test(t.operationAt)) {
return { status: 400, error: 'BAD_REQUEST', message: `transactions[${i}].operationAt must be ISO 8601 with offset` };
}
if (typeof t.amountSigned !== 'number' || !Number.isInteger(t.amountSigned) || t.amountSigned === 0) {
return { status: 400, error: 'BAD_REQUEST', message: `transactions[${i}].amountSigned must be a non-zero integer` };
}
if (typeof t.commission !== 'number' || !Number.isInteger(t.commission) || t.commission < 0) {
return { status: 400, error: 'BAD_REQUEST', message: `transactions[${i}].commission must be a non-negative integer` };
}
if (typeof t.description !== 'string' || !t.description) {
return { status: 400, error: 'BAD_REQUEST', message: `transactions[${i}].description must be a non-empty string` };
}
}
return null;
}
function validateSemantics(data: StatementFile): ValidationError | null {
if (data.statement.currency !== 'RUB') {
return { status: 422, error: 'VALIDATION_ERROR', message: `Unsupported currency: ${data.statement.currency}` };
}
for (let i = 0; i < data.transactions.length; i++) {
if (isNaN(Date.parse(data.transactions[i].operationAt))) {
return { status: 422, error: 'VALIDATION_ERROR', message: `Invalid date at transaction index ${i}` };
}
}
const fps = new Set<string>();
for (let i = 0; i < data.transactions.length; i++) {
const fp = computeFingerprint(data.statement.accountNumber, data.transactions[i]);
if (fps.has(fp)) {
return { status: 422, error: 'VALIDATION_ERROR', message: `Duplicate fingerprint found within file at transaction index ${i}` };
}
fps.add(fp);
}
return null;
}
export async function importStatement(
body: unknown,
): Promise<ImportStatementResponse | ValidationError> {
const structErr = validateStructure(body);
if (structErr) return structErr;
const data = body as StatementFile;
const semErr = validateSemantics(data);
if (semErr) return semErr;
const client = await pool.connect();
try {
await client.query('BEGIN');
// Find or create account
let accountId: number;
let isNewAccount = false;
const accResult = await client.query(
'SELECT id FROM accounts WHERE bank = $1 AND account_number = $2',
[data.bank, data.statement.accountNumber],
);
if (accResult.rows.length > 0) {
accountId = Number(accResult.rows[0].id);
} else {
const ins = await client.query(
'INSERT INTO accounts (bank, account_number, currency) VALUES ($1, $2, $3) RETURNING id',
[data.bank, data.statement.accountNumber, data.statement.currency],
);
accountId = Number(ins.rows[0].id);
isNewAccount = true;
}
// Create import record (counts updated after loop)
const accountNumberMasked = maskAccountNumber(data.statement.accountNumber);
const importResult = await client.query(
`INSERT INTO imports (account_id, bank, account_number_masked, imported_count, duplicates_skipped, total_in_file)
VALUES ($1, $2, $3, 0, 0, $4)
RETURNING id`,
[accountId, data.bank, accountNumberMasked, data.transactions.length],
);
const importId = Number(importResult.rows[0].id);
// Insert transactions
const insertedIds: number[] = [];
for (const tx of data.transactions) {
const fp = computeFingerprint(data.statement.accountNumber, tx);
const dir = determineDirection(tx.amountSigned, tx.description);
const result = await client.query(
`INSERT INTO transactions
(account_id, operation_at, amount_signed, commission, description, direction, fingerprint, category_id, is_category_confirmed, import_id)
VALUES ($1, $2, $3, $4, $5, $6, $7, NULL, FALSE, $8)
ON CONFLICT (account_id, fingerprint) DO NOTHING
RETURNING id`,
[accountId, tx.operationAt, tx.amountSigned, tx.commission, tx.description, dir, fp, importId],
);
if (result.rows.length > 0) {
insertedIds.push(Number(result.rows[0].id));
}
}
// Update import record with actual counts
const duplicatesSkipped = data.transactions.length - insertedIds.length;
await client.query(
`UPDATE imports SET imported_count = $1, duplicates_skipped = $2 WHERE id = $3`,
[insertedIds.length, duplicatesSkipped, importId],
);
// Auto-categorize newly inserted transactions
if (insertedIds.length > 0) {
await client.query(
`UPDATE transactions t
SET category_id = sub.category_id,
is_category_confirmed = NOT sub.requires_confirmation,
updated_at = NOW()
FROM (
SELECT DISTINCT ON (t2.id)
t2.id AS tx_id,
cr.category_id,
cr.requires_confirmation
FROM transactions t2
JOIN category_rules cr
ON cr.is_active = TRUE
AND (
(cr.match_type = 'contains' AND t2.description ILIKE '%' || cr.pattern || '%')
OR (cr.match_type = 'starts_with' AND t2.description ILIKE cr.pattern || '%')
)
WHERE t2.id = ANY($1::bigint[])
ORDER BY t2.id, cr.priority DESC, cr.id ASC
) sub
WHERE t.id = sub.tx_id`,
[insertedIds],
);
}
await client.query('COMMIT');
return {
accountId,
isNewAccount,
accountNumberMasked,
imported: insertedIds.length,
duplicatesSkipped: data.transactions.length - insertedIds.length,
totalInFile: data.transactions.length,
};
} catch (err) {
await client.query('ROLLBACK');
throw err;
} finally {
client.release();
}
}
export function isValidationError(r: unknown): r is ValidationError {
return (
typeof r === 'object' &&
r !== null &&
'status' in r &&
'error' in r &&
'message' in r
);
}