fix: make broker XLSX import atomic

2026-08-26 23:42:26 +03:00
parent 000c77f162
commit 12d096beeb
6 changed files with 86 additions and 18 deletions
--- a/backend/package.json
+++ b/backend/package.json
@@ -16,6 +16,7 @@
    "test:analytics:db": "NODE_ENV=test tsx src/services/analytics.integration.test.ts",
    "test:import:db": "NODE_ENV=test tsx src/services/import.integration.test.ts",
    "test:import:direction": "tsx src/services/import.test.ts",
+    "test:broker:xlsx": "tsx src/services/vtbBrokerXlsx.test.ts",
    "test:llm": "tsx src/scripts/testLlm.ts"
  },
  "dependencies": {
--- a/backend/src/routes/import.ts
+++ b/backend/src/routes/import.ts
@@ -1,5 +1,6 @@
 import { Router } from 'express';
 import multer from 'multer';
+import { pool } from '../db/pool';
 import { asyncHandler } from '../utils';
 import { importStatement, isValidationError } from '../services/import';
 import {
@@ -109,13 +110,24 @@ router.post(
      res.status(422).json({ error: 'VALIDATION_ERROR', message: error instanceof Error ? error.message : 'Не удалось обработать XLSX-отчёт' });
      return;
    }
-    const portfolio = await importPortfolio(converted.portfolio);
-    const cash = await importStatement(converted.cash);
-    if (isValidationError(cash)) {
-      res.status(cash.status).json({ error: cash.error, message: cash.message });
-      return;
+    const client = await pool.connect();
+    try {
+      await client.query('BEGIN');
+      const portfolio = await importPortfolio(converted.portfolio, pool, client);
+      const cash = await importStatement(converted.cash, pool, client);
+      if (isValidationError(cash)) {
+        await client.query('ROLLBACK');
+        res.status(cash.status).json({ error: cash.error, message: cash.message });
+        return;
+      }
+      await client.query('COMMIT');
+      res.json({ portfolio, cash });
+    } catch (error) {
+      await client.query('ROLLBACK');
+      throw error;
+    } finally {
+      client.release();
    }
-    res.json({ portfolio, cash });
  }),
 );

--- a/backend/src/services/import.ts
+++ b/backend/src/services/import.ts
@@ -1,4 +1,5 @@
 import crypto from 'crypto';
+import type { PoolClient } from 'pg';
 import { pool } from '../db/pool';
 import { maskAccountNumber } from '../utils';
 import type { StatementFile, ImportStatementResponse } from '@family-budget/shared';
@@ -160,6 +161,7 @@ function validateSemantics(data: StatementFile): ValidationError | null {
 export async function importStatement(
  body: unknown,
  db: Pick<typeof pool, 'connect'> = pool,
+  transactionClient?: PoolClient,
 ): Promise<ImportStatementResponse | ValidationError> {
  const structErr = validateStructure(body);
  if (structErr) return structErr;
@@ -168,9 +170,10 @@ export async function importStatement(
  const semErr = validateSemantics(data);
  if (semErr) return semErr;

-  const client = await db.connect();
+  const client = transactionClient ?? await db.connect();
+  const ownsTransaction = transactionClient == null;
  try {
-    await client.query('BEGIN');
+    if (ownsTransaction) await client.query('BEGIN');

    // Find or create account
    let accountId: number;
@@ -300,7 +303,7 @@ export async function importStatement(
      }
    }

-    await client.query('COMMIT');
+    if (ownsTransaction) await client.query('COMMIT');

    return {
      accountId,
@@ -311,10 +314,10 @@ export async function importStatement(
      totalInFile: data.transactions.length,
    };
  } catch (err) {
-    await client.query('ROLLBACK');
+    if (ownsTransaction) await client.query('ROLLBACK');
    throw err;
  } finally {
-    client.release();
+    if (ownsTransaction) client.release();
  }
 }

--- a/backend/src/services/portfolio.ts
+++ b/backend/src/services/portfolio.ts
@@ -1,4 +1,5 @@
 import crypto from 'crypto';
+import type { PoolClient } from 'pg';
 import { pool } from '../db/pool';
 import type { ImportPortfolioResponse, PortfolioFile, PortfolioTrade } from '@family-budget/shared';

@@ -45,13 +46,14 @@ export function validatePortfolio(body: unknown): asserts body is PortfolioFile
  }
 }

-export async function importPortfolio(body: unknown, db: Pick<typeof pool, 'connect'> = pool): Promise<ImportPortfolioResponse> {
+export async function importPortfolio(body: unknown, db: Pick<typeof pool, 'connect'> = pool, transactionClient?: PoolClient): Promise<ImportPortfolioResponse> {
  validatePortfolio(body);
  const data = body;
  const sourceHash = crypto.createHash('sha256').update(JSON.stringify(data)).digest('hex');
-  const client = await db.connect();
+  const client = transactionClient ?? await db.connect();
+  const ownsTransaction = transactionClient == null;
  try {
-    await client.query('BEGIN');
+    if (ownsTransaction) await client.query('BEGIN');
    const accountResult = await client.query(
      `INSERT INTO accounts (bank, account_number, currency, account_type)
       VALUES ($1, $2, 'RUB', 'brokerage')
@@ -69,7 +71,7 @@ export async function importPortfolio(body: unknown, db: Pick<typeof pool, 'conn
    );
    if (reportResult.rows.length === 0) {
      const existing = await client.query('SELECT id FROM portfolio_reports WHERE account_id = $1 AND source_hash = $2', [accountId, sourceHash]);
-      await client.query('COMMIT');
+      if (ownsTransaction) await client.query('COMMIT');
      return { accountId, reportId: Number(existing.rows[0].id), importedTrades: 0, duplicateTrades: data.trades.length, positions: 0 };
    }
    const reportId = Number(reportResult.rows[0].id);
@@ -92,12 +94,12 @@ export async function importPortfolio(body: unknown, db: Pick<typeof pool, 'conn
      );
      importedTrades += result.rowCount ?? 0;
    }
-    await client.query('COMMIT');
+    if (ownsTransaction) await client.query('COMMIT');
    return { accountId, reportId, importedTrades, duplicateTrades: data.trades.length - importedTrades, positions: data.positions.length };
  } catch (error) {
-    await client.query('ROLLBACK');
+    if (ownsTransaction) await client.query('ROLLBACK');
    throw error;
  } finally {
-    client.release();
+    if (ownsTransaction) client.release();
  }
 }
--- a/backend/src/services/vtbBrokerXlsx.test.ts
+++ b/backend/src/services/vtbBrokerXlsx.test.ts
@@ -0,0 +1,45 @@
+import assert from 'node:assert/strict';
+import zlib from 'zlib';
+import { convertVtbBrokerXlsx } from './vtbBrokerXlsx';
+
+function zip(files: Record<string, string>): Buffer {
+  let offset = 0;
+  const locals: Buffer[] = [];
+  const central: Buffer[] = [];
+  for (const [name, value] of Object.entries(files)) {
+    const nameBuffer = Buffer.from(name);
+    const data = zlib.deflateRawSync(Buffer.from(value));
+    const local = Buffer.alloc(30);
+    local.writeUInt32LE(0x04034b50, 0); local.writeUInt16LE(20, 4); local.writeUInt16LE(8, 8);
+    local.writeUInt32LE(data.length, 18); local.writeUInt32LE(Buffer.byteLength(value), 22);
+    local.writeUInt16LE(nameBuffer.length, 26);
+    const header = Buffer.alloc(46);
+    header.writeUInt32LE(0x02014b50, 0); header.writeUInt16LE(20, 4); header.writeUInt16LE(20, 6); header.writeUInt16LE(8, 10);
+    header.writeUInt32LE(data.length, 20); header.writeUInt32LE(Buffer.byteLength(value), 24); header.writeUInt16LE(nameBuffer.length, 28); header.writeUInt32LE(offset, 42);
+    locals.push(local, nameBuffer, data);
+    central.push(header, nameBuffer);
+    offset += local.length + nameBuffer.length + data.length;
+  }
+  const centralBuffer = Buffer.concat(central);
+  const end = Buffer.alloc(22);
+  end.writeUInt32LE(0x06054b50, 0); end.writeUInt16LE(Object.keys(files).length, 8); end.writeUInt16LE(Object.keys(files).length, 10); end.writeUInt32LE(centralBuffer.length, 12); end.writeUInt32LE(offset, 16);
+  return Buffer.concat([...locals, centralBuffer, end]);
+}
+
+const strings = ['Период с 01.08.2026 по 31.08.2026', '12345678901234567890 (RUR)', 'Дата формирования отчета', 'Движение денежных средств', 'Пополнение', 'Отчёт об остатках ценных бумаг', 'Движение ценных бумаг', 'Отчёт об остатках денежных средств', 'Заключенные в отчетном периоде сделки с ценными бумагами', 'Bond, RU0000000001', 'Покупка', 'Завершенные в отчетном периоде сделки с ценными бумагами'];
+const shared = `<sst>${strings.map((value) => `<si><t>${value}</t></si>`).join('')}</sst>`;
+const cell = (ref: string, value: string | number, sharedString = false) => `<c r="${ref}"${sharedString ? ' t="s"' : ''}><v>${value}</v></c>`;
+const row = (number: number, cells: string[]) => `<row r="${number}">${cells.join('')}</row>`;
+const sheet = `<worksheet><sheetData>${[
+  row(1, [cell('A1', 0, true)]), row(2, [cell('A2', 1, true)]), row(3, [cell('A3', 2, true), cell('B3', 45900)]),
+  row(10, [cell('A10', 3, true)]), row(11, [cell('B11', 45900), cell('C11', 1), cell('J11', 4, true)]),
+  row(12, [cell('A12', 5, true)]), row(13, [cell('A13', 6, true)]), row(15, [cell('A15', 7, true)]), row(16, []), row(17, []), row(18, [cell('L18', 0), cell('AF18', 1)]),
+  row(20, [cell('A20', 8, true)]), row(21, [cell('B21', 9, true), cell('C21', 45900), cell('F21', 10, true), cell('H21', 1)]), row(22, [cell('A22', 11, true)]),
+].join('')}</sheetData></worksheet>`;
+
+const result = convertVtbBrokerXlsx(zip({ 'xl/sharedStrings.xml': shared, 'xl/worksheets/sheet1.xml': sheet }));
+assert.equal(result.cash.transactions.length, 1);
+assert.equal(result.cash.statement.closingBalance, 100);
+assert.equal(result.portfolio.trades.length, 1);
+assert.throws(() => convertVtbBrokerXlsx(Buffer.from('not a zip')), /XLSX/);
+console.log('VTB broker XLSX: OK');
--- a/backend/src/services/vtbBrokerXlsx.ts
+++ b/backend/src/services/vtbBrokerXlsx.ts
@@ -3,6 +3,7 @@ import zlib from 'zlib';
 import type { PortfolioFile, StatementFile } from '@family-budget/shared';

 type Row = [number, Record<string, string>];
+const MAX_XLSX_UNCOMPRESSED_BYTES = 30 * 1024 * 1024;

 function xmlText(value: string): string {
  return value.replace(/<[^>]+>/g, '').replace(/&amp;/g, '&').replace(/&lt;/g, '<').replace(/&gt;/g, '>').replace(/&#(d+);/g, (_, code) => String.fromCharCode(Number(code))).replace(/\s+/g, ' ').trim();
@@ -14,15 +15,19 @@ function zipFiles(buffer: Buffer): Map<string, Buffer> {
  const files = new Map<string, Buffer>();
  let offset = buffer.readUInt32LE(end + 16);
  const count = buffer.readUInt16LE(end + 10);
+  let totalSize = 0;
  for (let i = 0; i < count; i++) {
    if (buffer.readUInt32LE(offset) !== 0x02014b50) throw new Error('Повреждён XLSX-архив');
    const method = buffer.readUInt16LE(offset + 10);
    const compressedSize = buffer.readUInt32LE(offset + 20);
+    const uncompressedSize = buffer.readUInt32LE(offset + 24);
    const nameLength = buffer.readUInt16LE(offset + 28);
    const extraLength = buffer.readUInt16LE(offset + 30);
    const commentLength = buffer.readUInt16LE(offset + 32);
    const localOffset = buffer.readUInt32LE(offset + 42);
    const name = buffer.subarray(offset + 46, offset + 46 + nameLength).toString('utf8');
+    totalSize += uncompressedSize;
+    if (totalSize > MAX_XLSX_UNCOMPRESSED_BYTES) throw new Error('XLSX-отчёт слишком большой после распаковки');
    if (buffer.readUInt32LE(localOffset) !== 0x04034b50) throw new Error('Повреждён XLSX-архив');
    const localNameLength = buffer.readUInt16LE(localOffset + 26);
    const localExtraLength = buffer.readUInt16LE(localOffset + 28);