From 000c77f16250dc5fcfd77948a44600af7482c07b Mon Sep 17 00:00:00 2001 From: admin Date: Wed, 26 Aug 2026 23:36:32 +0300 Subject: [PATCH] feat: import VTB broker XLSX reports --- backend/package.json | 2 +- backend/src/routes/import.ts | 32 +++++ backend/src/services/vtbBrokerXlsx.ts | 156 ++++++++++++++++++++++++ frontend/package.json | 2 +- frontend/src/api/portfolio.ts | 8 +- frontend/src/components/ImportModal.tsx | 49 ++++++-- shared/package.json | 2 +- shared/src/types/import.ts | 5 + shared/src/types/index.ts | 1 + 9 files changed, 241 insertions(+), 16 deletions(-) create mode 100644 backend/src/services/vtbBrokerXlsx.ts diff --git a/backend/package.json b/backend/package.json index 29d7e43..d09843c 100644 --- a/backend/package.json +++ b/backend/package.json @@ -1,6 +1,6 @@ { "name": "@family-budget/backend", - "version": "0.10.5", + "version": "0.11.0", "private": true, "scripts": { "dev": "tsx watch src/app.ts", diff --git a/backend/src/routes/import.ts b/backend/src/routes/import.ts index d1188b1..76e5d2b 100644 --- a/backend/src/routes/import.ts +++ b/backend/src/routes/import.ts @@ -6,6 +6,8 @@ import { convertPdfToStatement, isPdfConversionError, } from '../services/pdfToStatement'; +import { importPortfolio } from '../services/portfolio'; +import { convertVtbBrokerXlsx } from '../services/vtbBrokerXlsx'; const upload = multer({ storage: multer.memoryStorage(), @@ -28,6 +30,10 @@ function isJsonFile(file: { mimetype: string; originalname: string }): boolean { ); } +function isXlsxFile(file: { mimetype: string; originalname: string }): boolean { + return file.mimetype === 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' || file.originalname.toLowerCase().endsWith('.xlsx'); +} + const router = Router(); router.post( @@ -87,4 +93,30 @@ router.post( }), ); +router.post( + '/broker', + upload.single('file'), + asyncHandler(async (req, res) => { + const file = req.file; + if (!file || !isXlsxFile(file)) { + res.status(400).json({ error: 'BAD_REQUEST', message: 'Допустим только XLSX-отчёт ВТБ Брокер' }); + return; + } + let converted; + try { + converted = convertVtbBrokerXlsx(file.buffer); + } catch (error) { + res.status(422).json({ error: 'VALIDATION_ERROR', message: error instanceof Error ? error.message : 'Не удалось обработать XLSX-отчёт' }); + return; + } + const portfolio = await importPortfolio(converted.portfolio); + const cash = await importStatement(converted.cash); + if (isValidationError(cash)) { + res.status(cash.status).json({ error: cash.error, message: cash.message }); + return; + } + res.json({ portfolio, cash }); + }), +); + export default router; diff --git a/backend/src/services/vtbBrokerXlsx.ts b/backend/src/services/vtbBrokerXlsx.ts new file mode 100644 index 0000000..0cde4e7 --- /dev/null +++ b/backend/src/services/vtbBrokerXlsx.ts @@ -0,0 +1,156 @@ +import crypto from 'crypto'; +import zlib from 'zlib'; +import type { PortfolioFile, StatementFile } from '@family-budget/shared'; + +type Row = [number, Record]; + +function xmlText(value: string): string { + return value.replace(/<[^>]+>/g, '').replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>').replace(/&#(d+);/g, (_, code) => String.fromCharCode(Number(code))).replace(/\s+/g, ' ').trim(); +} + +function zipFiles(buffer: Buffer): Map { + const end = buffer.lastIndexOf(Buffer.from([0x50, 0x4b, 0x05, 0x06])); + if (end < 0) throw new Error('Файл не является XLSX-архивом'); + const files = new Map(); + let offset = buffer.readUInt32LE(end + 16); + const count = buffer.readUInt16LE(end + 10); + for (let i = 0; i < count; i++) { + if (buffer.readUInt32LE(offset) !== 0x02014b50) throw new Error('Повреждён XLSX-архив'); + const method = buffer.readUInt16LE(offset + 10); + const compressedSize = buffer.readUInt32LE(offset + 20); + const nameLength = buffer.readUInt16LE(offset + 28); + const extraLength = buffer.readUInt16LE(offset + 30); + const commentLength = buffer.readUInt16LE(offset + 32); + const localOffset = buffer.readUInt32LE(offset + 42); + const name = buffer.subarray(offset + 46, offset + 46 + nameLength).toString('utf8'); + if (buffer.readUInt32LE(localOffset) !== 0x04034b50) throw new Error('Повреждён XLSX-архив'); + const localNameLength = buffer.readUInt16LE(localOffset + 26); + const localExtraLength = buffer.readUInt16LE(localOffset + 28); + const data = buffer.subarray(localOffset + 30 + localNameLength + localExtraLength, localOffset + 30 + localNameLength + localExtraLength + compressedSize); + files.set(name, method === 0 ? data : method === 8 ? zlib.inflateRawSync(data) : (() => { throw new Error('Неподдерживаемое сжатие XLSX'); })()); + offset += 46 + nameLength + extraLength + commentLength; + } + return files; +} + +function readRows(buffer: Buffer): Row[] { + const files = zipFiles(buffer); + const sharedXml = files.get('xl/sharedStrings.xml')?.toString('utf8'); + const sheetXml = files.get('xl/worksheets/sheet1.xml')?.toString('utf8'); + if (!sharedXml || !sheetXml) throw new Error('В XLSX не найдены данные отчёта'); + const shared = [...sharedXml.matchAll(/([\s\S]*?)<\/si>/g)].map((match) => xmlText(match[1])); + return [...sheetXml.matchAll(/]*\br="(\d+)"[^>]*>([\s\S]*?)<\/row>/g)].map((row) => { + const cells: Record = {}; + for (const cell of row[2].matchAll(/]*\br="([A-Z]+)\d+"([^>]*)>([\s\S]*?)<\/c>/g)) { + const value = cell[3].match(/([\s\S]*?)<\/v>/)?.[1] ?? ''; + cells[cell[1]] = /\bt="s"/.test(cell[2]) && value ? shared[Number(value)] : xmlText(value); + } + return [Number(row[1]), cells]; + }); +} + +function text(value: unknown): string { + return String(value ?? '').replace(/\s+/g, ' ').trim(); +} + +function rowText(cells: Record): string { + return text(Object.values(cells).join(' ')); +} + +function findRow(rows: Row[], phrase: string, start = 0): number { + const index = rows.findIndex(([, cells], i) => i >= start && rowText(cells).toLowerCase().includes(phrase.toLowerCase())); + if (index < 0) throw new Error(`Не найден раздел: ${phrase}`); + return index; +} + +function excelDate(value: string | undefined): string | null { + if (!value) return null; + const date = new Date(Date.UTC(1899, 11, 30) + Number(value) * 86_400_000); + if (Number.isNaN(date.valueOf())) return null; + return `${date.toISOString().slice(0, 19)}+03:00`; +} + +function kopecks(value: string | undefined): number { + return value ? Math.round(Number(value.replace(',', '.')) * 100) : 0; +} + +function metadata(rows: Row[]): { account: string; period: [string, string]; reportedAt: string | null } { + const period = rows.map(([, cells]) => rowText(cells)).join(' ').match(/период с (\d{2}\.\d{2}\.\d{4}) по (\d{2}\.\d{2}\.\d{4})/i); + let account: string | null = null; + let reportedAt: string | null = null; + for (const [, cells] of rows.slice(0, 35)) { + const joined = rowText(cells); + account ??= joined.match(/(\d{20})\s*\(RUR\)/)?.[1] ?? null; + if (joined.includes('Дата формирования отчета')) { + reportedAt ??= Object.values(cells).map(excelDate).find(Boolean) ?? null; + } + } + if (!account || !period) throw new Error('Не удалось определить счёт или период отчёта'); + return { account, period: [period[1], period[2]], reportedAt }; +} + +function isoDate(value: string): string { + const [day, month, year] = value.split('.'); + return `${year}-${month}-${day}`; +} + +export function convertVtbBrokerXlsx(buffer: Buffer): { cash: StatementFile; portfolio: PortfolioFile } { + const rows = readRows(buffer); + const { account, period, reportedAt } = metadata(rows); + const holdingsStart = findRow(rows, 'Отчёт об остатках ценных бумаг'); + const movementStart = findRow(rows, 'Движение ценных бумаг', holdingsStart); + const cashStart = findRow(rows, 'Движение денежных средств'); + const tradesStart = findRow(rows, 'Заключенные в отчетном периоде сделки с ценными бумагами'); + const tradesEnd = findRow(rows, 'Завершенные в отчетном периоде сделки с ценными бумагами', tradesStart + 1); + const occurrences = new Map(); + const transactions = rows.slice(cashStart + 1, holdingsStart).flatMap(([, cells]) => { + if (!cells.B || !cells.C || !cells.J) return []; + const operationAt = excelDate(cells.B); + if (!operationAt) return []; + const amountSigned = kopecks(cells.C); + const description = text(`${text(cells.J)}. ${text(cells.P)}`.replace(/\. $/, '')); + const digest = crypto.createHash('sha256').update(`${operationAt}|${amountSigned}|${description}`).digest('hex').slice(0, 16); + const occurrence = (occurrences.get(digest) ?? 0) + 1; + occurrences.set(digest, occurrence); + return [{ operationAt, amountSigned, commission: 0, description, sourceId: `vtb-broker-cash:${digest}:${occurrence}` }]; + }); + if (!transactions.length) throw new Error('Операции движения денежных средств не найдены'); + const balanceStart = findRow(rows, 'Отчёт об остатках денежных средств'); + const openingBalance = kopecks(rows[balanceStart + 3]?.[1].L); + const closingBalance = kopecks(rows[balanceStart + 3]?.[1].AF); + if (openingBalance + transactions.reduce((sum, tx) => sum + tx.amountSigned, 0) !== closingBalance) throw new Error('Баланс cash-операций не сходится с отчётом'); + const positions = rows.slice(holdingsStart + 1, movementStart).flatMap(([, cells]) => { + const instrument = text(cells.B); + if (!instrument || instrument.toLowerCase().startsWith('итого') || !/RU[A-Z0-9]{10}/.test(instrument)) return []; + return [{ instrument, isin: instrument.split(', ').find((part) => /^RU[A-Z0-9]{10}$/.test(part)) ?? null, quantity: cells.L || cells.M || cells.I || cells.J, price: cells.P || null, valuation: cells.AF || cells.AJ || null }]; + }); + const trades = rows.slice(tradesStart + 1, tradesEnd).flatMap(([number, cells]) => { + if (!cells.B || !cells.C || !cells.F) return []; + const concludedAt = excelDate(cells.C); + if (!concludedAt) return []; + const tradeId = text(cells.Z); + return [{ + sourceId: `vtb-broker-trade:${tradeId || number}`, + instrument: text(cells.B), + isin: text(cells.B).split(', ').find((part) => /^RU[A-Z0-9]{10}$/.test(part)) ?? null, + concludedAt, + side: text(cells.F), + quantity: text(cells.H), + priceCurrency: text(cells.I) || null, + price: text(cells.J) || null, + settlementCurrency: text(cells.L) || null, + settlementAmount: text(cells.M) || null, + nkd: text(cells.O) || null, + settlementCommission: text(cells.P) || null, + tradeCommission: text(cells.R) || null, + orderId: text(cells.W) || null, + tradeId: tradeId || null, + venue: text(cells.AK) || null, + comment: text(cells.AN) || null, + }]; + }); + return { + cash: { schemaVersion: '1.0', bank: 'VTB_BROKER', statement: { accountNumber: account, currency: 'RUB', openingBalance, closingBalance, exportedAt: reportedAt ?? transactions[transactions.length - 1].operationAt }, transactions }, + portfolio: { schemaVersion: 'broker-portfolio-1.0', bank: 'VTB_BROKER', accountNumber: account, reportPeriod: { from: isoDate(period[0]), to: isoDate(period[1]) }, reportedAt, positions, trades }, + }; +} diff --git a/frontend/package.json b/frontend/package.json index 086752a..468d649 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "@family-budget/frontend", - "version": "0.11.3", + "version": "0.12.0", "private": true, "type": "module", "scripts": { diff --git a/frontend/src/api/portfolio.ts b/frontend/src/api/portfolio.ts index 929fbe8..c405287 100644 --- a/frontend/src/api/portfolio.ts +++ b/frontend/src/api/portfolio.ts @@ -1,6 +1,12 @@ -import type { ImportPortfolioResponse, PortfolioFile } from '@family-budget/shared'; +import type { ImportBrokerReportResponse, ImportPortfolioResponse, PortfolioFile } from '@family-budget/shared'; import { api } from './client'; export function importPortfolio(data: PortfolioFile): Promise { return api.post('/api/import/portfolio', data); } + +export function importBrokerReport(file: File): Promise { + const formData = new FormData(); + formData.append('file', file); + return api.postFormData('/api/import/broker', formData); +} diff --git a/frontend/src/components/ImportModal.tsx b/frontend/src/components/ImportModal.tsx index 0bb1f77..c04e578 100644 --- a/frontend/src/components/ImportModal.tsx +++ b/frontend/src/components/ImportModal.tsx @@ -1,7 +1,7 @@ import { useState, useRef } from 'react'; -import type { ImportPortfolioResponse, ImportStatementResponse, PortfolioFile } from '@family-budget/shared'; +import type { ImportBrokerReportResponse, ImportPortfolioResponse, ImportStatementResponse, PortfolioFile } from '@family-budget/shared'; import { importStatement } from '../api/import'; -import { importPortfolio } from '../api/portfolio'; +import { importBrokerReport, importPortfolio } from '../api/portfolio'; import { updateAccount } from '../api/accounts'; interface Props { @@ -10,7 +10,7 @@ interface Props { } export function ImportModal({ onClose, onDone }: Props) { - const [result, setResult] = useState(null); + const [result, setResult] = useState(null); const [error, setError] = useState(''); const [loading, setLoading] = useState(false); const [alias, setAlias] = useState(''); @@ -27,9 +27,10 @@ export function ImportModal({ onClose, onDone }: Props) { const type = file.type; const isPdf = type === 'application/pdf' || name.endsWith('.pdf'); const isJson = type === 'application/json' || name.endsWith('.json'); + const isXlsx = type === 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' || name.endsWith('.xlsx'); - if (!isPdf && !isJson) { - setError('Допустимы только файлы PDF или JSON'); + if (!isPdf && !isJson && !isXlsx) { + setError('Допустимы только файлы PDF, JSON или XLSX'); return; } @@ -39,7 +40,9 @@ export function ImportModal({ onClose, onDone }: Props) { try { const data = isJson ? JSON.parse(await file.text()) : null; - const resp = data?.schemaVersion === 'broker-portfolio-1.0' + const resp = isXlsx + ? await importBrokerReport(file) + : data?.schemaVersion === 'broker-portfolio-1.0' ? await importPortfolio(data as PortfolioFile) : await importStatement(file); setResult(resp); @@ -53,7 +56,7 @@ export function ImportModal({ onClose, onDone }: Props) { }; const handleSaveAlias = async () => { - if (!result || 'reportId' in result || !alias.trim()) return; + if (!result || 'reportId' in result || 'cash' in result || !alias.trim()) return; try { await updateAccount(result.accountId, { alias: alias.trim() }); setAliasSaved(true); @@ -63,6 +66,7 @@ export function ImportModal({ onClose, onDone }: Props) { }; const isPortfolioResult = result != null && 'reportId' in result; + const isBrokerResult = result != null && 'cash' in result; return (

- Выберите PDF/JSON выписки или JSON брокерского портфеля + Выберите PDF/JSON выписки, JSON портфеля или XLSX-отчёт ВТБ Брокер

@@ -103,10 +107,31 @@ export function ImportModal({ onClose, onDone }: Props) { {result && (
-

{isPortfolioResult ? 'Импорт портфеля завершён' : 'Импорт завершён'}

+

{isBrokerResult ? 'Импорт брокерского отчёта завершён' : isPortfolioResult ? 'Импорт портфеля завершён' : 'Импорт завершён'}

- {isPortfolioResult ? <> + {isBrokerResult ? <> + + + + + + + + + + + + + + + + + + + + + : isPortfolioResult ? <> @@ -144,7 +169,7 @@ export function ImportModal({ onClose, onDone }: Props) {
Импортировано cash-операций{result.cash.imported}
Дубликатов cash-операций{result.cash.duplicatesSkipped}
Импортировано сделок{result.portfolio.importedTrades}
Дубликатов сделок{result.portfolio.duplicateTrades}
Позиций{result.portfolio.positions}
Импортировано сделок {result.importedTrades}
- {!isPortfolioResult && result.isNewAccount && !aliasSaved && ( + {!isPortfolioResult && !isBrokerResult && result.isNewAccount && !aliasSaved && (
diff --git a/shared/package.json b/shared/package.json index a280663..c2aa476 100644 --- a/shared/package.json +++ b/shared/package.json @@ -1,6 +1,6 @@ { "name": "@family-budget/shared", - "version": "0.5.1", + "version": "0.6.0", "private": true, "main": "dist/index.js", "types": "dist/index.d.ts", diff --git a/shared/src/types/import.ts b/shared/src/types/import.ts index 7c5fbbd..14a7622 100644 --- a/shared/src/types/import.ts +++ b/shared/src/types/import.ts @@ -93,3 +93,8 @@ export interface ImportPortfolioResponse { duplicateTrades: number; positions: number; } + +export interface ImportBrokerReportResponse { + cash: ImportStatementResponse; + portfolio: ImportPortfolioResponse; +} diff --git a/shared/src/types/index.ts b/shared/src/types/index.ts index c349e01..81023df 100644 --- a/shared/src/types/index.ts +++ b/shared/src/types/index.ts @@ -43,6 +43,7 @@ export type { PortfolioPosition, PortfolioTrade, ImportPortfolioResponse, + ImportBrokerReportResponse, } from './import'; export type {