admin/family_budget
1
0
Fork 1
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: import VTB broker XLSX reports

Browse Source
This commit is contained in:
admin
2026-08-26 23:36:32 +03:00
parent 4bd3436ef6
commit 000c77f162
9 changed files with 241 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
backend/package.json
View File

@@ -1,6 +1,6 @@
{
"name": "@family-budget/backend",
"version": "0.10.5",
"version": "0.11.0",
"private": true,
"scripts": {
"dev": "tsx watch src/app.ts",

32
backend/src/routes/import.ts
View File

@@ -6,6 +6,8 @@ import {
convertPdfToStatement,
isPdfConversionError,
} from '../services/pdfToStatement';
import { importPortfolio } from '../services/portfolio';
import { convertVtbBrokerXlsx } from '../services/vtbBrokerXlsx';
const upload = multer({
storage: multer.memoryStorage(),
@@ -28,6 +30,10 @@ function isJsonFile(file: { mimetype: string; originalname: string }): boolean {
);
}
function isXlsxFile(file: { mimetype: string; originalname: string }): boolean {
return file.mimetype === 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' || file.originalname.toLowerCase().endsWith('.xlsx');
}
const router = Router();
router.post(
@@ -87,4 +93,30 @@ router.post(
}),
);
router.post(
'/broker',
upload.single('file'),
asyncHandler(async (req, res) => {
const file = req.file;
if (!file || !isXlsxFile(file)) {
res.status(400).json({ error: 'BAD_REQUEST', message: 'Допустим только XLSX-отчёт ВТБ Брокер' });
return;
}
let converted;
try {
converted = convertVtbBrokerXlsx(file.buffer);
} catch (error) {
res.status(422).json({ error: 'VALIDATION_ERROR', message: error instanceof Error ? error.message : 'Не удалось обработать XLSX-отчёт' });
return;
}
const portfolio = await importPortfolio(converted.portfolio);
const cash = await importStatement(converted.cash);
if (isValidationError(cash)) {
res.status(cash.status).json({ error: cash.error, message: cash.message });
return;
}
res.json({ portfolio, cash });
}),
);
export default router;

156
backend/src/services/vtbBrokerXlsx.ts Normal file
View File

@@ -0,0 +1,156 @@
import crypto from 'crypto';
import zlib from 'zlib';
import type { PortfolioFile, StatementFile } from '@family-budget/shared';
type Row = [number, Record<string, string>];
function xmlText(value: string): string {
return value.replace(/<[^>]+>/g, '').replace(/&amp;/g, '&').replace(/&lt;/g, '<').replace(/&gt;/g, '>').replace(/&#(d+);/g, (_, code) => String.fromCharCode(Number(code))).replace(/\s+/g, ' ').trim();
}
function zipFiles(buffer: Buffer): Map<string, Buffer> {
const end = buffer.lastIndexOf(Buffer.from([0x50, 0x4b, 0x05, 0x06]));
if (end < 0) throw new Error('Файл не является XLSX-архивом');
const files = new Map<string, Buffer>();
let offset = buffer.readUInt32LE(end + 16);
const count = buffer.readUInt16LE(end + 10);
for (let i = 0; i < count; i++) {
if (buffer.readUInt32LE(offset) !== 0x02014b50) throw new Error('Повреждён XLSX-архив');
const method = buffer.readUInt16LE(offset + 10);
const compressedSize = buffer.readUInt32LE(offset + 20);
const nameLength = buffer.readUInt16LE(offset + 28);
const extraLength = buffer.readUInt16LE(offset + 30);
const commentLength = buffer.readUInt16LE(offset + 32);
const localOffset = buffer.readUInt32LE(offset + 42);
const name = buffer.subarray(offset + 46, offset + 46 + nameLength).toString('utf8');
if (buffer.readUInt32LE(localOffset) !== 0x04034b50) throw new Error('Повреждён XLSX-архив');
const localNameLength = buffer.readUInt16LE(localOffset + 26);
const localExtraLength = buffer.readUInt16LE(localOffset + 28);
const data = buffer.subarray(localOffset + 30 + localNameLength + localExtraLength, localOffset + 30 + localNameLength + localExtraLength + compressedSize);
files.set(name, method === 0 ? data : method === 8 ? zlib.inflateRawSync(data) : (() => { throw new Error('Неподдерживаемое сжатие XLSX'); })());
offset += 46 + nameLength + extraLength + commentLength;
}
return files;
}
function readRows(buffer: Buffer): Row[] {
const files = zipFiles(buffer);
const sharedXml = files.get('xl/sharedStrings.xml')?.toString('utf8');
const sheetXml = files.get('xl/worksheets/sheet1.xml')?.toString('utf8');
if (!sharedXml || !sheetXml) throw new Error('В XLSX не найдены данные отчёта');
const shared = [...sharedXml.matchAll(/<si>([\s\S]*?)<\/si>/g)].map((match) => xmlText(match[1]));
return [...sheetXml.matchAll(/<row[^>]*\br="(\d+)"[^>]*>([\s\S]*?)<\/row>/g)].map((row) => {
const cells: Record<string, string> = {};
for (const cell of row[2].matchAll(/<c[^>]*\br="([A-Z]+)\d+"([^>]*)>([\s\S]*?)<\/c>/g)) {
const value = cell[3].match(/<v>([\s\S]*?)<\/v>/)?.[1] ?? '';
cells[cell[1]] = /\bt="s"/.test(cell[2]) && value ? shared[Number(value)] : xmlText(value);
}
return [Number(row[1]), cells];
});
}
function text(value: unknown): string {
return String(value ?? '').replace(/\s+/g, ' ').trim();
}
function rowText(cells: Record<string, string>): string {
return text(Object.values(cells).join(' '));
}
function findRow(rows: Row[], phrase: string, start = 0): number {
const index = rows.findIndex(([, cells], i) => i >= start && rowText(cells).toLowerCase().includes(phrase.toLowerCase()));
if (index < 0) throw new Error(`Не найден раздел: ${phrase}`);
return index;
}
function excelDate(value: string | undefined): string | null {
if (!value) return null;
const date = new Date(Date.UTC(1899, 11, 30) + Number(value) * 86_400_000);
if (Number.isNaN(date.valueOf())) return null;
return `${date.toISOString().slice(0, 19)}+03:00`;
}
function kopecks(value: string | undefined): number {
return value ? Math.round(Number(value.replace(',', '.')) * 100) : 0;
}
function metadata(rows: Row[]): { account: string; period: [string, string]; reportedAt: string | null } {
const period = rows.map(([, cells]) => rowText(cells)).join(' ').match(/период с (\d{2}\.\d{2}\.\d{4}) по (\d{2}\.\d{2}\.\d{4})/i);
let account: string | null = null;
let reportedAt: string | null = null;
for (const [, cells] of rows.slice(0, 35)) {
const joined = rowText(cells);
account ??= joined.match(/(\d{20})\s*\(RUR\)/)?.[1] ?? null;
if (joined.includes('Дата формирования отчета')) {
reportedAt ??= Object.values(cells).map(excelDate).find(Boolean) ?? null;
}
}
if (!account || !period) throw new Error('Не удалось определить счёт или период отчёта');
return { account, period: [period[1], period[2]], reportedAt };
}
function isoDate(value: string): string {
const [day, month, year] = value.split('.');
return `${year}-${month}-${day}`;
}
export function convertVtbBrokerXlsx(buffer: Buffer): { cash: StatementFile; portfolio: PortfolioFile } {
const rows = readRows(buffer);
const { account, period, reportedAt } = metadata(rows);
const holdingsStart = findRow(rows, 'Отчёт об остатках ценных бумаг');
const movementStart = findRow(rows, 'Движение ценных бумаг', holdingsStart);
const cashStart = findRow(rows, 'Движение денежных средств');
const tradesStart = findRow(rows, 'Заключенные в отчетном периоде сделки с ценными бумагами');
const tradesEnd = findRow(rows, 'Завершенные в отчетном периоде сделки с ценными бумагами', tradesStart + 1);
const occurrences = new Map<string, number>();
const transactions = rows.slice(cashStart + 1, holdingsStart).flatMap(([, cells]) => {
if (!cells.B || !cells.C || !cells.J) return [];
const operationAt = excelDate(cells.B);
if (!operationAt) return [];
const amountSigned = kopecks(cells.C);
const description = text(`${text(cells.J)}. ${text(cells.P)}`.replace(/\. $/, ''));
const digest = crypto.createHash('sha256').update(`${operationAt}|${amountSigned}|${description}`).digest('hex').slice(0, 16);
const occurrence = (occurrences.get(digest) ?? 0) + 1;
occurrences.set(digest, occurrence);
return [{ operationAt, amountSigned, commission: 0, description, sourceId: `vtb-broker-cash:${digest}:${occurrence}` }];
});
if (!transactions.length) throw new Error('Операции движения денежных средств не найдены');
const balanceStart = findRow(rows, 'Отчёт об остатках денежных средств');
const openingBalance = kopecks(rows[balanceStart + 3]?.[1].L);
const closingBalance = kopecks(rows[balanceStart + 3]?.[1].AF);
if (openingBalance + transactions.reduce((sum, tx) => sum + tx.amountSigned, 0) !== closingBalance) throw new Error('Баланс cash-операций не сходится с отчётом');
const positions = rows.slice(holdingsStart + 1, movementStart).flatMap(([, cells]) => {
const instrument = text(cells.B);
if (!instrument || instrument.toLowerCase().startsWith('итого') || !/RU[A-Z0-9]{10}/.test(instrument)) return [];
return [{ instrument, isin: instrument.split(', ').find((part) => /^RU[A-Z0-9]{10}$/.test(part)) ?? null, quantity: cells.L || cells.M || cells.I || cells.J, price: cells.P || null, valuation: cells.AF || cells.AJ || null }];
});
const trades = rows.slice(tradesStart + 1, tradesEnd).flatMap(([number, cells]) => {
if (!cells.B || !cells.C || !cells.F) return [];
const concludedAt = excelDate(cells.C);
if (!concludedAt) return [];
const tradeId = text(cells.Z);
return [{
sourceId: `vtb-broker-trade:${tradeId || number}`,
instrument: text(cells.B),
isin: text(cells.B).split(', ').find((part) => /^RU[A-Z0-9]{10}$/.test(part)) ?? null,
concludedAt,
side: text(cells.F),
quantity: text(cells.H),
priceCurrency: text(cells.I) || null,
price: text(cells.J) || null,
settlementCurrency: text(cells.L) || null,
settlementAmount: text(cells.M) || null,
nkd: text(cells.O) || null,
settlementCommission: text(cells.P) || null,
tradeCommission: text(cells.R) || null,
orderId: text(cells.W) || null,
tradeId: tradeId || null,
venue: text(cells.AK) || null,
comment: text(cells.AN) || null,
}];
});
return {
cash: { schemaVersion: '1.0', bank: 'VTB_BROKER', statement: { accountNumber: account, currency: 'RUB', openingBalance, closingBalance, exportedAt: reportedAt ?? transactions[transactions.length - 1].operationAt }, transactions },
portfolio: { schemaVersion: 'broker-portfolio-1.0', bank: 'VTB_BROKER', accountNumber: account, reportPeriod: { from: isoDate(period[0]), to: isoDate(period[1]) }, reportedAt, positions, trades },
};
}

2
frontend/package.json
View File

@@ -1,6 +1,6 @@
{
"name": "@family-budget/frontend",
"version": "0.11.3",
"version": "0.12.0",
"private": true,
"type": "module",
"scripts": {

8
frontend/src/api/portfolio.ts
View File

@@ -1,6 +1,12 @@
import type { ImportPortfolioResponse, PortfolioFile } from '@family-budget/shared';
import type { ImportBrokerReportResponse, ImportPortfolioResponse, PortfolioFile } from '@family-budget/shared';
import { api } from './client';
export function importPortfolio(data: PortfolioFile): Promise<ImportPortfolioResponse> {
return api.post('/api/import/portfolio', data);
}
export function importBrokerReport(file: File): Promise<ImportBrokerReportResponse> {
const formData = new FormData();
formData.append('file', file);
return api.postFormData('/api/import/broker', formData);
}

49
frontend/src/components/ImportModal.tsx
View File

@@ -1,7 +1,7 @@
import { useState, useRef } from 'react';
import type { ImportPortfolioResponse, ImportStatementResponse, PortfolioFile } from '@family-budget/shared';
import type { ImportBrokerReportResponse, ImportPortfolioResponse, ImportStatementResponse, PortfolioFile } from '@family-budget/shared';
import { importStatement } from '../api/import';
import { importPortfolio } from '../api/portfolio';
import { importBrokerReport, importPortfolio } from '../api/portfolio';
import { updateAccount } from '../api/accounts';
interface Props {
@@ -10,7 +10,7 @@ interface Props {
}
export function ImportModal({ onClose, onDone }: Props) {
const [result, setResult] = useState<ImportStatementResponse | ImportPortfolioResponse | null>(null);
const [result, setResult] = useState<ImportStatementResponse | ImportPortfolioResponse | ImportBrokerReportResponse | null>(null);
const [error, setError] = useState('');
const [loading, setLoading] = useState(false);
const [alias, setAlias] = useState('');
@@ -27,9 +27,10 @@ export function ImportModal({ onClose, onDone }: Props) {
const type = file.type;
const isPdf = type === 'application/pdf' || name.endsWith('.pdf');
const isJson = type === 'application/json' || name.endsWith('.json');
const isXlsx = type === 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' || name.endsWith('.xlsx');
if (!isPdf && !isJson) {
setError('Допустимы только файлы PDF или JSON');
if (!isPdf && !isJson && !isXlsx) {
setError('Допустимы только файлы PDF, JSON или XLSX');
return;
}
@@ -39,7 +40,9 @@ export function ImportModal({ onClose, onDone }: Props) {
try {
const data = isJson ? JSON.parse(await file.text()) : null;
const resp = data?.schemaVersion === 'broker-portfolio-1.0'
const resp = isXlsx
? await importBrokerReport(file)
: data?.schemaVersion === 'broker-portfolio-1.0'
? await importPortfolio(data as PortfolioFile)
: await importStatement(file);
setResult(resp);
@@ -53,7 +56,7 @@ export function ImportModal({ onClose, onDone }: Props) {
};
const handleSaveAlias = async () => {
if (!result || 'reportId' in result || !alias.trim()) return;
if (!result || 'reportId' in result || 'cash' in result || !alias.trim()) return;
try {
await updateAccount(result.accountId, { alias: alias.trim() });
setAliasSaved(true);
@@ -63,6 +66,7 @@ export function ImportModal({ onClose, onDone }: Props) {
};
const isPortfolioResult = result != null && 'reportId' in result;
const isBrokerResult = result != null && 'cash' in result;
return (
<div
@@ -85,12 +89,12 @@ export function ImportModal({ onClose, onDone }: Props) {
{!result && (
<div className="import-upload">
<p className="import-upload__description">
Выберите PDF/JSON выписки или JSON брокерского портфеля
Выберите PDF/JSON выписки, JSON портфеля или XLSX-отчёт ВТБ Брокер
</p>
<input
ref={fileRef}
type="file"
accept=".pdf,.json,application/pdf,application/json"
accept=".pdf,.json,.xlsx,application/pdf,application/json,application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
onChange={handleFileChange}
className="import-upload__input"
/>
@@ -103,10 +107,31 @@ export function ImportModal({ onClose, onDone }: Props) {
{result && (
<div className="import-result">
<div className="import-result__icon" aria-hidden="true">✓</div>
<h3 className="import-result__title">{isPortfolioResult ? 'Импорт портфеля завершён' : 'Импорт завершён'}</h3>
<h3 className="import-result__title">{isBrokerResult ? 'Импорт брокерского отчёта завершён' : isPortfolioResult ? 'Импорт портфеля завершён' : 'Импорт завершён'}</h3>
<table className="import-result__stats">
<tbody className="import-result__stats-body">
{isPortfolioResult ? <>
{isBrokerResult ? <>
<tr className="import-result__stat-row">
<td className="import-result__stat-cell import-result__stat-cell--label">Импортировано cash-операций</td>
<td className="import-result__stat-cell import-result__stat-cell--value">{result.cash.imported}</td>
</tr>
<tr className="import-result__stat-row">
<td className="import-result__stat-cell import-result__stat-cell--label">Дубликатов cash-операций</td>
<td className="import-result__stat-cell import-result__stat-cell--value">{result.cash.duplicatesSkipped}</td>
</tr>
<tr className="import-result__stat-row">
<td className="import-result__stat-cell import-result__stat-cell--label">Импортировано сделок</td>
<td className="import-result__stat-cell import-result__stat-cell--value">{result.portfolio.importedTrades}</td>
</tr>
<tr className="import-result__stat-row">
<td className="import-result__stat-cell import-result__stat-cell--label">Дубликатов сделок</td>
<td className="import-result__stat-cell import-result__stat-cell--value">{result.portfolio.duplicateTrades}</td>
</tr>
<tr className="import-result__stat-row">
<td className="import-result__stat-cell import-result__stat-cell--label">Позиций</td>
<td className="import-result__stat-cell import-result__stat-cell--value">{result.portfolio.positions}</td>
</tr>
</> : isPortfolioResult ? <>
<tr className="import-result__stat-row">
<td className="import-result__stat-cell import-result__stat-cell--label">Импортировано сделок</td>
<td className="import-result__stat-cell import-result__stat-cell--value">{result.importedTrades}</td>
@@ -144,7 +169,7 @@ export function ImportModal({ onClose, onDone }: Props) {
</tbody>
</table>
{!isPortfolioResult && result.isNewAccount && !aliasSaved && (
{!isPortfolioResult && !isBrokerResult && result.isNewAccount && !aliasSaved && (
<div className="import-result__alias">
<label className="import-result__alias-label">Алиас для нового счёта</label>
<div className="import-result__alias-row">

2
shared/package.json
View File

@@ -1,6 +1,6 @@
{
"name": "@family-budget/shared",
"version": "0.5.1",
"version": "0.6.0",
"private": true,
"main": "dist/index.js",
"types": "dist/index.d.ts",

5
shared/src/types/import.ts
View File

@@ -93,3 +93,8 @@ export interface ImportPortfolioResponse {
duplicateTrades: number;
positions: number;
}
export interface ImportBrokerReportResponse {
cash: ImportStatementResponse;
portfolio: ImportPortfolioResponse;
}

1
shared/src/types/index.ts
View File

@@ -43,6 +43,7 @@ export type {
PortfolioPosition,
PortfolioTrade,
ImportPortfolioResponse,
ImportBrokerReportResponse,
} from './import';
export type {