feat: import VTB broker XLSX reports
This commit is contained in:
156
backend/src/services/vtbBrokerXlsx.ts
Normal file
156
backend/src/services/vtbBrokerXlsx.ts
Normal file
@@ -0,0 +1,156 @@
|
||||
import crypto from 'crypto';
|
||||
import zlib from 'zlib';
|
||||
import type { PortfolioFile, StatementFile } from '@family-budget/shared';
|
||||
|
||||
type Row = [number, Record<string, string>];
|
||||
|
||||
function xmlText(value: string): string {
|
||||
return value.replace(/<[^>]+>/g, '').replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>').replace(/&#(d+);/g, (_, code) => String.fromCharCode(Number(code))).replace(/\s+/g, ' ').trim();
|
||||
}
|
||||
|
||||
function zipFiles(buffer: Buffer): Map<string, Buffer> {
|
||||
const end = buffer.lastIndexOf(Buffer.from([0x50, 0x4b, 0x05, 0x06]));
|
||||
if (end < 0) throw new Error('Файл не является XLSX-архивом');
|
||||
const files = new Map<string, Buffer>();
|
||||
let offset = buffer.readUInt32LE(end + 16);
|
||||
const count = buffer.readUInt16LE(end + 10);
|
||||
for (let i = 0; i < count; i++) {
|
||||
if (buffer.readUInt32LE(offset) !== 0x02014b50) throw new Error('Повреждён XLSX-архив');
|
||||
const method = buffer.readUInt16LE(offset + 10);
|
||||
const compressedSize = buffer.readUInt32LE(offset + 20);
|
||||
const nameLength = buffer.readUInt16LE(offset + 28);
|
||||
const extraLength = buffer.readUInt16LE(offset + 30);
|
||||
const commentLength = buffer.readUInt16LE(offset + 32);
|
||||
const localOffset = buffer.readUInt32LE(offset + 42);
|
||||
const name = buffer.subarray(offset + 46, offset + 46 + nameLength).toString('utf8');
|
||||
if (buffer.readUInt32LE(localOffset) !== 0x04034b50) throw new Error('Повреждён XLSX-архив');
|
||||
const localNameLength = buffer.readUInt16LE(localOffset + 26);
|
||||
const localExtraLength = buffer.readUInt16LE(localOffset + 28);
|
||||
const data = buffer.subarray(localOffset + 30 + localNameLength + localExtraLength, localOffset + 30 + localNameLength + localExtraLength + compressedSize);
|
||||
files.set(name, method === 0 ? data : method === 8 ? zlib.inflateRawSync(data) : (() => { throw new Error('Неподдерживаемое сжатие XLSX'); })());
|
||||
offset += 46 + nameLength + extraLength + commentLength;
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
function readRows(buffer: Buffer): Row[] {
|
||||
const files = zipFiles(buffer);
|
||||
const sharedXml = files.get('xl/sharedStrings.xml')?.toString('utf8');
|
||||
const sheetXml = files.get('xl/worksheets/sheet1.xml')?.toString('utf8');
|
||||
if (!sharedXml || !sheetXml) throw new Error('В XLSX не найдены данные отчёта');
|
||||
const shared = [...sharedXml.matchAll(/<si>([\s\S]*?)<\/si>/g)].map((match) => xmlText(match[1]));
|
||||
return [...sheetXml.matchAll(/<row[^>]*\br="(\d+)"[^>]*>([\s\S]*?)<\/row>/g)].map((row) => {
|
||||
const cells: Record<string, string> = {};
|
||||
for (const cell of row[2].matchAll(/<c[^>]*\br="([A-Z]+)\d+"([^>]*)>([\s\S]*?)<\/c>/g)) {
|
||||
const value = cell[3].match(/<v>([\s\S]*?)<\/v>/)?.[1] ?? '';
|
||||
cells[cell[1]] = /\bt="s"/.test(cell[2]) && value ? shared[Number(value)] : xmlText(value);
|
||||
}
|
||||
return [Number(row[1]), cells];
|
||||
});
|
||||
}
|
||||
|
||||
function text(value: unknown): string {
|
||||
return String(value ?? '').replace(/\s+/g, ' ').trim();
|
||||
}
|
||||
|
||||
function rowText(cells: Record<string, string>): string {
|
||||
return text(Object.values(cells).join(' '));
|
||||
}
|
||||
|
||||
function findRow(rows: Row[], phrase: string, start = 0): number {
|
||||
const index = rows.findIndex(([, cells], i) => i >= start && rowText(cells).toLowerCase().includes(phrase.toLowerCase()));
|
||||
if (index < 0) throw new Error(`Не найден раздел: ${phrase}`);
|
||||
return index;
|
||||
}
|
||||
|
||||
function excelDate(value: string | undefined): string | null {
|
||||
if (!value) return null;
|
||||
const date = new Date(Date.UTC(1899, 11, 30) + Number(value) * 86_400_000);
|
||||
if (Number.isNaN(date.valueOf())) return null;
|
||||
return `${date.toISOString().slice(0, 19)}+03:00`;
|
||||
}
|
||||
|
||||
function kopecks(value: string | undefined): number {
|
||||
return value ? Math.round(Number(value.replace(',', '.')) * 100) : 0;
|
||||
}
|
||||
|
||||
function metadata(rows: Row[]): { account: string; period: [string, string]; reportedAt: string | null } {
|
||||
const period = rows.map(([, cells]) => rowText(cells)).join(' ').match(/период с (\d{2}\.\d{2}\.\d{4}) по (\d{2}\.\d{2}\.\d{4})/i);
|
||||
let account: string | null = null;
|
||||
let reportedAt: string | null = null;
|
||||
for (const [, cells] of rows.slice(0, 35)) {
|
||||
const joined = rowText(cells);
|
||||
account ??= joined.match(/(\d{20})\s*\(RUR\)/)?.[1] ?? null;
|
||||
if (joined.includes('Дата формирования отчета')) {
|
||||
reportedAt ??= Object.values(cells).map(excelDate).find(Boolean) ?? null;
|
||||
}
|
||||
}
|
||||
if (!account || !period) throw new Error('Не удалось определить счёт или период отчёта');
|
||||
return { account, period: [period[1], period[2]], reportedAt };
|
||||
}
|
||||
|
||||
function isoDate(value: string): string {
|
||||
const [day, month, year] = value.split('.');
|
||||
return `${year}-${month}-${day}`;
|
||||
}
|
||||
|
||||
export function convertVtbBrokerXlsx(buffer: Buffer): { cash: StatementFile; portfolio: PortfolioFile } {
|
||||
const rows = readRows(buffer);
|
||||
const { account, period, reportedAt } = metadata(rows);
|
||||
const holdingsStart = findRow(rows, 'Отчёт об остатках ценных бумаг');
|
||||
const movementStart = findRow(rows, 'Движение ценных бумаг', holdingsStart);
|
||||
const cashStart = findRow(rows, 'Движение денежных средств');
|
||||
const tradesStart = findRow(rows, 'Заключенные в отчетном периоде сделки с ценными бумагами');
|
||||
const tradesEnd = findRow(rows, 'Завершенные в отчетном периоде сделки с ценными бумагами', tradesStart + 1);
|
||||
const occurrences = new Map<string, number>();
|
||||
const transactions = rows.slice(cashStart + 1, holdingsStart).flatMap(([, cells]) => {
|
||||
if (!cells.B || !cells.C || !cells.J) return [];
|
||||
const operationAt = excelDate(cells.B);
|
||||
if (!operationAt) return [];
|
||||
const amountSigned = kopecks(cells.C);
|
||||
const description = text(`${text(cells.J)}. ${text(cells.P)}`.replace(/\. $/, ''));
|
||||
const digest = crypto.createHash('sha256').update(`${operationAt}|${amountSigned}|${description}`).digest('hex').slice(0, 16);
|
||||
const occurrence = (occurrences.get(digest) ?? 0) + 1;
|
||||
occurrences.set(digest, occurrence);
|
||||
return [{ operationAt, amountSigned, commission: 0, description, sourceId: `vtb-broker-cash:${digest}:${occurrence}` }];
|
||||
});
|
||||
if (!transactions.length) throw new Error('Операции движения денежных средств не найдены');
|
||||
const balanceStart = findRow(rows, 'Отчёт об остатках денежных средств');
|
||||
const openingBalance = kopecks(rows[balanceStart + 3]?.[1].L);
|
||||
const closingBalance = kopecks(rows[balanceStart + 3]?.[1].AF);
|
||||
if (openingBalance + transactions.reduce((sum, tx) => sum + tx.amountSigned, 0) !== closingBalance) throw new Error('Баланс cash-операций не сходится с отчётом');
|
||||
const positions = rows.slice(holdingsStart + 1, movementStart).flatMap(([, cells]) => {
|
||||
const instrument = text(cells.B);
|
||||
if (!instrument || instrument.toLowerCase().startsWith('итого') || !/RU[A-Z0-9]{10}/.test(instrument)) return [];
|
||||
return [{ instrument, isin: instrument.split(', ').find((part) => /^RU[A-Z0-9]{10}$/.test(part)) ?? null, quantity: cells.L || cells.M || cells.I || cells.J, price: cells.P || null, valuation: cells.AF || cells.AJ || null }];
|
||||
});
|
||||
const trades = rows.slice(tradesStart + 1, tradesEnd).flatMap(([number, cells]) => {
|
||||
if (!cells.B || !cells.C || !cells.F) return [];
|
||||
const concludedAt = excelDate(cells.C);
|
||||
if (!concludedAt) return [];
|
||||
const tradeId = text(cells.Z);
|
||||
return [{
|
||||
sourceId: `vtb-broker-trade:${tradeId || number}`,
|
||||
instrument: text(cells.B),
|
||||
isin: text(cells.B).split(', ').find((part) => /^RU[A-Z0-9]{10}$/.test(part)) ?? null,
|
||||
concludedAt,
|
||||
side: text(cells.F),
|
||||
quantity: text(cells.H),
|
||||
priceCurrency: text(cells.I) || null,
|
||||
price: text(cells.J) || null,
|
||||
settlementCurrency: text(cells.L) || null,
|
||||
settlementAmount: text(cells.M) || null,
|
||||
nkd: text(cells.O) || null,
|
||||
settlementCommission: text(cells.P) || null,
|
||||
tradeCommission: text(cells.R) || null,
|
||||
orderId: text(cells.W) || null,
|
||||
tradeId: tradeId || null,
|
||||
venue: text(cells.AK) || null,
|
||||
comment: text(cells.AN) || null,
|
||||
}];
|
||||
});
|
||||
return {
|
||||
cash: { schemaVersion: '1.0', bank: 'VTB_BROKER', statement: { accountNumber: account, currency: 'RUB', openingBalance, closingBalance, exportedAt: reportedAt ?? transactions[transactions.length - 1].operationAt }, transactions },
|
||||
portfolio: { schemaVersion: 'broker-portfolio-1.0', bank: 'VTB_BROKER', accountNumber: account, reportPeriod: { from: isoDate(period[0]), to: isoDate(period[1]) }, reportedAt, positions, trades },
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user