diff --git a/backend/src/routes/import.ts b/backend/src/routes/import.ts index d1188b1..a988215 100644 --- a/backend/src/routes/import.ts +++ b/backend/src/routes/import.ts @@ -3,7 +3,7 @@ import multer from 'multer'; import { asyncHandler } from '../utils'; import { importStatement, isValidationError } from '../services/import'; import { - convertPdfToStatement, + convertPdfToStatementStreaming, isPdfConversionError, } from '../services/pdfToStatement'; @@ -28,6 +28,10 @@ function isJsonFile(file: { mimetype: string; originalname: string }): boolean { ); } +function sseWrite(res: import('express').Response, data: Record) { + res.write(`data: ${JSON.stringify(data)}\n\n`); +} + const router = Router(); router.post( @@ -51,28 +55,73 @@ router.post( return; } - let body: unknown; - if (isPdfFile(file)) { - const converted = await convertPdfToStatement(file.buffer); - if (isPdfConversionError(converted)) { - res.status(converted.status).json({ - error: converted.error, - message: converted.message, - }); - return; - } - body = converted; - } else { + res.setHeader('Content-Type', 'text/event-stream'); + res.setHeader('Cache-Control', 'no-cache'); + res.setHeader('Connection', 'keep-alive'); + res.setHeader('X-Accel-Buffering', 'no'); + res.flushHeaders(); + try { - body = JSON.parse(file.buffer.toString('utf-8')); - } catch { - res.status(400).json({ - error: 'BAD_REQUEST', - message: 'Некорректный JSON-файл', + const converted = await convertPdfToStatementStreaming( + file.buffer, + (stage, progress, message) => { + sseWrite(res, { stage, progress, message }); + }, + ); + + if (isPdfConversionError(converted)) { + sseWrite(res, { + stage: 'error', + message: converted.message, + }); + res.end(); + return; + } + + sseWrite(res, { + stage: 'import', + progress: 88, + message: 'Импорт в базу данных...', + }); + + const result = await importStatement(converted); + if (isValidationError(result)) { + sseWrite(res, { + stage: 'error', + message: (result as { message: string }).message, + }); + res.end(); + return; + } + + sseWrite(res, { + stage: 'done', + progress: 100, + result, + }); + } catch (err) { + console.error('SSE import error:', err); + sseWrite(res, { + stage: 'error', + message: 'Внутренняя ошибка сервера', }); - return; } + + res.end(); + return; + } + + // JSON files — synchronous response as before + let body: unknown; + try { + body = JSON.parse(file.buffer.toString('utf-8')); + } catch { + res.status(400).json({ + error: 'BAD_REQUEST', + message: 'Некорректный JSON-файл', + }); + return; } const result = await importStatement(body); diff --git a/backend/src/services/pdfToStatement.ts b/backend/src/services/pdfToStatement.ts index 59fca3d..d351ae7 100644 --- a/backend/src/services/pdfToStatement.ts +++ b/backend/src/services/pdfToStatement.ts @@ -108,6 +108,7 @@ export async function convertPdfToStatement( const openai = new OpenAI({ apiKey: config.llmApiKey, ...(config.llmApiBaseUrl && { baseURL: config.llmApiBaseUrl }), + timeout: 5 * 60 * 1000, }); try { @@ -130,29 +131,7 @@ export async function convertPdfToStatement( }; } - const jsonMatch = content.match(/\{[\s\S]*\}/); - const jsonStr = jsonMatch ? jsonMatch[0] : content; - let parsed: unknown; - try { - parsed = JSON.parse(jsonStr); - } catch { - return { - status: 422, - error: 'VALIDATION_ERROR', - message: 'Результат конвертации не является валидным JSON', - }; - } - - const data = parsed as Record; - if (data.schemaVersion !== '1.0') { - return { - status: 422, - error: 'VALIDATION_ERROR', - message: 'Результат конвертации не соответствует схеме 1.0', - }; - } - - return parsed as StatementFile; + return parseConversionResult(content); } catch (err) { console.error('LLM conversion error:', err); return { @@ -162,3 +141,125 @@ export async function convertPdfToStatement( }; } } + +export type ProgressStage = 'pdf' | 'llm' | 'import'; +export type OnProgress = (stage: ProgressStage, progress: number, message: string) => void; + +const EXPECTED_CHARS = 15_000; + +export async function convertPdfToStatementStreaming( + buffer: Buffer, + onProgress: OnProgress, +): Promise { + if (!config.llmApiKey || config.llmApiKey.trim() === '') { + return { + status: 503, + error: 'SERVICE_UNAVAILABLE', + message: 'Конвертация PDF недоступна: не задан LLM_API_KEY', + }; + } + + onProgress('pdf', 2, 'Извлечение текста из PDF...'); + + let text: string; + try { + const result = await getPdfParse()(buffer); + text = result.text || ''; + } catch (err) { + console.error('PDF extraction error:', err); + return { + status: 400, + error: 'BAD_REQUEST', + message: 'Не удалось обработать PDF-файл', + }; + } + + if (!text || text.trim().length === 0) { + return { + status: 400, + error: 'BAD_REQUEST', + message: 'Не удалось извлечь текст из PDF', + }; + } + + onProgress('pdf', 8, 'Текст извлечён, отправка в LLM...'); + + const openai = new OpenAI({ + apiKey: config.llmApiKey, + ...(config.llmApiBaseUrl && { baseURL: config.llmApiBaseUrl }), + timeout: 5 * 60 * 1000, + }); + + try { + const stream = await openai.chat.completions.create({ + model: config.llmModel, + messages: [ + { role: 'system', content: PDF2JSON_PROMPT }, + { role: 'user', content: `Текст выписки:\n\n${text}` }, + ], + temperature: 0, + max_tokens: 32768, + stream: true, + }); + + let accumulated = ''; + let charsReceived = 0; + + for await (const chunk of stream) { + const delta = chunk.choices[0]?.delta?.content; + if (delta) { + accumulated += delta; + charsReceived += delta.length; + const llmProgress = Math.min( + 85, + Math.round((charsReceived / EXPECTED_CHARS) * 75 + 10), + ); + onProgress('llm', llmProgress, 'Конвертация через LLM...'); + } + } + + const content = accumulated.trim(); + if (!content) { + return { + status: 422, + error: 'VALIDATION_ERROR', + message: 'Результат конвертации пуст', + }; + } + + return parseConversionResult(content); + } catch (err) { + console.error('LLM streaming error:', err); + return { + status: 502, + error: 'BAD_GATEWAY', + message: 'Временная ошибка конвертации', + }; + } +} + +function parseConversionResult(content: string): StatementFile | PdfConversionError { + const jsonMatch = content.match(/\{[\s\S]*\}/); + const jsonStr = jsonMatch ? jsonMatch[0] : content; + let parsed: unknown; + try { + parsed = JSON.parse(jsonStr); + } catch { + return { + status: 422, + error: 'VALIDATION_ERROR', + message: 'Результат конвертации не является валидным JSON', + }; + } + + const data = parsed as Record; + if (data.schemaVersion !== '1.0') { + return { + status: 422, + error: 'VALIDATION_ERROR', + message: 'Результат конвертации не соответствует схеме 1.0', + }; + } + + return parsed as StatementFile; +} diff --git a/frontend/nginx.conf b/frontend/nginx.conf index 4423f5b..6e4284a 100644 --- a/frontend/nginx.conf +++ b/frontend/nginx.conf @@ -3,6 +3,22 @@ server { root /usr/share/nginx/html; index index.html; + # Import endpoint — SSE streaming, long timeout, no buffering + location /api/import { + proxy_pass http://family-budget-backend:3000; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_cookie_path / /; + proxy_connect_timeout 5s; + proxy_read_timeout 600s; + proxy_buffering off; + chunked_transfer_encoding off; + client_max_body_size 15m; + } + # API — проксируем на backend (сервис из docker-compose) location /api { proxy_pass http://family-budget-backend:3000; diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 135747f..e37f15b 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -1,5 +1,6 @@ import { Routes, Route, Navigate } from 'react-router-dom'; import { useAuth } from './context/AuthContext'; +import { ImportProvider } from './context/ImportContext'; import { Layout } from './components/Layout'; import { LoginPage } from './pages/LoginPage'; import { HistoryPage } from './pages/HistoryPage'; @@ -18,14 +19,16 @@ export function App() { } return ( - - - } /> - } /> - } /> - } /> - } /> - - + + + + } /> + } /> + } /> + } /> + } /> + + + ); } diff --git a/frontend/src/api/import.ts b/frontend/src/api/import.ts index af936b5..293eb34 100644 --- a/frontend/src/api/import.ts +++ b/frontend/src/api/import.ts @@ -11,3 +11,74 @@ export async function importStatement( formData, ); } + +export interface SseProgressEvent { + stage: 'pdf' | 'llm' | 'import'; + progress: number; + message: string; +} + +export interface SseDoneEvent { + stage: 'done'; + progress: 100; + result: ImportStatementResponse; +} + +export interface SseErrorEvent { + stage: 'error'; + message: string; +} + +export type SseEvent = SseProgressEvent | SseDoneEvent | SseErrorEvent; + +export async function importStatementStream( + file: File, + onEvent: (event: SseEvent) => void, +): Promise { + const formData = new FormData(); + formData.append('file', file); + + const res = await fetch('/api/import/statement', { + method: 'POST', + body: formData, + credentials: 'include', + }); + + if (!res.ok) { + let msg = 'Ошибка импорта'; + try { + const body = await res.json(); + if (body.message) msg = body.message; + } catch { /* use default */ } + onEvent({ stage: 'error', message: msg }); + return; + } + + const reader = res.body?.getReader(); + if (!reader) { + onEvent({ stage: 'error', message: 'Streaming не поддерживается' }); + return; + } + + const decoder = new TextDecoder(); + let buffer = ''; + + while (true) { + const { done, value } = await reader.read(); + if (done) break; + + buffer += decoder.decode(value, { stream: true }); + + const lines = buffer.split('\n'); + buffer = lines.pop() ?? ''; + + for (const line of lines) { + const trimmed = line.trim(); + if (!trimmed.startsWith('data: ')) continue; + try { + const parsed = JSON.parse(trimmed.slice(6)) as SseEvent; + onEvent(parsed); + } catch { /* skip malformed lines */ } + } + } +} diff --git a/frontend/src/components/ImportModal.tsx b/frontend/src/components/ImportModal.tsx index 107ffda..bae22db 100644 --- a/frontend/src/components/ImportModal.tsx +++ b/frontend/src/components/ImportModal.tsx @@ -2,6 +2,7 @@ import { useState, useRef } from 'react'; import type { ImportStatementResponse } from '@family-budget/shared'; import { importStatement } from '../api/import'; import { updateAccount } from '../api/accounts'; +import { useImport } from '../context/ImportContext'; interface Props { onClose: () => void; @@ -9,13 +10,19 @@ interface Props { } export function ImportModal({ onClose, onDone }: Props) { - const [result, setResult] = useState(null); - const [error, setError] = useState(''); - const [loading, setLoading] = useState(false); + const { importState, startImport, clearImport } = useImport(); + + const [jsonResult, setJsonResult] = useState(null); + const [jsonError, setJsonError] = useState(''); + const [jsonLoading, setJsonLoading] = useState(false); const [alias, setAlias] = useState(''); const [aliasSaved, setAliasSaved] = useState(false); const fileRef = useRef(null); + const result = importState.result ?? jsonResult; + const error = importState.error || jsonError; + const loading = importState.active || jsonLoading; + const handleFileChange = async ( e: React.ChangeEvent, ) => { @@ -28,26 +35,47 @@ export function ImportModal({ onClose, onDone }: Props) { const isJson = type === 'application/json' || name.endsWith('.json'); if (!isPdf && !isJson) { - setError('Допустимы только файлы PDF или JSON'); + setJsonError('Допустимы только файлы PDF или JSON'); return; } - setLoading(true); - setError(''); - setResult(null); + setJsonError(''); + setJsonResult(null); + if (isPdf) { + startImport(file); + return; + } + + setJsonLoading(true); try { const resp = await importStatement(file); - setResult(resp); + setJsonResult(resp); } catch (err: unknown) { const msg = err instanceof Error ? err.message : 'Ошибка импорта'; - setError(msg); + setJsonError(msg); } finally { - setLoading(false); + setJsonLoading(false); } }; + const handleClose = () => { + if (importState.active) { + if (window.confirm('Импорт продолжится в фоне. Закрыть окно?')) { + onClose(); + } + } else { + if (result || importState.error) clearImport(); + onClose(); + } + }; + + const handleDone = () => { + clearImport(); + onDone(); + }; + const handleSaveAlias = async () => { if (!result || !alias.trim()) return; try { @@ -58,12 +86,21 @@ export function ImportModal({ onClose, onDone }: Props) { } }; + const stageLabel = (stage: string) => { + switch (stage) { + case 'pdf': return 'Извлечение текста...'; + case 'llm': return 'Конвертация через LLM...'; + case 'import': return 'Сохранение в базу...'; + default: return 'Обработка...'; + } + }; + return ( -
+
e.stopPropagation()}>

Импорт выписки

-
@@ -71,7 +108,7 @@ export function ImportModal({ onClose, onDone }: Props) {
{error &&
{error}
} - {!result && ( + {!result && !importState.active && (

Выберите файл выписки (PDF или JSON, формат 1.0)

- {loading && ( + {jsonLoading && (
Импорт...
)}
)} + {importState.active && ( +
+
+
+
+
+

+ {stageLabel(importState.stage)} {importState.progress}% +

+
+
+ )} + {result && (
@@ -149,16 +203,15 @@ export function ImportModal({ onClose, onDone }: Props) {
{result ? ( - ) : ( )}
diff --git a/frontend/src/components/Layout.tsx b/frontend/src/components/Layout.tsx index fa38ca7..4390f97 100644 --- a/frontend/src/components/Layout.tsx +++ b/frontend/src/components/Layout.tsx @@ -1,6 +1,70 @@ -import { useState, type ReactNode } from 'react'; +import { useState, useEffect, useRef, useCallback, type ReactNode } from 'react'; import { NavLink } from 'react-router-dom'; import { useAuth } from '../context/AuthContext'; +import { useImport } from '../context/ImportContext'; + +function ImportProgressBar() { + const { importState, clearImport } = useImport(); + const [visible, setVisible] = useState(false); + const hideTimerRef = useRef | undefined>(undefined); + + const isActive = importState.active; + const isDone = importState.stage === 'done'; + const isError = importState.stage === 'error'; + const showBar = isActive || isDone || isError; + + useEffect(() => { + if (showBar) { + setVisible(true); + if (hideTimerRef.current) clearTimeout(hideTimerRef.current); + } + if (isDone || isError) { + hideTimerRef.current = setTimeout(() => { + setVisible(false); + clearImport(); + }, 10_000); + } + return () => { + if (hideTimerRef.current) clearTimeout(hideTimerRef.current); + }; + }, [showBar, isDone, isError, clearImport]); + + const handleClick = useCallback(() => { + if (hideTimerRef.current) clearTimeout(hideTimerRef.current); + setVisible(false); + clearImport(); + }, [clearImport]); + + if (!visible) return null; + + const barClass = isError + ? 'import-progress-bar import-progress-bar--error' + : isDone + ? 'import-progress-bar import-progress-bar--done' + : 'import-progress-bar'; + + const labelText = isError + ? `Ошибка импорта: ${importState.message}` + : isDone && importState.result + ? `Импорт завершён — ${importState.result.imported} операций` + : `${importState.message} ${importState.progress}%`; + + return ( +
+
+ +
+ ); +} export function Layout({ children }: { children: ReactNode }) { const { user, logout } = useAuth(); @@ -10,6 +74,8 @@ export function Layout({ children }: { children: ReactNode }) { return (
+ +