7 Commits

Author SHA1 Message Date
vakabunga
67fed57118 fix: disable gzip and pad SSE events to prevent proxy buffering
Add gzip off to Nginx import location — the global gzip on was
buffering text/event-stream responses. Pad each SSE event to 4 KB
with comment lines to push past any remaining proxy buffer threshold.
2026-03-14 19:45:33 +03:00
45a6f3d374 Merge pull request 'fix: eliminate SSE buffering through Nginx proxy' (#8) from fix/sse-proxy-buffering into main
Reviewed-on: #8
2026-03-14 14:31:16 +00:00
vakabunga
aaf8cacf75 fix: eliminate SSE buffering through Nginx proxy
Add 2 KB padding comment after headers to push past proxy buffer
threshold, enable TCP_NODELAY on the socket, and remove erroneous
chunked_transfer_encoding off from Nginx that caused full response
buffering.
2026-03-14 17:30:52 +03:00
vakabunga
e28d0f46d0 fix: reopen result modal from progress bar, faster progress, handle LLM context error
- Move import modal visibility into ImportContext so the Layout
  progress pill can reopen the result dialog after the modal was closed.
- Raise LLM progress cap from 85% to 98% and drop the intermediate
  -import 88%- SSE event to eliminate the visual stall after LLM finishes.
- Detect LLM context-length errors (n_keep >= n_ctx) and surface a
  clear message instead of generic -Временная ошибка конвертации-.
2026-03-14 17:05:55 +03:00
22be09c101 Merge pull request 'fix: adaptive LLM progress estimation and emit 85% on stream end' (#6) from fix/adaptive-llm-progress into main
Reviewed-on: #6
2026-03-14 13:43:27 +00:00
vakabunga
78c4730196 fix: adaptive LLM progress estimation and emit 85% on stream end
Hardcoded EXPECTED_CHARS (15k) caused progress to stall at ~20-25% for
short statements. Now expected size is derived from input text length.
Also emit an explicit 85% event when the LLM stream finishes, and
throttle SSE events to 300ms to reduce browser overhead.
2026-03-14 16:41:12 +03:00
f2d0c91488 Merge pull request 'feat: stream PDF import progress via SSE with global progress bar' (#5) from feature/pdf-import-sse-streaming into main
Reviewed-on: #5
2026-03-14 13:18:57 +00:00
7 changed files with 65 additions and 26 deletions

View File

@@ -28,8 +28,14 @@ function isJsonFile(file: { mimetype: string; originalname: string }): boolean {
); );
} }
const SSE_PAD_TARGET = 4096;
function sseWrite(res: import('express').Response, data: Record<string, unknown>) { function sseWrite(res: import('express').Response, data: Record<string, unknown>) {
res.write(`data: ${JSON.stringify(data)}\n\n`); const payload = `data: ${JSON.stringify(data)}\n\n`;
const pad = Math.max(0, SSE_PAD_TARGET - payload.length);
// SSE comment lines (": ...") are ignored by the browser but push
// data past proxy buffer thresholds so each event is delivered immediately.
res.write(pad > 0 ? `: ${' '.repeat(pad)}\n${payload}` : payload);
} }
const router = Router(); const router = Router();
@@ -60,6 +66,7 @@ router.post(
res.setHeader('Cache-Control', 'no-cache'); res.setHeader('Cache-Control', 'no-cache');
res.setHeader('Connection', 'keep-alive'); res.setHeader('Connection', 'keep-alive');
res.setHeader('X-Accel-Buffering', 'no'); res.setHeader('X-Accel-Buffering', 'no');
res.socket?.setNoDelay(true);
res.flushHeaders(); res.flushHeaders();
try { try {
@@ -79,12 +86,6 @@ router.post(
return; return;
} }
sseWrite(res, {
stage: 'import',
progress: 88,
message: 'Импорт в базу данных...',
});
const result = await importStatement(converted); const result = await importStatement(converted);
if (isValidationError(result)) { if (isValidationError(result)) {
sseWrite(res, { sseWrite(res, {

View File

@@ -137,7 +137,7 @@ export async function convertPdfToStatement(
return { return {
status: 502, status: 502,
error: 'BAD_GATEWAY', error: 'BAD_GATEWAY',
message: 'Временная ошибка конвертации', message: extractLlmErrorMessage(err),
}; };
} }
} }
@@ -145,7 +145,10 @@ export async function convertPdfToStatement(
export type ProgressStage = 'pdf' | 'llm' | 'import'; export type ProgressStage = 'pdf' | 'llm' | 'import';
export type OnProgress = (stage: ProgressStage, progress: number, message: string) => void; export type OnProgress = (stage: ProgressStage, progress: number, message: string) => void;
const EXPECTED_CHARS = 15_000; const LLM_PROGRESS_MIN = 10;
const LLM_PROGRESS_MAX = 98;
const LLM_PROGRESS_RANGE = LLM_PROGRESS_MAX - LLM_PROGRESS_MIN;
const THROTTLE_MS = 300;
export async function convertPdfToStatementStreaming( export async function convertPdfToStatementStreaming(
buffer: Buffer, buffer: Buffer,
@@ -202,21 +205,33 @@ export async function convertPdfToStatementStreaming(
stream: true, stream: true,
}); });
// Estimate expected output size as ~2x the input PDF text length, clamped
const expectedChars = Math.max(2_000, Math.min(text.length * 2, 30_000));
let accumulated = ''; let accumulated = '';
let charsReceived = 0; let charsReceived = 0;
let lastEmitTime = 0;
for await (const chunk of stream) { for await (const chunk of stream) {
const delta = chunk.choices[0]?.delta?.content; const delta = chunk.choices[0]?.delta?.content;
if (delta) { if (delta) {
accumulated += delta; accumulated += delta;
charsReceived += delta.length; charsReceived += delta.length;
const now = Date.now();
if (now - lastEmitTime >= THROTTLE_MS) {
const ratio = Math.min(1, charsReceived / expectedChars);
const llmProgress = Math.min( const llmProgress = Math.min(
85, LLM_PROGRESS_MAX,
Math.round((charsReceived / EXPECTED_CHARS) * 75 + 10), Math.round(ratio * LLM_PROGRESS_RANGE + LLM_PROGRESS_MIN),
); );
onProgress('llm', llmProgress, 'Конвертация через LLM...'); onProgress('llm', llmProgress, 'Конвертация через LLM...');
lastEmitTime = now;
} }
} }
}
onProgress('llm', LLM_PROGRESS_MAX, 'LLM завершил, обработка результата...');
const content = accumulated.trim(); const content = accumulated.trim();
if (!content) { if (!content) {
@@ -233,11 +248,25 @@ export async function convertPdfToStatementStreaming(
return { return {
status: 502, status: 502,
error: 'BAD_GATEWAY', error: 'BAD_GATEWAY',
message: 'Временная ошибка конвертации', message: extractLlmErrorMessage(err),
}; };
} }
} }
function extractLlmErrorMessage(err: unknown): string {
const raw = String(
(err as Record<string, unknown>)?.message ??
(err as Record<string, Record<string, unknown>>)?.error?.message ?? '',
);
if (/context.length|n_ctx|too.many.tokens|maximum.context/i.test(raw)) {
return 'PDF-файл слишком большой для обработки. Попробуйте файл с меньшим количеством операций или используйте модель с большим контекстным окном.';
}
if (/timeout|timed?\s*out|ETIMEDOUT|ECONNREFUSED/i.test(raw)) {
return 'LLM-сервер не отвечает. Проверьте, что сервер запущен и доступен.';
}
return 'Временная ошибка конвертации';
}
function parseConversionResult(content: string): StatementFile | PdfConversionError { function parseConversionResult(content: string): StatementFile | PdfConversionError {
const jsonMatch = content.match(/\{[\s\S]*\}/); const jsonMatch = content.match(/\{[\s\S]*\}/);
const jsonStr = jsonMatch ? jsonMatch[0] : content; const jsonStr = jsonMatch ? jsonMatch[0] : content;

View File

@@ -15,7 +15,7 @@ server {
proxy_connect_timeout 5s; proxy_connect_timeout 5s;
proxy_read_timeout 600s; proxy_read_timeout 600s;
proxy_buffering off; proxy_buffering off;
chunked_transfer_encoding off; gzip off;
client_max_body_size 15m; client_max_body_size 15m;
} }

View File

@@ -66,13 +66,11 @@ export function ImportModal({ onClose, onDone }: Props) {
onClose(); onClose();
} }
} else { } else {
if (result || importState.error) clearImport();
onClose(); onClose();
} }
}; };
const handleDone = () => { const handleDone = () => {
clearImport();
onDone(); onDone();
}; };

View File

@@ -4,7 +4,7 @@ import { useAuth } from '../context/AuthContext';
import { useImport } from '../context/ImportContext'; import { useImport } from '../context/ImportContext';
function ImportProgressBar() { function ImportProgressBar() {
const { importState, clearImport } = useImport(); const { importState, clearImport, openModal } = useImport();
const [visible, setVisible] = useState(false); const [visible, setVisible] = useState(false);
const hideTimerRef = useRef<ReturnType<typeof setTimeout> | undefined>(undefined); const hideTimerRef = useRef<ReturnType<typeof setTimeout> | undefined>(undefined);
@@ -31,9 +31,9 @@ function ImportProgressBar() {
const handleClick = useCallback(() => { const handleClick = useCallback(() => {
if (hideTimerRef.current) clearTimeout(hideTimerRef.current); if (hideTimerRef.current) clearTimeout(hideTimerRef.current);
openModal();
setVisible(false); setVisible(false);
clearImport(); }, [openModal]);
}, [clearImport]);
if (!visible) return null; if (!visible) return null;

View File

@@ -20,6 +20,9 @@ export interface ImportProgress {
interface ImportContextValue { interface ImportContextValue {
importState: ImportProgress; importState: ImportProgress;
showModal: boolean;
openModal: () => void;
closeModal: () => void;
startImport: (file: File) => void; startImport: (file: File) => void;
clearImport: () => void; clearImport: () => void;
} }
@@ -35,8 +38,12 @@ const ImportContext = createContext<ImportContextValue | null>(null);
export function ImportProvider({ children }: { children: ReactNode }) { export function ImportProvider({ children }: { children: ReactNode }) {
const [importState, setImportState] = useState<ImportProgress>(INITIAL); const [importState, setImportState] = useState<ImportProgress>(INITIAL);
const [showModal, setShowModal] = useState(false);
const runningRef = useRef(false); const runningRef = useRef(false);
const openModal = useCallback(() => setShowModal(true), []);
const closeModal = useCallback(() => setShowModal(false), []);
const startImport = useCallback((file: File) => { const startImport = useCallback((file: File) => {
if (runningRef.current) return; if (runningRef.current) return;
runningRef.current = true; runningRef.current = true;
@@ -92,7 +99,9 @@ export function ImportProvider({ children }: { children: ReactNode }) {
}, []); }, []);
return ( return (
<ImportContext.Provider value={{ importState, startImport, clearImport }}> <ImportContext.Provider value={{
importState, showModal, openModal, closeModal, startImport, clearImport,
}}>
{children} {children}
</ImportContext.Provider> </ImportContext.Provider>
); );

View File

@@ -18,6 +18,7 @@ import { TransactionTable } from '../components/TransactionTable';
import { Pagination } from '../components/Pagination'; import { Pagination } from '../components/Pagination';
import { EditTransactionModal } from '../components/EditTransactionModal'; import { EditTransactionModal } from '../components/EditTransactionModal';
import { ImportModal } from '../components/ImportModal'; import { ImportModal } from '../components/ImportModal';
import { useImport } from '../context/ImportContext';
import { toISODate } from '../utils/format'; import { toISODate } from '../utils/format';
const PARAM_KEYS = [ const PARAM_KEYS = [
@@ -125,7 +126,7 @@ export function HistoryPage() {
const [accounts, setAccounts] = useState<Account[]>([]); const [accounts, setAccounts] = useState<Account[]>([]);
const [categories, setCategories] = useState<Category[]>([]); const [categories, setCategories] = useState<Category[]>([]);
const [editingTx, setEditingTx] = useState<Transaction | null>(null); const [editingTx, setEditingTx] = useState<Transaction | null>(null);
const [showImport, setShowImport] = useState(false); const { showModal: showImport, openModal: openImport, closeModal: closeImport, clearImport } = useImport();
useEffect(() => { useEffect(() => {
getAccounts().then(setAccounts).catch(() => {}); getAccounts().then(setAccounts).catch(() => {});
@@ -197,7 +198,8 @@ export function HistoryPage() {
}; };
const handleImportDone = () => { const handleImportDone = () => {
setShowImport(false); closeImport();
clearImport();
fetchData(); fetchData();
getAccounts().then(setAccounts).catch(() => {}); getAccounts().then(setAccounts).catch(() => {});
}; };
@@ -208,7 +210,7 @@ export function HistoryPage() {
<h1>История операций</h1> <h1>История операций</h1>
<button <button
className="btn btn-primary" className="btn btn-primary"
onClick={() => setShowImport(true)} onClick={openImport}
> >
Импорт выписки Импорт выписки
</button> </button>
@@ -262,7 +264,7 @@ export function HistoryPage() {
{showImport && ( {showImport && (
<ImportModal <ImportModal
onClose={() => setShowImport(false)} onClose={closeImport}
onDone={handleImportDone} onDone={handleImportDone}
/> />
)} )}