Compare commits
7 Commits
feature/pd
...
fix/sse-gz
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
67fed57118 | ||
| 45a6f3d374 | |||
|
|
aaf8cacf75 | ||
|
|
e28d0f46d0 | ||
| 22be09c101 | |||
|
|
78c4730196 | ||
| f2d0c91488 |
@@ -28,8 +28,14 @@ function isJsonFile(file: { mimetype: string; originalname: string }): boolean {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const SSE_PAD_TARGET = 4096;
|
||||||
|
|
||||||
function sseWrite(res: import('express').Response, data: Record<string, unknown>) {
|
function sseWrite(res: import('express').Response, data: Record<string, unknown>) {
|
||||||
res.write(`data: ${JSON.stringify(data)}\n\n`);
|
const payload = `data: ${JSON.stringify(data)}\n\n`;
|
||||||
|
const pad = Math.max(0, SSE_PAD_TARGET - payload.length);
|
||||||
|
// SSE comment lines (": ...") are ignored by the browser but push
|
||||||
|
// data past proxy buffer thresholds so each event is delivered immediately.
|
||||||
|
res.write(pad > 0 ? `: ${' '.repeat(pad)}\n${payload}` : payload);
|
||||||
}
|
}
|
||||||
|
|
||||||
const router = Router();
|
const router = Router();
|
||||||
@@ -60,6 +66,7 @@ router.post(
|
|||||||
res.setHeader('Cache-Control', 'no-cache');
|
res.setHeader('Cache-Control', 'no-cache');
|
||||||
res.setHeader('Connection', 'keep-alive');
|
res.setHeader('Connection', 'keep-alive');
|
||||||
res.setHeader('X-Accel-Buffering', 'no');
|
res.setHeader('X-Accel-Buffering', 'no');
|
||||||
|
res.socket?.setNoDelay(true);
|
||||||
res.flushHeaders();
|
res.flushHeaders();
|
||||||
|
|
||||||
try {
|
try {
|
||||||
@@ -79,12 +86,6 @@ router.post(
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
sseWrite(res, {
|
|
||||||
stage: 'import',
|
|
||||||
progress: 88,
|
|
||||||
message: 'Импорт в базу данных...',
|
|
||||||
});
|
|
||||||
|
|
||||||
const result = await importStatement(converted);
|
const result = await importStatement(converted);
|
||||||
if (isValidationError(result)) {
|
if (isValidationError(result)) {
|
||||||
sseWrite(res, {
|
sseWrite(res, {
|
||||||
|
|||||||
@@ -137,7 +137,7 @@ export async function convertPdfToStatement(
|
|||||||
return {
|
return {
|
||||||
status: 502,
|
status: 502,
|
||||||
error: 'BAD_GATEWAY',
|
error: 'BAD_GATEWAY',
|
||||||
message: 'Временная ошибка конвертации',
|
message: extractLlmErrorMessage(err),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -145,7 +145,10 @@ export async function convertPdfToStatement(
|
|||||||
export type ProgressStage = 'pdf' | 'llm' | 'import';
|
export type ProgressStage = 'pdf' | 'llm' | 'import';
|
||||||
export type OnProgress = (stage: ProgressStage, progress: number, message: string) => void;
|
export type OnProgress = (stage: ProgressStage, progress: number, message: string) => void;
|
||||||
|
|
||||||
const EXPECTED_CHARS = 15_000;
|
const LLM_PROGRESS_MIN = 10;
|
||||||
|
const LLM_PROGRESS_MAX = 98;
|
||||||
|
const LLM_PROGRESS_RANGE = LLM_PROGRESS_MAX - LLM_PROGRESS_MIN;
|
||||||
|
const THROTTLE_MS = 300;
|
||||||
|
|
||||||
export async function convertPdfToStatementStreaming(
|
export async function convertPdfToStatementStreaming(
|
||||||
buffer: Buffer,
|
buffer: Buffer,
|
||||||
@@ -202,22 +205,34 @@ export async function convertPdfToStatementStreaming(
|
|||||||
stream: true,
|
stream: true,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// Estimate expected output size as ~2x the input PDF text length, clamped
|
||||||
|
const expectedChars = Math.max(2_000, Math.min(text.length * 2, 30_000));
|
||||||
|
|
||||||
let accumulated = '';
|
let accumulated = '';
|
||||||
let charsReceived = 0;
|
let charsReceived = 0;
|
||||||
|
let lastEmitTime = 0;
|
||||||
|
|
||||||
for await (const chunk of stream) {
|
for await (const chunk of stream) {
|
||||||
const delta = chunk.choices[0]?.delta?.content;
|
const delta = chunk.choices[0]?.delta?.content;
|
||||||
if (delta) {
|
if (delta) {
|
||||||
accumulated += delta;
|
accumulated += delta;
|
||||||
charsReceived += delta.length;
|
charsReceived += delta.length;
|
||||||
const llmProgress = Math.min(
|
|
||||||
85,
|
const now = Date.now();
|
||||||
Math.round((charsReceived / EXPECTED_CHARS) * 75 + 10),
|
if (now - lastEmitTime >= THROTTLE_MS) {
|
||||||
);
|
const ratio = Math.min(1, charsReceived / expectedChars);
|
||||||
onProgress('llm', llmProgress, 'Конвертация через LLM...');
|
const llmProgress = Math.min(
|
||||||
|
LLM_PROGRESS_MAX,
|
||||||
|
Math.round(ratio * LLM_PROGRESS_RANGE + LLM_PROGRESS_MIN),
|
||||||
|
);
|
||||||
|
onProgress('llm', llmProgress, 'Конвертация через LLM...');
|
||||||
|
lastEmitTime = now;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
onProgress('llm', LLM_PROGRESS_MAX, 'LLM завершил, обработка результата...');
|
||||||
|
|
||||||
const content = accumulated.trim();
|
const content = accumulated.trim();
|
||||||
if (!content) {
|
if (!content) {
|
||||||
return {
|
return {
|
||||||
@@ -233,11 +248,25 @@ export async function convertPdfToStatementStreaming(
|
|||||||
return {
|
return {
|
||||||
status: 502,
|
status: 502,
|
||||||
error: 'BAD_GATEWAY',
|
error: 'BAD_GATEWAY',
|
||||||
message: 'Временная ошибка конвертации',
|
message: extractLlmErrorMessage(err),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function extractLlmErrorMessage(err: unknown): string {
|
||||||
|
const raw = String(
|
||||||
|
(err as Record<string, unknown>)?.message ??
|
||||||
|
(err as Record<string, Record<string, unknown>>)?.error?.message ?? '',
|
||||||
|
);
|
||||||
|
if (/context.length|n_ctx|too.many.tokens|maximum.context/i.test(raw)) {
|
||||||
|
return 'PDF-файл слишком большой для обработки. Попробуйте файл с меньшим количеством операций или используйте модель с большим контекстным окном.';
|
||||||
|
}
|
||||||
|
if (/timeout|timed?\s*out|ETIMEDOUT|ECONNREFUSED/i.test(raw)) {
|
||||||
|
return 'LLM-сервер не отвечает. Проверьте, что сервер запущен и доступен.';
|
||||||
|
}
|
||||||
|
return 'Временная ошибка конвертации';
|
||||||
|
}
|
||||||
|
|
||||||
function parseConversionResult(content: string): StatementFile | PdfConversionError {
|
function parseConversionResult(content: string): StatementFile | PdfConversionError {
|
||||||
const jsonMatch = content.match(/\{[\s\S]*\}/);
|
const jsonMatch = content.match(/\{[\s\S]*\}/);
|
||||||
const jsonStr = jsonMatch ? jsonMatch[0] : content;
|
const jsonStr = jsonMatch ? jsonMatch[0] : content;
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ server {
|
|||||||
proxy_connect_timeout 5s;
|
proxy_connect_timeout 5s;
|
||||||
proxy_read_timeout 600s;
|
proxy_read_timeout 600s;
|
||||||
proxy_buffering off;
|
proxy_buffering off;
|
||||||
chunked_transfer_encoding off;
|
gzip off;
|
||||||
client_max_body_size 15m;
|
client_max_body_size 15m;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -66,13 +66,11 @@ export function ImportModal({ onClose, onDone }: Props) {
|
|||||||
onClose();
|
onClose();
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
if (result || importState.error) clearImport();
|
|
||||||
onClose();
|
onClose();
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
const handleDone = () => {
|
const handleDone = () => {
|
||||||
clearImport();
|
|
||||||
onDone();
|
onDone();
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ import { useAuth } from '../context/AuthContext';
|
|||||||
import { useImport } from '../context/ImportContext';
|
import { useImport } from '../context/ImportContext';
|
||||||
|
|
||||||
function ImportProgressBar() {
|
function ImportProgressBar() {
|
||||||
const { importState, clearImport } = useImport();
|
const { importState, clearImport, openModal } = useImport();
|
||||||
const [visible, setVisible] = useState(false);
|
const [visible, setVisible] = useState(false);
|
||||||
const hideTimerRef = useRef<ReturnType<typeof setTimeout> | undefined>(undefined);
|
const hideTimerRef = useRef<ReturnType<typeof setTimeout> | undefined>(undefined);
|
||||||
|
|
||||||
@@ -31,9 +31,9 @@ function ImportProgressBar() {
|
|||||||
|
|
||||||
const handleClick = useCallback(() => {
|
const handleClick = useCallback(() => {
|
||||||
if (hideTimerRef.current) clearTimeout(hideTimerRef.current);
|
if (hideTimerRef.current) clearTimeout(hideTimerRef.current);
|
||||||
|
openModal();
|
||||||
setVisible(false);
|
setVisible(false);
|
||||||
clearImport();
|
}, [openModal]);
|
||||||
}, [clearImport]);
|
|
||||||
|
|
||||||
if (!visible) return null;
|
if (!visible) return null;
|
||||||
|
|
||||||
|
|||||||
@@ -20,6 +20,9 @@ export interface ImportProgress {
|
|||||||
|
|
||||||
interface ImportContextValue {
|
interface ImportContextValue {
|
||||||
importState: ImportProgress;
|
importState: ImportProgress;
|
||||||
|
showModal: boolean;
|
||||||
|
openModal: () => void;
|
||||||
|
closeModal: () => void;
|
||||||
startImport: (file: File) => void;
|
startImport: (file: File) => void;
|
||||||
clearImport: () => void;
|
clearImport: () => void;
|
||||||
}
|
}
|
||||||
@@ -35,8 +38,12 @@ const ImportContext = createContext<ImportContextValue | null>(null);
|
|||||||
|
|
||||||
export function ImportProvider({ children }: { children: ReactNode }) {
|
export function ImportProvider({ children }: { children: ReactNode }) {
|
||||||
const [importState, setImportState] = useState<ImportProgress>(INITIAL);
|
const [importState, setImportState] = useState<ImportProgress>(INITIAL);
|
||||||
|
const [showModal, setShowModal] = useState(false);
|
||||||
const runningRef = useRef(false);
|
const runningRef = useRef(false);
|
||||||
|
|
||||||
|
const openModal = useCallback(() => setShowModal(true), []);
|
||||||
|
const closeModal = useCallback(() => setShowModal(false), []);
|
||||||
|
|
||||||
const startImport = useCallback((file: File) => {
|
const startImport = useCallback((file: File) => {
|
||||||
if (runningRef.current) return;
|
if (runningRef.current) return;
|
||||||
runningRef.current = true;
|
runningRef.current = true;
|
||||||
@@ -92,7 +99,9 @@ export function ImportProvider({ children }: { children: ReactNode }) {
|
|||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<ImportContext.Provider value={{ importState, startImport, clearImport }}>
|
<ImportContext.Provider value={{
|
||||||
|
importState, showModal, openModal, closeModal, startImport, clearImport,
|
||||||
|
}}>
|
||||||
{children}
|
{children}
|
||||||
</ImportContext.Provider>
|
</ImportContext.Provider>
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ import { TransactionTable } from '../components/TransactionTable';
|
|||||||
import { Pagination } from '../components/Pagination';
|
import { Pagination } from '../components/Pagination';
|
||||||
import { EditTransactionModal } from '../components/EditTransactionModal';
|
import { EditTransactionModal } from '../components/EditTransactionModal';
|
||||||
import { ImportModal } from '../components/ImportModal';
|
import { ImportModal } from '../components/ImportModal';
|
||||||
|
import { useImport } from '../context/ImportContext';
|
||||||
import { toISODate } from '../utils/format';
|
import { toISODate } from '../utils/format';
|
||||||
|
|
||||||
const PARAM_KEYS = [
|
const PARAM_KEYS = [
|
||||||
@@ -125,7 +126,7 @@ export function HistoryPage() {
|
|||||||
const [accounts, setAccounts] = useState<Account[]>([]);
|
const [accounts, setAccounts] = useState<Account[]>([]);
|
||||||
const [categories, setCategories] = useState<Category[]>([]);
|
const [categories, setCategories] = useState<Category[]>([]);
|
||||||
const [editingTx, setEditingTx] = useState<Transaction | null>(null);
|
const [editingTx, setEditingTx] = useState<Transaction | null>(null);
|
||||||
const [showImport, setShowImport] = useState(false);
|
const { showModal: showImport, openModal: openImport, closeModal: closeImport, clearImport } = useImport();
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
getAccounts().then(setAccounts).catch(() => {});
|
getAccounts().then(setAccounts).catch(() => {});
|
||||||
@@ -197,7 +198,8 @@ export function HistoryPage() {
|
|||||||
};
|
};
|
||||||
|
|
||||||
const handleImportDone = () => {
|
const handleImportDone = () => {
|
||||||
setShowImport(false);
|
closeImport();
|
||||||
|
clearImport();
|
||||||
fetchData();
|
fetchData();
|
||||||
getAccounts().then(setAccounts).catch(() => {});
|
getAccounts().then(setAccounts).catch(() => {});
|
||||||
};
|
};
|
||||||
@@ -208,7 +210,7 @@ export function HistoryPage() {
|
|||||||
<h1>История операций</h1>
|
<h1>История операций</h1>
|
||||||
<button
|
<button
|
||||||
className="btn btn-primary"
|
className="btn btn-primary"
|
||||||
onClick={() => setShowImport(true)}
|
onClick={openImport}
|
||||||
>
|
>
|
||||||
Импорт выписки
|
Импорт выписки
|
||||||
</button>
|
</button>
|
||||||
@@ -262,7 +264,7 @@ export function HistoryPage() {
|
|||||||
|
|
||||||
{showImport && (
|
{showImport && (
|
||||||
<ImportModal
|
<ImportModal
|
||||||
onClose={() => setShowImport(false)}
|
onClose={closeImport}
|
||||||
onDone={handleImportDone}
|
onDone={handleImportDone}
|
||||||
/>
|
/>
|
||||||
)}
|
)}
|
||||||
|
|||||||
Reference in New Issue
Block a user