132 lines
13 KiB
TypeScript
132 lines
13 KiB
TypeScript
import {chromium,type Route} from 'playwright';
|
|
import {requestPublic,safeUrl} from './http.ts';
|
|
import {waitForHost,limitHost,HostCooldown} from './throttle.ts';
|
|
import {accessMessage,purchaseOffer,classifyResponse,result,evidenceUrl} from './access.ts';
|
|
import type {Check} from './model.ts';
|
|
|
|
import {provider} from './provider.ts';
|
|
export function browserRequestAllowed(raw:string,root:string,method:string,resource:string){
|
|
const url=safeUrl(raw);if(!url)return false;const u=new URL(url);
|
|
return u.protocol==='https:'&&provider(u.href)===root&&['GET','HEAD','OPTIONS'].includes(method)&&!['image','media','font','manifest'].includes(resource);
|
|
}
|
|
export async function checkBrowser(url:string,parentSignal?:AbortSignal):Promise<Check>{
|
|
const root=provider(url);if(!root)return result(url,'unknown','Для этого сайта нет правил проверки полного текста.');
|
|
const controller=new AbortController(),timer=setTimeout(()=>controller.abort(),30000);
|
|
const abort=()=>controller.abort();parentSignal?.addEventListener('abort',abort,{once:true});if(parentSignal?.aborted)controller.abort();
|
|
let browser:Awaited<ReturnType<typeof chromium.launch>>|undefined;
|
|
let requests=0,totalBytes=0,failed=false,navigations=0,mainCode=200,redirectTarget:string|null=null,limit:Check|null=null;
|
|
const active=new Set<Promise<void>>();
|
|
const slots:Promise<void>[]=Array.from({length:10},()=>Promise.resolve());
|
|
try{
|
|
controller.signal.throwIfAborted();browser=await chromium.launch({headless:true,chromiumSandbox:true,timeout:10000,env:{PATH:process.env.PATH??'/usr/bin:/bin',HOME:'/tmp',TMPDIR:'/tmp',LANG:'C.UTF-8'},args:[
|
|
'--proxy-server=http://127.0.0.1:9','--proxy-bypass-list=<-loopback>','--disable-quic','--force-webrtc-ip-handling-policy=disable_non_proxied_udp',
|
|
]});
|
|
controller.signal.throwIfAborted();const context=await browser.newContext({serviceWorkers:'block',acceptDownloads:false,javaScriptEnabled:!['znanium.ru','ibooks.ru','springer.com'].includes(root),viewport:{width:1280,height:900}});
|
|
controller.signal.addEventListener('abort',()=>{void context.close().catch(()=>{});},{once:true});
|
|
await context.routeWebSocket('**/*',socket=>socket.close());
|
|
const handle=async(route:Route)=>{
|
|
const request=route.request(),target=request.url(),navigation=request.isNavigationRequest();
|
|
try{
|
|
if(controller.signal.aborted||limit||!browserRequestAllowed(target,root,request.method(),request.resourceType())){await route.abort();return;}
|
|
if(totalBytes>32000000)throw new Error('Превышен лимит ресурсов страницы');
|
|
if(navigation){if(++navigations>8)throw new Error('Слишком много переходов');await waitForHost(new URL(target).hostname,controller.signal);}
|
|
const allHeaders=await request.allHeaders(),headers:Record<string,string>={};for(const key of ['accept','accept-language','origin','referer','cookie','content-type','user-agent']){const value=allHeaders[key];if(value)headers[key]=value;}
|
|
const response=await requestPublic(target,{signal:controller.signal,maxBytes:8000000,method:request.method() as 'GET'|'HEAD'|'OPTIONS',headers});
|
|
totalBytes+=response.body.length;if(totalBytes>32000000)throw new Error('Превышен лимит ресурсов страницы');
|
|
if([429,503].includes(response.code)){
|
|
await limitHost(new URL(target).hostname,response.code,response.headers['retry-after']);
|
|
limit=result(url,'blocked',`Сайт остановил автоматическую проверку (HTTP ${response.code}). Доступ к полному тексту не определён.`);
|
|
}
|
|
if(navigation&&request.frame()===page.mainFrame())mainCode=response.code;
|
|
const fulfilled:Record<string,string>={};
|
|
for(const [key,value] of Object.entries(response.headers))if(value!==undefined&&!['content-encoding','content-length','transfer-encoding','connection'].includes(key))fulfilled[key]=Array.isArray(value)?value.join('\n'):String(value);
|
|
if([301,302,303,307,308].includes(response.code)&&response.headers.location){
|
|
if(!navigation||request.frame()!==page.mainFrame())throw new Error('Перенаправление ресурса не подтверждено');
|
|
const next=safeUrl(response.headers.location,target);if(!next||!browserRequestAllowed(next,root,'GET','document'))throw new Error('Перенаправление не разрешено');
|
|
redirectTarget=next;delete fulfilled.location;delete fulfilled['content-type'];
|
|
// Chromium routing may skip native redirect hops. Start each hop explicitly through the pinned boundary.
|
|
await route.fulfill({status:200,headers:fulfilled,contentType:'text/html',body:'<!doctype html><html><body></body></html>'});return;
|
|
}
|
|
await route.fulfill({status:response.code,headers:fulfilled,body:response.body});
|
|
}catch(error){
|
|
if(error instanceof HostCooldown)limit=result(url,'blocked',error.message);
|
|
else if(navigation||['script','xhr','fetch','stylesheet'].includes(request.resourceType()))failed=true;
|
|
await route.abort().catch(()=>{});
|
|
}
|
|
};
|
|
// Every request is fulfilled by the pinned Node transport. The dead proxy also prevents direct browser egress.
|
|
await context.route('**/*',route=>{
|
|
const request=route.request();
|
|
if(controller.signal.aborted||limit||!browserRequestAllowed(request.url(),root,request.method(),request.resourceType()))return route.abort().catch(()=>{});
|
|
if(++requests>100){failed=true;return route.abort().catch(()=>{});}
|
|
const index=(requests-1)%slots.length,task=slots[index].then(()=>handle(route));slots[index]=task.catch(()=>{});active.add(task);void task.finally(()=>active.delete(task));return task;
|
|
});
|
|
const page=await context.newPage();
|
|
context.on('page',other=>{if(other!==page)void other.close().catch(()=>{});});
|
|
page.setDefaultTimeout(5000);page.setDefaultNavigationTimeout(20000);
|
|
const navigate=async(raw:string)=>{let next:string=raw;for(let hop=0;hop<5;hop++){redirectTarget=null;await page.goto(next,{waitUntil:'domcontentloaded'});if(!redirectTarget)return;next=redirectTarget;}throw new Error('Слишком много перенаправлений');};
|
|
const capture=async()=>{
|
|
await page.locator('body').waitFor({state:'attached'});
|
|
// Only rendered text is evidence; hidden login templates and site chrome cannot decide book access.
|
|
const text=await page.locator('body').evaluate(body=>{
|
|
const walker=document.createTreeWalker(body,NodeFilter.SHOW_TEXT),parts:string[]=[];let node:Node|null,size=0;
|
|
while((node=walker.nextNode())&&size<100000){const el=node.parentElement;if(!el||el.closest('header,footer,nav,script,style,[role="navigation"],[role="banner"],[role="contentinfo"]'))continue;const style=getComputedStyle(el);if(style.visibility==='hidden'||style.display==='none'||!el.getClientRects().length)continue;const text=node.textContent??'';parts.push(text);size+=text.length;}
|
|
return parts.join(' ');
|
|
});
|
|
const offers=await page.locator('.book-offers__item,.app-article-access').evaluateAll(nodes=>nodes.filter(node=>{const style=getComputedStyle(node);return style.display!=='none'&&style.visibility!=='hidden'&&node.getClientRects().length>0;}).map(node=>{
|
|
const clone=node.cloneNode(true) as Element,original=[...node.querySelectorAll('*')],copied=[...clone.querySelectorAll('*')];
|
|
original.forEach((el,i)=>{const style=getComputedStyle(el);if(style.display==='none'||style.visibility==='hidden'||!el.getClientRects().length)copied[i].remove();});
|
|
return clone.outerHTML;
|
|
}).join(''));
|
|
return {text,offers,url:page.url(),title:await page.title()};
|
|
};
|
|
const settle=async()=>{
|
|
await page.waitForFunction(()=>{
|
|
const text=document.body?.innerText??'';
|
|
return /чтение книги недоступно|необходимо авторизоваться|публикация изъята|чтобы получить полный доступ|книга недоступна для чтения|для продолжения работы с сайтом|библиографическое описание|not found|captcha|проверка браузера/i.test(text)||!!document.querySelector('a[href*="/course-viewer/"],a[href*="/viewer/"],a[href*="reader.new.book.ru"],.book-offers__item,.app-article-access');
|
|
},undefined,{timeout:12000}).catch(()=>{});
|
|
};
|
|
await navigate(url);await settle();if(['urait.ru','book.ru'].includes(root))await page.waitForLoadState('networkidle',{timeout:4000}).catch(()=>{});let snapshot=await capture();
|
|
if(root==='book.ru'){
|
|
// CSS tooltips become part of the accessible name on hover; retain a stable DOM-text locator.
|
|
const preview=page.locator('a[href*="reader.new.book.ru"]').filter({hasText:'Ознакомиться',visible:true}).first();
|
|
if(await preview.count()){
|
|
await preview.hover({timeout:3000}).catch(()=>{});
|
|
await page.waitForFunction(()=>/гостевом\s+режиме|доступны\s+только\s+\d+\s+страниц/i.test(document.body?.innerText??''),undefined,{timeout:3000}).catch(()=>{});snapshot=await capture();
|
|
const tooltip=await preview.evaluate(el=>[el,el.parentElement].filter((node):node is HTMLElement=>!!node).flatMap(node=>['::before','::after'].map(pseudo=>{const style=getComputedStyle(node,pseudo);return style.display!=='none'&&style.visibility!=='hidden'&&style.opacity!=='0'&&!['none','normal'].includes(style.content)?style.content.replace(/^["']|["']$/g,''):'';})).join(' ')).catch(()=>'');
|
|
snapshot.text+=' '+tooltip;
|
|
const message=accessMessage(snapshot.url,snapshot.text);if(message)return {...message,url,code:mainCode,finalUrl:evidenceUrl(snapshot.url)};
|
|
// BOOK.ru distinguishes its locked orange preview action from a full reading action.
|
|
const lockedPreview=await preview.evaluate(el=>el.className.includes('book_mode_orange__')&&!![...el.querySelectorAll('[class*="book_marker2__"]')].find(node=>{const style=getComputedStyle(node);return style.display!=='none'&&style.visibility!=='hidden'&&node.getClientRects().length>0;}));
|
|
const fullRead=await page.locator('a[href*="reader.new.book.ru"]').filter({hasText:/^\s*Читать(?: полностью)?\s*$/i}).count();
|
|
if(lockedPreview&&!fullRead)return result(url,'restricted','Доступен только ознакомительный режим; полный текст книги недоступен.',{code:mainCode,finalUrl:evidenceUrl(snapshot.url),evidence:{url:evidenceUrl(snapshot.url),text:'Ознакомиться',rule:'bookru-locked-preview'}});
|
|
|
|
}
|
|
}
|
|
const known=accessMessage(snapshot.url,snapshot.text)??purchaseOffer(snapshot.url,snapshot.offers);
|
|
if(known)return {...known,url,code:mainCode,finalUrl:evidenceUrl(snapshot.url)};
|
|
if(root==='urait.ru'||root==='ibooks.ru'){
|
|
// Follow only a book reader link already present on this exact card, never subscription/purchase controls.
|
|
const modern=root==='urait.ru'?await page.locator('a[href*="/course-viewer/"]').first().getAttribute('href').catch(()=>null):null;
|
|
const href=root==='urait.ru'?modern??await page.locator('a[href*="/viewer/"]').first().getAttribute('href').catch(()=>null):snapshot.text.match(/https:\/\/ibooks\.ru\/bookshelf\/\d+\/reading/)?.[0]??null;
|
|
const reader=href?safeUrl(href,snapshot.url):null;
|
|
if(reader&&provider(reader)===root){await navigate(reader);await settle();snapshot=await capture();
|
|
if(root==='urait.ru'&&new URL(snapshot.url).pathname.startsWith('/viewer/')){
|
|
const nextHref=await page.locator('a[href*="/course-viewer/"]').first().getAttribute('href').catch(()=>null),next=nextHref?safeUrl(nextHref,snapshot.url):null;
|
|
if(next&&provider(next)===root){await navigate(next);await settle();snapshot=await capture();}
|
|
}
|
|
const message=accessMessage(snapshot.url,snapshot.text);if(message)return {...message,url,code:mainCode,finalUrl:evidenceUrl(snapshot.url)};
|
|
}
|
|
}
|
|
if(limit)return limit;
|
|
const escape=(text:string)=>text.replaceAll('&','&').replaceAll('<','<');
|
|
const classified=classifyResponse(url,{code:mainCode,url:snapshot.url,body:'<html><title>'+escape(snapshot.title)+'</title><body>'+escape(snapshot.text)+'</body></html>',type:'text/html'});
|
|
if(mainCode===403&&classified.status==='blocked')await limitHost(new URL(snapshot.url).hostname,403);
|
|
if(failed&&classified.status==='unknown')classified.reason='Страница загрузилась не полностью. Доступ к полному тексту не подтверждён.';
|
|
return classified;
|
|
}catch(error){
|
|
if(limit)return limit;
|
|
return result(url,'unknown',controller.signal.aborted?'Не удалось определить доступ к полному тексту за отведённое время.':'Не удалось завершить браузерную проверку. Доступ к полному тексту не подтверждён.');
|
|
}finally{clearTimeout(timer);parentSignal?.removeEventListener('abort',abort);controller.abort();await browser?.close().catch(()=>{});await Promise.allSettled(active);}
|
|
}
|