41 lines
4.2 KiB
TypeScript
41 lines
4.2 KiB
TypeScript
import {parseHTML} from 'linkedom';
|
|
import type {Check,Reference} from './model';
|
|
import {HostCooldown,limitHost} from './throttle.ts';
|
|
import {safeUrl,retrieve} from './http.ts';
|
|
import {classifyResponse,result} from './access.ts';
|
|
import {provider} from './provider.ts';
|
|
export function clean(text:string){return text.replace(/\s+/g,' ').trim();}
|
|
export function trimUrl(value:string){let v=value.replace(/[.,;:!?]+$/g,'');for(const [open,close] of [['(',')'],['[',']'],['{','}']])while(v.endsWith(close)&&v.split(close).length>v.split(open).length)v=v.slice(0,-1);return v;}
|
|
export function parseCatalog(html:string,url:string,root:string){
|
|
const {document}=parseHTML(html),courses=new Map<string,{url:string;name:string}>(),pages=new Set<string>();
|
|
for(const a of document.querySelectorAll('a[href]')){const href=safeUrl(a.getAttribute('href')!,url);if(!href)continue;
|
|
const u=new URL(href),r=new URL(root);if(u.origin!==r.origin)continue;
|
|
if(new RegExp('^'+r.pathname+'/[0-9]+\\.html$').test(u.pathname))courses.set(u.origin+u.pathname,{url:u.origin+u.pathname,name:clean(a.textContent??'')});
|
|
if(new RegExp('^'+r.pathname+'/page[0-9]+\\.html$').test(u.pathname)&&(url===root||u.search===new URL(url).search))pages.add(href);}
|
|
return {courses:[...courses.values()],pages:[...pages]};}
|
|
export function parseCourse(html:string,url:string){
|
|
const {document}=parseHTML(html),refs=new Map<string,Reference>();
|
|
const sections=[...document.querySelectorAll('.pud__section')].filter(s=>/список литературы|основная литература|дополнительная литература|reading list|bibliography/i.test(s.querySelector('h4,h3,h2')?.textContent??''));
|
|
let found=sections.length>0;
|
|
if(!found){const h=[...document.querySelectorAll('h2,h3,h4')].find(h=>/^(список литературы|литература|reading list|bibliography)$/i.test(clean(h.textContent??'')));if(h){found=true;let n=h.nextElementSibling;while(n&&!/^H[1-4]$/.test(n.tagName)){sections.push(n);n=n.nextElementSibling;}}}
|
|
for(const s of sections){const items=[...s.querySelectorAll('li')];for(const item of items.length?items:[s]){
|
|
const title=clean(item.textContent??''),urls=[...item.querySelectorAll('a[href]')].map(a=>a.getAttribute('href')!);
|
|
urls.push(...(title.match(/https?:\/\/[^\s<>"«»]+/gi)??[]).map(trimUrl));
|
|
for(const raw of urls){const normalized=safeUrl(raw,url);if(normalized&&normalized!==url)refs.set(normalized,{url:normalized,title:title.slice(0,2000)});}}}
|
|
const content=[...document.querySelectorAll('.pud__section')].map(s=>clean(s.textContent??'')).join('\n')||clean(document.querySelector('main')?.textContent??document.querySelector('h1')?.textContent??'');
|
|
return {name:clean(document.querySelector('h1')?.textContent??''),refs:[...refs.values()],literature:found,content};}
|
|
export async function fingerprint(v:string){return [...new Uint8Array(await crypto.subtle.digest('SHA-256',new TextEncoder().encode(v)))].map(n=>n.toString(16).padStart(2,'0')).join('');}
|
|
export {safeUrl,publicIp,resolvePublicHost,retrieve} from './http.ts';
|
|
export {classifyResponse as classify} from './access.ts';
|
|
export async function checkLink(url:string):Promise<Check>{
|
|
try{
|
|
const signal=AbortSignal.timeout(60000);
|
|
// Known libraries contain hidden auth/paywall templates; only rendered evidence can decide their access.
|
|
if(provider(url))return await (await import('./browser.ts')).checkBrowser(url,signal);
|
|
const page=await retrieve(url,false,signal),check=classifyResponse(url,page);
|
|
if(page.code===403&&check.status==='blocked')await limitHost(new URL(page.url).hostname,403);
|
|
if(check.evidence&&['auth','restricted'].includes(check.status))return {...check,status:'unknown',reason:'В HTML есть признаки ограничения, но доступ к полному тексту не подтверждён отрисованной страницей.',evidence:undefined};
|
|
return check;
|
|
}catch(e){const error=e as Error;return result(url,error instanceof HostCooldown?'blocked':error.name==='AbortError'?'timeout':'network',error instanceof HostCooldown?error.message:'Не удалось загрузить материал. Доступ к полному тексту не подтверждён.');}
|
|
}
|