64 lines
5.5 KiB
TypeScript
64 lines
5.5 KiB
TypeScript
import {lookup} from 'node:dns/promises';
|
|
import {request as httpRequest,type IncomingHttpHeaders} from 'node:http';
|
|
import {request as httpsRequest} from 'node:https';
|
|
import {createUnzip,createBrotliDecompress} from 'node:zlib';
|
|
import {waitForHost,limitHost} from './throttle.ts';
|
|
|
|
export function safeUrl(value:string,base?:string):string|null{
|
|
try{const u=new URL(value,base),h=u.hostname.toLowerCase();
|
|
if(!['https:','http:'].includes(u.protocol)||u.username||u.password||(u.port&&!['80','443'].includes(u.port)))return null;
|
|
if(!h.includes('.')||h.includes(':')||/^\d+[.\d]*$/.test(h)||h.endsWith('.')||/(^|\.)(localhost|local|internal|test|invalid|example|onion|arpa)$/.test(h))return null;
|
|
u.hash='';return u.href;
|
|
}catch{return null;}
|
|
}
|
|
export function publicIp(ip:string){
|
|
if(ip.includes(':')){const fields=ip.toLowerCase().split(':');return /^[23][0-9a-f]{3}:/i.test(ip)&&fields[0]!=='2002'&&fields[0]!=='3fff'&&!(fields[0]==='2001'&&(parseInt(fields[1]||'0',16)<=0x1ff||parseInt(fields[1],16)===0xdb8));}
|
|
const p=ip.split('.').map(Number);if(p.length!==4||p.some(n=>!Number.isInteger(n)||n<0||n>255))return false;
|
|
const [a,b,c]=p;return !(a===0||a===10||a===127||a>=224||a===169&&b===254||a===172&&b>=16&&b<=31||a===192&&(b===168||b===0||b===88&&c===99)||a===100&&b>=64&&b<=127||a===198&&(b===18||b===19||b===51&&c===100)||a===203&&b===0&&c===113);
|
|
}
|
|
export async function resolvePublicHost(host:string){
|
|
const addresses=await lookup(host,{all:true,verbatim:true});
|
|
if(!addresses.length)throw new Error('DNS: адрес сервера не найден');
|
|
if(addresses.some(a=>!publicIp(a.address)))throw new Error('DNS указывает на непубличный адрес; запрос запрещён');
|
|
return addresses.find(a=>a.family===4)??addresses[0];
|
|
}
|
|
export type PublicResponse={code:number;headers:IncomingHttpHeaders;body:Buffer;type:string};
|
|
// One pinned socket, no implicit redirects/proxies; browser redirects pass this boundary again.
|
|
export async function requestPublic(raw:string,options:{signal?:AbortSignal;maxBytes?:number;headers?:Record<string,string>;method?:'GET'|'HEAD'|'OPTIONS'}={}):Promise<PublicResponse>{
|
|
const url=safeUrl(raw);if(!url)throw new Error('Адрес не разрешён политикой безопасности');
|
|
const signal=AbortSignal.any([AbortSignal.timeout(12000),...(options.signal?[options.signal]:[])]);
|
|
const aborted=new Promise<never>((_,reject)=>{if(signal.aborted)reject(new DOMException('Истекло время ожидания','AbortError'));else signal.addEventListener('abort',()=>reject(new DOMException('Истекло время ожидания','AbortError')),{once:true});});
|
|
const target=new URL(url),ip=await Promise.race([resolvePublicHost(target.hostname),aborted]);
|
|
return new Promise((resolve,reject)=>{
|
|
const request=(target.protocol==='https:'?httpsRequest:httpRequest)(target,{
|
|
agent:false,family:ip.family,signal,lookup:(_hostname,_options,callback)=>callback(null,ip.address,ip.family),method:options.method??'GET',
|
|
headers:{'User-Agent':'Sverka/1.3 (academic bibliography access check)','Accept':'text/html,application/pdf;q=0.9,*/*;q=0.8','Accept-Encoding':'identity',...options.headers},
|
|
},response=>{
|
|
const code=response.statusCode??0,headers=response.headers,type=String(headers['content-type']??'');
|
|
if(code>=300&&code<400||[429,503].includes(code)||options.method==='HEAD'){response.destroy();resolve({code,headers,type,body:Buffer.alloc(0)});return;}
|
|
void(async()=>{
|
|
const encoding=String(headers['content-encoding']??''),stream=/gzip|deflate/i.test(encoding)?response.pipe(createUnzip()):/br/i.test(encoding)?response.pipe(createBrotliDecompress()):response;
|
|
response.on('error',error=>stream.destroy(error));response.on('aborted',()=>stream.destroy(new Error('Соединение прервано')));
|
|
request.once('error',error=>{response.destroy();stream.destroy(error);});
|
|
const chunks:Buffer[]=[];let size=0;
|
|
try{for await(const chunk of stream){const bytes=Buffer.from(chunk);size+=bytes.length;if(size>(options.maxBytes??3000000))throw new Error('Превышен допустимый размер загрузки');chunks.push(bytes);}resolve({code,headers,type,body:Buffer.concat(chunks)});}
|
|
catch(error){response.destroy();stream.destroy();reject(error);}
|
|
})();
|
|
});request.once('error',reject);request.end();
|
|
});
|
|
}
|
|
export function decodeBody(body:Buffer,type:string){try{return new TextDecoder(/charset=([\w-]+)/i.exec(type)?.[1]??'utf-8').decode(body);}catch{return body.toString('utf8');}}
|
|
export async function retrieve(raw:string,source=false,parentSignal?:AbortSignal){
|
|
let url=safeUrl(raw);if(!url)throw new Error('Адрес не разрешён политикой безопасности');
|
|
const signal=AbortSignal.any([AbortSignal.timeout(30000),...(parentSignal?[parentSignal]:[])]);
|
|
for(let hop=0;hop<5;hop++){
|
|
await waitForHost(new URL(url).hostname,signal);
|
|
const result=await requestPublic(url,{signal,maxBytes:source?3000000:1500000});
|
|
if(result.code!==403||source)await limitHost(new URL(url).hostname,result.code,result.headers['retry-after']);
|
|
const location=result.headers.location;
|
|
if(location&&result.code>=300&&result.code<400){const next=safeUrl(location,url);if(!next)throw new Error('Небезопасное перенаправление');url=next;continue;}
|
|
return {code:result.code,url,type:result.type,body:decodeBody(result.body,result.type)};
|
|
}
|
|
throw new Error('Слишком много перенаправлений');
|
|
}
|