tools/rastreador.py
Archivo real del paquete para implementar la función. Descárgalo del ZIP para editarlo; no contiene claves privadas.
#!/usr/bin/env python3
"""Rastreador explícito de contenido público. Respeta robots, límites y host HTTPS.
NO se ejecuta automáticamente durante la instalación. Cache privado en AppInitDev_IA_MCP/cache_v2/.
"""
import hashlib, json, re, sys, time
from datetime import datetime, timezone, timedelta
from pathlib import Path
from urllib.parse import urlsplit
from urllib.request import Request, build_opener, HTTPRedirectHandler
from urllib.error import URLError, HTTPError
from urllib.robotparser import RobotFileParser
HERE=Path(__file__).resolve().parent.parent
def permitido(url):
try:
u=urlsplit(url)
return (u.scheme=='https' and u.hostname=='appinitdev.com' and not u.port and
not u.username and not u.password and not u.query and not u.fragment and
not re.search(r'/(?:admin|login|reportes|privado|backup|backups|\.git|\.well-known)/',u.path,re.I) and
not re.search(r'\.(?:json|pdf|zip|xml|png|jpg|mp4|css|js|svg|webp)$',u.path,re.I))
except (ValueError,TypeError):return False
class SafeRedirect(HTTPRedirectHandler):
def redirect_request(self,req,fp,code,msg,headers,newurl):
if not permitido(newurl):raise URLError('Redirección no permitida')
return super().redirect_request(req,fp,code,msg,headers,newurl)
def guardar(path,obj):
path.parent.mkdir(parents=True,exist_ok=True)
p=path.with_suffix('.tmp');p.write_text(json.dumps(obj,ensure_ascii=False,indent=2),encoding='utf8');p.replace(path)
def key(url):return hashlib.sha256(url.encode('utf8')).hexdigest()
def main():
root=HERE.parent
conf=json.loads((HERE/'CONFIG.json').read_text(encoding='utf8'))['rastreador']
cap=max(1,min(2000,int(conf.get('maximo_paginas_por_ejecucion',80))))
gap=max(.5,float(conf.get('retraso_segundos',.8)))
timeout=max(3,min(20,int(conf.get('timeout_segundos',9))))
maxbytes=max(40000,min(1000000,int(conf.get('maximo_html_bytes',700000))))
ttl=max(1,int(conf.get('maximo_dias_cache',30)))
ua=str(conf.get('user_agent','AppInitDevKnowledgeIndexer/2.0'))[:180]
catalog=root/'datos/conocimiento.json'
if not catalog.exists():raise SystemExit('Primero ejecuta INSTALAR_IA.py para generar datos/conocimiento.json')
rows=json.loads(catalog.read_text(encoding='utf8'))['items']
opener=build_opener(SafeRedirect())
robots=RobotFileParser()
try:
with opener.open(Request('https://appinitdev.com/robots.txt',headers={'User-Agent':ua}),timeout=timeout) as r:
raw=r.read(300000).decode('utf8','replace')
if r.status!=200:raise ValueError('robots.txt sin HTTP 200')
robots.parse(raw.splitlines())
except (HTTPError,URLError,OSError,ValueError) as e:
raise SystemExit('No se pudo verificar robots.txt: por seguridad se detiene el rastreo. '+str(e))
paths=HERE/'cache_v2/descargas';paths.mkdir(parents=True,exist_ok=True)
audit=[];tried=0;downloaded=0;cache_hits=0;blocked=0
for doc in rows:
url=doc.get('url','')
if not permitido(url) or not robots.can_fetch(ua,url):
blocked+=1;continue
target=paths/(key(url)+'.json')
old={}
if target.is_file():
try:old=json.loads(target.read_text(encoding='utf8'))
except (OSError,ValueError):pass
when=old.get('comprobado_utc')
if when:
try:
if datetime.fromisoformat(when)>datetime.now(timezone.utc)-timedelta(days=ttl):
cache_hits+=1;continue
except ValueError:pass
if tried>=cap:break
tried+=1
headers={'User-Agent':ua,'Accept':'text/html','Accept-Encoding':'identity'}
if old.get('etag'):headers['If-None-Match']=old['etag']
if old.get('last_modified'):headers['If-Modified-Since']=old['last_modified']
event={'url':url}
try:
try:r=opener.open(Request(url,headers=headers),timeout=timeout)
except HTTPError as e:
if e.code==304:
if old.get('html'):
old['comprobado_utc']=datetime.now(timezone.utc).isoformat(timespec='seconds')
guardar(target,old);cache_hits+=1;continue
raise
with r:
final=r.url
if not permitido(final):raise ValueError('Destino final no permitido')
if 'text/html' not in (r.headers.get('Content-Type','')).lower():raise ValueError('Respuesta no HTML')
if int(r.headers.get('Content-Length','0') or '0')>maxbytes:raise ValueError('HTML demasiado grande')
data=r.read(maxbytes+1)
if len(data)>maxbytes:raise ValueError('HTML demasiado grande')
html=data.decode('utf8','replace')
head=html[:12000]
if re.search(r'<meta\s+[^>]*name\s*=\s*["\']?(?:robots|googlebot)["\']?[^>]*content\s*=\s*["\']?[^>]*noindex',head,re.I):
raise ValueError('Página noindex')
if re.search(r'<meta\s+[^>]*content\s*=\s*["\']?[^>]*noindex[^>]*name\s*=\s*["\']?(?:robots|googlebot)',head,re.I):
raise ValueError('Página noindex')
# Cabecera X-Robots-Tag de noindex.
if 'noindex' in r.headers.get('X-Robots-Tag','').lower():raise ValueError('X-Robots-Tag noindex')
guardar(target,{'url':url,'url_final':final,'html':html,'etag':r.headers.get('ETag'),
'last_modified':r.headers.get('Last-Modified'),
'comprobado_utc':datetime.now(timezone.utc).isoformat(timespec='seconds'),
'sha256':hashlib.sha256(data).hexdigest(),'tipo':'html_publico'})
downloaded+=1;event['estado']='descargada'
except (HTTPError,URLError,OSError,ValueError,TimeoutError) as e:
event['estado']='no_descargada';event['detalle']=str(e)[:100]
if isinstance(e,HTTPError) and e.code in (429,503):
audit.append(event);print('Límite de servicio remoto; detenemos el lote.');break
audit.append(event)
if tried<cap:time.sleep(gap)
summary={'fecha_utc':datetime.now(timezone.utc).isoformat(timespec='seconds'),
'total_en_catalogo':len(rows),'intentos':tried,'descargadas':downloaded,
'cache_vigente':cache_hits,'excluidas_por_reglas_o_robots':blocked,
'fallos':[v for v in audit if v['estado']!='descargada'][:100],
'nota':'Caché privado: no subir a public_html. No se rastrean dominios externos.'}
guardar(HERE/'reportes/rastreo.json',summary)
print(json.dumps({k:v for k,v in summary.items() if k!='fallos'},ensure_ascii=False,indent=2))
print('Vuelve a ejecutar INSTALAR_IA.py para incorporar el texto descargado al índice.')
if __name__=='__main__': main()