AppInitDev
CÓDIGO FUENTE PÚBLICO

tools/rastreador.py

Archivo real del paquete para implementar la función. Descárgalo del ZIP para editarlo; no contiene claves privadas.

#!/usr/bin/env python3
"""Rastreador explícito de contenido público. Respeta robots, límites y host HTTPS.
NO se ejecuta automáticamente durante la instalación. Cache privado en AppInitDev_IA_MCP/cache_v2/.
"""
import hashlib, json, re, sys, time
from datetime import datetime, timezone, timedelta
from pathlib import Path
from urllib.parse import urlsplit
from urllib.request import Request, build_opener, HTTPRedirectHandler
from urllib.error import URLError, HTTPError
from urllib.robotparser import RobotFileParser

HERE=Path(__file__).resolve().parent.parent

def permitido(url):
    try:
        u=urlsplit(url)
        return (u.scheme=='https' and u.hostname=='appinitdev.com' and not u.port and
                not u.username and not u.password and not u.query and not u.fragment and
                not re.search(r'/(?:admin|login|reportes|privado|backup|backups|\.git|\.well-known)/',u.path,re.I) and
                not re.search(r'\.(?:json|pdf|zip|xml|png|jpg|mp4|css|js|svg|webp)$',u.path,re.I))
    except (ValueError,TypeError):return False

class SafeRedirect(HTTPRedirectHandler):
    def redirect_request(self,req,fp,code,msg,headers,newurl):
        if not permitido(newurl):raise URLError('Redirección no permitida')
        return super().redirect_request(req,fp,code,msg,headers,newurl)

def guardar(path,obj):
    path.parent.mkdir(parents=True,exist_ok=True)
    p=path.with_suffix('.tmp');p.write_text(json.dumps(obj,ensure_ascii=False,indent=2),encoding='utf8');p.replace(path)

def key(url):return hashlib.sha256(url.encode('utf8')).hexdigest()

def main():
    root=HERE.parent
    conf=json.loads((HERE/'CONFIG.json').read_text(encoding='utf8'))['rastreador']
    cap=max(1,min(2000,int(conf.get('maximo_paginas_por_ejecucion',80))))
    gap=max(.5,float(conf.get('retraso_segundos',.8)))
    timeout=max(3,min(20,int(conf.get('timeout_segundos',9))))
    maxbytes=max(40000,min(1000000,int(conf.get('maximo_html_bytes',700000))))
    ttl=max(1,int(conf.get('maximo_dias_cache',30)))
    ua=str(conf.get('user_agent','AppInitDevKnowledgeIndexer/2.0'))[:180]
    catalog=root/'datos/conocimiento.json'
    if not catalog.exists():raise SystemExit('Primero ejecuta INSTALAR_IA.py para generar datos/conocimiento.json')
    rows=json.loads(catalog.read_text(encoding='utf8'))['items']
    opener=build_opener(SafeRedirect())
    robots=RobotFileParser()
    try:
        with opener.open(Request('https://appinitdev.com/robots.txt',headers={'User-Agent':ua}),timeout=timeout) as r:
            raw=r.read(300000).decode('utf8','replace')
            if r.status!=200:raise ValueError('robots.txt sin HTTP 200')
        robots.parse(raw.splitlines())
    except (HTTPError,URLError,OSError,ValueError) as e:
        raise SystemExit('No se pudo verificar robots.txt: por seguridad se detiene el rastreo. '+str(e))
    paths=HERE/'cache_v2/descargas';paths.mkdir(parents=True,exist_ok=True)
    audit=[];tried=0;downloaded=0;cache_hits=0;blocked=0
    for doc in rows:
        url=doc.get('url','')
        if not permitido(url) or not robots.can_fetch(ua,url):
            blocked+=1;continue
        target=paths/(key(url)+'.json')
        old={}
        if target.is_file():
            try:old=json.loads(target.read_text(encoding='utf8'))
            except (OSError,ValueError):pass
        when=old.get('comprobado_utc')
        if when:
            try:
                if datetime.fromisoformat(when)>datetime.now(timezone.utc)-timedelta(days=ttl):
                    cache_hits+=1;continue
            except ValueError:pass
        if tried>=cap:break
        tried+=1
        headers={'User-Agent':ua,'Accept':'text/html','Accept-Encoding':'identity'}
        if old.get('etag'):headers['If-None-Match']=old['etag']
        if old.get('last_modified'):headers['If-Modified-Since']=old['last_modified']
        event={'url':url}
        try:
            try:r=opener.open(Request(url,headers=headers),timeout=timeout)
            except HTTPError as e:
                if e.code==304:
                    if old.get('html'):
                        old['comprobado_utc']=datetime.now(timezone.utc).isoformat(timespec='seconds')
                        guardar(target,old);cache_hits+=1;continue
                raise
            with r:
                final=r.url
                if not permitido(final):raise ValueError('Destino final no permitido')
                if 'text/html' not in (r.headers.get('Content-Type','')).lower():raise ValueError('Respuesta no HTML')
                if int(r.headers.get('Content-Length','0') or '0')>maxbytes:raise ValueError('HTML demasiado grande')
                data=r.read(maxbytes+1)
                if len(data)>maxbytes:raise ValueError('HTML demasiado grande')
                html=data.decode('utf8','replace')
                head=html[:12000]
                if re.search(r'<meta\s+[^>]*name\s*=\s*["\']?(?:robots|googlebot)["\']?[^>]*content\s*=\s*["\']?[^>]*noindex',head,re.I):
                    raise ValueError('Página noindex')
                if re.search(r'<meta\s+[^>]*content\s*=\s*["\']?[^>]*noindex[^>]*name\s*=\s*["\']?(?:robots|googlebot)',head,re.I):
                    raise ValueError('Página noindex')
                # Cabecera X-Robots-Tag de noindex.
                if 'noindex' in r.headers.get('X-Robots-Tag','').lower():raise ValueError('X-Robots-Tag noindex')
                guardar(target,{'url':url,'url_final':final,'html':html,'etag':r.headers.get('ETag'),
                              'last_modified':r.headers.get('Last-Modified'),
                              'comprobado_utc':datetime.now(timezone.utc).isoformat(timespec='seconds'),
                              'sha256':hashlib.sha256(data).hexdigest(),'tipo':'html_publico'})
                downloaded+=1;event['estado']='descargada'
        except (HTTPError,URLError,OSError,ValueError,TimeoutError) as e:
            event['estado']='no_descargada';event['detalle']=str(e)[:100]
            if isinstance(e,HTTPError) and e.code in (429,503):
                audit.append(event);print('Límite de servicio remoto; detenemos el lote.');break
        audit.append(event)
        if tried<cap:time.sleep(gap)
    summary={'fecha_utc':datetime.now(timezone.utc).isoformat(timespec='seconds'),
             'total_en_catalogo':len(rows),'intentos':tried,'descargadas':downloaded,
             'cache_vigente':cache_hits,'excluidas_por_reglas_o_robots':blocked,
             'fallos':[v for v in audit if v['estado']!='descargada'][:100],
             'nota':'Caché privado: no subir a public_html. No se rastrean dominios externos.'}
    guardar(HERE/'reportes/rastreo.json',summary)
    print(json.dumps({k:v for k,v in summary.items() if k!='fallos'},ensure_ascii=False,indent=2))
    print('Vuelve a ejecutar INSTALAR_IA.py para incorporar el texto descargado al índice.')

if __name__=='__main__': main()