parser.py

import requests
import re
import psycopg2
import os
from bs4 import BeautifulSoup
from dotenv import load_dotenv

load_dotenv()

conn = psycopg2.connect(
    dbname=os.getenv('DB_NAME', 'parser_db'),
    user=os.getenv('DB_USER', 'postgres'),
    password=os.getenv('DB_PASSWORD'),
    host=os.getenv('DB_HOST', 'localhost')
)
cursor = conn.cursor()

def clean_text(text, limit=1000):
    """Очищает текст и обрезает до limit символов"""
    cleaned = ' '.join(text.split())
    return cleaned[:limit]

with open('sites.txt', 'r') as file:
    urls = [line.strip() for line in file if line.strip() and not line.startswith('#')]

headers = {'User-Agent': 'Mozilla/5.0'}

for url in urls:
    try:
        print(f"Парсим: {url}")
        response = requests.get(url, headers=headers, timeout=10)
        response.encoding = 'utf-8'
        soup = BeautifulSoup(response.text, 'html.parser')
        
        title = soup.title.text.strip() if soup.title else "—"
        text = soup.get_text()
        
        phones = re.findall(r'\+7\s?\(?\d{3}\)?\s?\d{3}[-\s]?\d{2}[-\s]?\d{2}', text)
        phone = phones[0] if phones else "—"
        
        # Описание (первые 1000 символов)
        description = clean_text(text, 1000)
        
        cursor.execute("SELECT id FROM sites WHERE url = %s", (url,))
        if cursor.fetchone() is None:
            cursor.execute("""
                INSERT INTO sites (url, title, phone, description) 
                VALUES (%s, %s, %s, %s)
            """, (url, title, phone, description))
            conn.commit()
            print(f"  ✓ Заголовок: {title[:50]}...")
            print(f"  ✓ Телефон: {phone}")
            print(f"  ✓ Описание: {description[:100]}...")
        else:
            print(f"  ⊙ Уже есть в базе")
            
    except Exception as e:
        print(f"  ✗ Ошибка: {e}")

cursor.close()
conn.close()
print("✅ Парсинг завершён!")