import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin, urlparse
import pandas as pd

# Starting URL
base_url = 'https://zainezq.com/'
visited = set()
to_visit = [base_url]
found_urls = []

def is_internal(link):
    return urlparse(link).netloc == urlparse(base_url).netloc or urlparse(link).netloc == ''

while to_visit:
    url = to_visit.pop(0)
    if url in visited:
        continue
    try:
        print(f'Crawling: {url}')
        response = requests.get(url, timeout=10)
        visited.add(url)
        if response.status_code == 200:
            soup = BeautifulSoup(response.text, 'html.parser')
            found_urls.append(url)
            for a_tag in soup.find_all('a', href=True):
                href = a_tag['href']
                full_url = urljoin(url, href.split('#')[0])
                if is_internal(full_url) and full_url not in visited and full_url not in to_visit:
                    to_visit.append(full_url)
    except Exception as e:
        print(f'Error fetching {url}: {e}')

# Export to Excel
df = pd.DataFrame(found_urls, columns=["URL"])
df.to_excel("website_pages.xlsx", index=False)

print("✅ Sitemap saved to website_pages.xlsx")