39 lines
1.2 KiB
Python
39 lines
1.2 KiB
Python
import requests
|
|
from bs4 import BeautifulSoup
|
|
from urllib.parse import urljoin, urlparse
|
|
import pandas as pd
|
|
|
|
# Starting URL
|
|
base_url = 'https://zainezq.com/'
|
|
visited = set()
|
|
to_visit = [base_url]
|
|
found_urls = []
|
|
|
|
def is_internal(link):
|
|
return urlparse(link).netloc == urlparse(base_url).netloc or urlparse(link).netloc == ''
|
|
|
|
while to_visit:
|
|
url = to_visit.pop(0)
|
|
if url in visited:
|
|
continue
|
|
try:
|
|
print(f'Crawling: {url}')
|
|
response = requests.get(url, timeout=10)
|
|
visited.add(url)
|
|
if response.status_code == 200:
|
|
soup = BeautifulSoup(response.text, 'html.parser')
|
|
found_urls.append(url)
|
|
for a_tag in soup.find_all('a', href=True):
|
|
href = a_tag['href']
|
|
full_url = urljoin(url, href.split('#')[0])
|
|
if is_internal(full_url) and full_url not in visited and full_url not in to_visit:
|
|
to_visit.append(full_url)
|
|
except Exception as e:
|
|
print(f'Error fetching {url}: {e}')
|
|
|
|
# Export to Excel
|
|
df = pd.DataFrame(found_urls, columns=["URL"])
|
|
df.to_excel("website_pages.xlsx", index=False)
|
|
|
|
print("✅ Sitemap saved to website_pages.xlsx")
|