initial commit

legacy/crawler.py

@@ -0,0 +1,38 @@
+import requests
+from bs4 import BeautifulSoup
+from urllib.parse import urljoin, urlparse
+import pandas as pd
+
+# Starting URL
+base_url = 'https://zainezq.com/'
+visited = set()
+to_visit = [base_url]
+found_urls = []
+
+def is_internal(link):
+    return urlparse(link).netloc == urlparse(base_url).netloc or urlparse(link).netloc == ''
+
+while to_visit:
+    url = to_visit.pop(0)
+    if url in visited:
+        continue
+    try:
+        print(f'Crawling: {url}')
+        response = requests.get(url, timeout=10)
+        visited.add(url)
+        if response.status_code == 200:
+            soup = BeautifulSoup(response.text, 'html.parser')
+            found_urls.append(url)
+            for a_tag in soup.find_all('a', href=True):
+                href = a_tag['href']
+                full_url = urljoin(url, href.split('#')[0])
+                if is_internal(full_url) and full_url not in visited and full_url not in to_visit:
+                    to_visit.append(full_url)
+    except Exception as e:
+        print(f'Error fetching {url}: {e}')
+
+# Export to Excel
+df = pd.DataFrame(found_urls, columns=["URL"])
+df.to_excel("website_pages.xlsx", index=False)
+
+print("✅ Sitemap saved to website_pages.xlsx")