c6a69c6ada
Adds a second scraper for bookys-ebooks.com that harvests direct 1fichier download URLs into a .txt file. The existing EBoek scraper core is unchanged; only the GUI gained a Mode menu entry to switch between the two. Bookys specifics: - Visible (never headless) browser, since the site is behind Cloudflare. The challenge is auto-detected: the scraper polls for real listing items and continues on its own once it clears, no button needed. - Persistent Chrome profile in ~/.eboek_scraper/bookys_chrome_profile so the Cloudflare clearance cookie survives between runs. Removed the hardcoded user-agent override, which claimed Chrome/120 against a real Chrome/150 and invalidated that cookie on every launch. - Pop-up ads are avoided by never clicking host links: hrefs are read and navigated to directly, with window.open neutered via CDP as a backstop. - Host links are read with textContent rather than Selenium's .text, which returns empty for these elements because they sit in a collapsed container. - Only 1fichier hosts are followed; other hosts on a page are ignored. Build configs list the new modules as hidden imports in all four places. gui_main imports the Bookys window lazily, so PyInstaller's static analysis would otherwise miss it and the .exe would crash on switching modes. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01NVJc2XuTvqBHSuqotetseS
105 lines
4.4 KiB
Python
105 lines
4.4 KiB
Python
"""
|
|
QThread wrapper for BookysScraper, translating callback events into Qt signals.
|
|
"""
|
|
|
|
from PyQt5.QtCore import QThread, pyqtSignal
|
|
from .bookys_scraper import BookysScraper
|
|
|
|
|
|
class BookysThread(QThread):
|
|
"""Runs BookysScraper off the GUI thread and emits progress signals."""
|
|
|
|
# High-level lifecycle
|
|
scraping_started = pyqtSignal(dict)
|
|
scraping_completed = pyqtSignal(dict)
|
|
error_occurred = pyqtSignal(str)
|
|
|
|
# Cloudflare handshake
|
|
cloudflare_waiting = pyqtSignal(str) # message
|
|
cloudflare_passed = pyqtSignal(int) # item_count
|
|
cloudflare_timeout = pyqtSignal(int) # timeout seconds
|
|
|
|
# Progress
|
|
page_started = pyqtSignal(int, int, int, str) # page_number, index, total, url
|
|
page_items_found = pyqtSignal(int, int) # page_number, item_count
|
|
page_completed = pyqtSignal(int, int) # page_number, items_processed
|
|
book_started = pyqtSignal(int, int, int, str) # page_number, book_index, total, url
|
|
book_completed = pyqtSignal(str, int) # title, links_found
|
|
link_found = pyqtSignal(str, int) # url, running_total
|
|
|
|
# Catch-all textual status for the log
|
|
status_update = pyqtSignal(str)
|
|
|
|
def __init__(self, category="bd", start_page=1, end_page=1,
|
|
output_file=None, timing_config=None):
|
|
super().__init__()
|
|
self.category = category
|
|
self.start_page = start_page
|
|
self.end_page = end_page
|
|
self.output_file = output_file
|
|
self.timing_config = timing_config
|
|
self.scraper = None
|
|
self._is_running = False
|
|
|
|
def run(self):
|
|
try:
|
|
self._is_running = True
|
|
self.scraper = BookysScraper(
|
|
progress_callback=self._handle_progress,
|
|
category=self.category,
|
|
output_file=self.output_file,
|
|
timing_config=self.timing_config,
|
|
)
|
|
summary = self.scraper.scrape(self.start_page, self.end_page)
|
|
self.scraping_completed.emit(summary)
|
|
except Exception as e:
|
|
self.error_occurred.emit(f"Unexpected error: {e}")
|
|
finally:
|
|
if self.scraper:
|
|
self.scraper.close()
|
|
self._is_running = False
|
|
|
|
def _handle_progress(self, event_type, data):
|
|
try:
|
|
if event_type == "scraping_started":
|
|
self.scraping_started.emit(data)
|
|
elif event_type == "scraping_completed":
|
|
# Emitted from run() as well; skip here to avoid a double signal.
|
|
pass
|
|
elif event_type in ("cloudflare_waiting", "cloudflare_check"):
|
|
self.cloudflare_waiting.emit(data.get("message", "Waiting for page..."))
|
|
elif event_type == "cloudflare_passed":
|
|
self.cloudflare_passed.emit(data.get("item_count", 0))
|
|
elif event_type == "cloudflare_timeout":
|
|
self.cloudflare_timeout.emit(data.get("timeout", 0))
|
|
elif event_type == "page_started":
|
|
self.page_started.emit(
|
|
data.get("page_number", 1), data.get("page_index", 1),
|
|
data.get("total_pages", 1), data.get("url", ""))
|
|
elif event_type == "page_items_found":
|
|
self.page_items_found.emit(data.get("page_number", 1),
|
|
data.get("item_count", 0))
|
|
elif event_type == "page_completed":
|
|
self.page_completed.emit(data.get("page_number", 1),
|
|
data.get("items_processed", 0))
|
|
elif event_type == "book_started":
|
|
self.book_started.emit(
|
|
data.get("page_number", 1), data.get("book_index", 1),
|
|
data.get("total_books", 1), data.get("url", ""))
|
|
elif event_type == "book_completed":
|
|
self.book_completed.emit(data.get("title", ""),
|
|
data.get("links_found", 0))
|
|
elif event_type == "link_found":
|
|
self.link_found.emit(data.get("url", ""), data.get("total", 0))
|
|
else:
|
|
self.status_update.emit(f"{event_type}: {data}")
|
|
except Exception as e:
|
|
self.error_occurred.emit(f"Signal emission error: {e}")
|
|
|
|
def request_stop(self):
|
|
if self.scraper:
|
|
self.scraper.request_stop()
|
|
|
|
def is_running(self):
|
|
return self._is_running and self.isRunning()
|