feat: Add Bookys scraper alongside EBoek with startup mode chooser
Adds a second scraper for bookys-ebooks.com that harvests direct 1fichier download URLs into a .txt file. The existing EBoek scraper core is unchanged; only the GUI gained a Mode menu entry to switch between the two. Bookys specifics: - Visible (never headless) browser, since the site is behind Cloudflare. The challenge is auto-detected: the scraper polls for real listing items and continues on its own once it clears, no button needed. - Persistent Chrome profile in ~/.eboek_scraper/bookys_chrome_profile so the Cloudflare clearance cookie survives between runs. Removed the hardcoded user-agent override, which claimed Chrome/120 against a real Chrome/150 and invalidated that cookie on every launch. - Pop-up ads are avoided by never clicking host links: hrefs are read and navigated to directly, with window.open neutered via CDP as a backstop. - Host links are read with textContent rather than Selenium's .text, which returns empty for these elements because they sit in a collapsed container. - Only 1fichier hosts are followed; other hosts on a page are ignored. Build configs list the new modules as hidden imports in all four places. gui_main imports the Bookys window lazily, so PyInstaller's static analysis would otherwise miss it and the .exe would crash on switching modes. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01NVJc2XuTvqBHSuqotetseS
This commit is contained in:
104
core/bookys_thread.py
Normal file
104
core/bookys_thread.py
Normal file
@@ -0,0 +1,104 @@
|
||||
"""
|
||||
QThread wrapper for BookysScraper, translating callback events into Qt signals.
|
||||
"""
|
||||
|
||||
from PyQt5.QtCore import QThread, pyqtSignal
|
||||
from .bookys_scraper import BookysScraper
|
||||
|
||||
|
||||
class BookysThread(QThread):
|
||||
"""Runs BookysScraper off the GUI thread and emits progress signals."""
|
||||
|
||||
# High-level lifecycle
|
||||
scraping_started = pyqtSignal(dict)
|
||||
scraping_completed = pyqtSignal(dict)
|
||||
error_occurred = pyqtSignal(str)
|
||||
|
||||
# Cloudflare handshake
|
||||
cloudflare_waiting = pyqtSignal(str) # message
|
||||
cloudflare_passed = pyqtSignal(int) # item_count
|
||||
cloudflare_timeout = pyqtSignal(int) # timeout seconds
|
||||
|
||||
# Progress
|
||||
page_started = pyqtSignal(int, int, int, str) # page_number, index, total, url
|
||||
page_items_found = pyqtSignal(int, int) # page_number, item_count
|
||||
page_completed = pyqtSignal(int, int) # page_number, items_processed
|
||||
book_started = pyqtSignal(int, int, int, str) # page_number, book_index, total, url
|
||||
book_completed = pyqtSignal(str, int) # title, links_found
|
||||
link_found = pyqtSignal(str, int) # url, running_total
|
||||
|
||||
# Catch-all textual status for the log
|
||||
status_update = pyqtSignal(str)
|
||||
|
||||
def __init__(self, category="bd", start_page=1, end_page=1,
|
||||
output_file=None, timing_config=None):
|
||||
super().__init__()
|
||||
self.category = category
|
||||
self.start_page = start_page
|
||||
self.end_page = end_page
|
||||
self.output_file = output_file
|
||||
self.timing_config = timing_config
|
||||
self.scraper = None
|
||||
self._is_running = False
|
||||
|
||||
def run(self):
|
||||
try:
|
||||
self._is_running = True
|
||||
self.scraper = BookysScraper(
|
||||
progress_callback=self._handle_progress,
|
||||
category=self.category,
|
||||
output_file=self.output_file,
|
||||
timing_config=self.timing_config,
|
||||
)
|
||||
summary = self.scraper.scrape(self.start_page, self.end_page)
|
||||
self.scraping_completed.emit(summary)
|
||||
except Exception as e:
|
||||
self.error_occurred.emit(f"Unexpected error: {e}")
|
||||
finally:
|
||||
if self.scraper:
|
||||
self.scraper.close()
|
||||
self._is_running = False
|
||||
|
||||
def _handle_progress(self, event_type, data):
|
||||
try:
|
||||
if event_type == "scraping_started":
|
||||
self.scraping_started.emit(data)
|
||||
elif event_type == "scraping_completed":
|
||||
# Emitted from run() as well; skip here to avoid a double signal.
|
||||
pass
|
||||
elif event_type in ("cloudflare_waiting", "cloudflare_check"):
|
||||
self.cloudflare_waiting.emit(data.get("message", "Waiting for page..."))
|
||||
elif event_type == "cloudflare_passed":
|
||||
self.cloudflare_passed.emit(data.get("item_count", 0))
|
||||
elif event_type == "cloudflare_timeout":
|
||||
self.cloudflare_timeout.emit(data.get("timeout", 0))
|
||||
elif event_type == "page_started":
|
||||
self.page_started.emit(
|
||||
data.get("page_number", 1), data.get("page_index", 1),
|
||||
data.get("total_pages", 1), data.get("url", ""))
|
||||
elif event_type == "page_items_found":
|
||||
self.page_items_found.emit(data.get("page_number", 1),
|
||||
data.get("item_count", 0))
|
||||
elif event_type == "page_completed":
|
||||
self.page_completed.emit(data.get("page_number", 1),
|
||||
data.get("items_processed", 0))
|
||||
elif event_type == "book_started":
|
||||
self.book_started.emit(
|
||||
data.get("page_number", 1), data.get("book_index", 1),
|
||||
data.get("total_books", 1), data.get("url", ""))
|
||||
elif event_type == "book_completed":
|
||||
self.book_completed.emit(data.get("title", ""),
|
||||
data.get("links_found", 0))
|
||||
elif event_type == "link_found":
|
||||
self.link_found.emit(data.get("url", ""), data.get("total", 0))
|
||||
else:
|
||||
self.status_update.emit(f"{event_type}: {data}")
|
||||
except Exception as e:
|
||||
self.error_occurred.emit(f"Signal emission error: {e}")
|
||||
|
||||
def request_stop(self):
|
||||
if self.scraper:
|
||||
self.scraper.request_stop()
|
||||
|
||||
def is_running(self):
|
||||
return self._is_running and self.isRunning()
|
||||
Reference in New Issue
Block a user