diff --git a/EBoek_Scraper.spec b/EBoek_Scraper.spec index 0240cad..4c7ea45 100644 --- a/EBoek_Scraper.spec +++ b/EBoek_Scraper.spec @@ -6,7 +6,7 @@ a = Analysis( pathex=[], binaries=[], datas=[], - hiddenimports=['PyQt5.QtCore', 'PyQt5.QtGui', 'PyQt5.QtWidgets', 'selenium', 'selenium.webdriver', 'selenium.webdriver.chrome', 'core.scraper', 'core.scraper_thread', 'core.credentials', 'gui.main_window', 'gui.login_dialog', 'gui.progress_dialog', 'utils.validators'], + hiddenimports=['PyQt5.QtCore', 'PyQt5.QtGui', 'PyQt5.QtWidgets', 'selenium', 'selenium.webdriver', 'selenium.webdriver.chrome', 'core.scraper', 'core.scraper_thread', 'core.credentials', 'gui.main_window', 'gui.login_dialog', 'gui.progress_dialog', 'utils.validators', 'core.bookys_scraper', 'core.bookys_thread', 'gui.bookys_window', 'gui.startup_dialog'], hookspath=[], hooksconfig={}, runtime_hooks=[], diff --git a/core/bookys_scraper.py b/core/bookys_scraper.py new file mode 100644 index 0000000..adf1631 --- /dev/null +++ b/core/bookys_scraper.py @@ -0,0 +1,451 @@ +""" +Bookys-ebooks.com scraper. + +Unlike the EBoek scraper (which downloads files), this scraper harvests the +direct 1fichier.com download URLs for each item in a category and writes them, +one per line, to a plain text file. + +Key differences from the EBoek scraper: + * Never runs headless - the site is behind Cloudflare and the user must pass + the "I am not a robot" challenge manually. The scraper auto-detects when the + challenge clears and continues on its own. + * Never clicks host/download links. The site spawns pop-up ads on click, so we + read the href and navigate to it directly (driver.get). Pop-ups are also + neutralised by overriding window.open on every document. + * Only follows 1fichier hosts (per requirements), ignoring other file hosts. +""" + +from selenium import webdriver +from selenium.webdriver.common.by import By +from selenium.webdriver.chrome.options import Options +import time +import random +from pathlib import Path + +import urllib3 +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + + +# Category slug -> listing path on the site. Only BD is enabled for now. +CATEGORIES = { + "bd": { + "name": "Bandes dessinées (BD)", + "path": "bandes-dessinees/bd", + }, +} + +BASE_DOMAIN = "https://www7.bookys-ebooks.com" + +# Persistent Chrome profile for the Bookys scraper. Keeping this between runs is +# what stops Cloudflare from re-challenging on every launch. +DEFAULT_PROFILE_DIR = Path.home() / ".eboek_scraper" / "bookys_chrome_profile" + + +class BookysScraper: + """Harvests 1fichier download URLs from bookys-ebooks.com category listings.""" + + # Selectors, kept together so a future site change is easy to patch. + ITEM_SELECTOR = ".bys-items-container a.bys-item" + HOST_LINK_SELECTOR = "a.bys-link.bys-host" + + def __init__(self, progress_callback=None, category="bd", output_file=None, + timing_config=None, profile_dir=None): + """ + Args: + progress_callback (callable): callback(event_type: str, data: dict) + category (str): category slug (see CATEGORIES) + output_file (str|Path): path to the .txt file to write URLs into + timing_config (dict): optional timing overrides + profile_dir (str|Path): persistent Chrome profile directory. Keeping a + profile between runs preserves the Cloudflare clearance cookie, so + the challenge doesn't reappear on every launch. + """ + self.progress_callback = progress_callback + self._stop_requested = False + self.category = category if category in CATEGORIES else "bd" + self.output_file = Path(output_file) if output_file else ( + Path.home() / "Downloads" / "bookys_1fichier_links.txt" + ) + # Dedicated profile - deliberately NOT the user's everyday Chrome profile, + # which Chrome refuses to open while a normal Chrome window is running. + self.profile_dir = Path(profile_dir) if profile_dir else DEFAULT_PROFILE_DIR + + self.timing = timing_config or {} + self._setup_timing_defaults() + + # Collected URLs (also written to disk as we go) and a dedupe set. + self.collected_urls = [] + self._seen_urls = set() + + # Popup-neutralising script injected into every new document. + self._popup_kill_js = "window.open = function(){ return null; };" + + chrome_options = Options() + # NOTE: never headless - Cloudflare needs a real, visible browser. + + # Persistent profile: this is what keeps the Cloudflare clearance cookie + # alive between runs, so the challenge isn't shown on every single launch. + try: + self.profile_dir.mkdir(parents=True, exist_ok=True) + except Exception: + pass + chrome_options.add_argument(f'--user-data-dir={self.profile_dir}') + chrome_options.add_argument('--profile-directory=Default') + + chrome_options.add_argument('--ignore-ssl-errors') + chrome_options.add_argument('--ignore-certificate-errors') + chrome_options.add_argument('--disable-blink-features=AutomationControlled') + chrome_options.add_experimental_option("excludeSwitches", + ["enable-automation", "enable-logging"]) + chrome_options.add_experimental_option('useAutomationExtension', False) + chrome_options.add_argument('--log-level=3') + # No user-agent override on purpose: a spoofed UA that disagrees with the + # real Chrome build invalidates the clearance cookie and re-triggers the + # challenge. Letting Chrome send its genuine UA is both safer and stealthier. + + try: + self.driver = webdriver.Chrome(options=chrome_options) + except Exception as e: + # Most common cause: the profile is already locked by another run. + self._emit("scraper_init_failed", { + "error": str(e), + "profile_dir": str(self.profile_dir), + "hint": "If a previous scraper Chrome window is still open, close it " + "and try again (the profile can only be used by one at a time).", + }) + raise + self.driver.execute_script( + "Object.defineProperty(navigator, 'webdriver', {get: () => undefined})" + ) + # Kill window.open before any page script runs, on every navigation. + try: + self.driver.execute_cdp_cmd( + 'Page.addScriptToEvaluateOnNewDocument', + {'source': self._popup_kill_js} + ) + except Exception: + # Non-Chromium drivers won't support CDP; per-page fallback still runs. + pass + + self._emit("scraper_initialized", {"category": self.category, + "output_file": str(self.output_file)}) + + # ------------------------------------------------------------------ helpers + + def _emit(self, event_type, data): + if self.progress_callback: + try: + self.progress_callback(event_type, data) + except Exception: + pass + + def request_stop(self): + """Ask the scraper to stop gracefully at the next checkpoint.""" + self._stop_requested = True + self._emit("stop_requested", {}) + + def _setup_timing_defaults(self): + defaults = { + 'action_delay_min': 0.8, + 'action_delay_max': 2.5, + 'page_break_min': 3, + 'page_break_max': 8, + } + for key, value in defaults.items(): + self.timing.setdefault(key, value) + + def _delay(self, min_sec=None, max_sec=None): + if self._stop_requested: + return + if min_sec is None: + min_sec = self.timing['action_delay_min'] + if max_sec is None: + max_sec = self.timing['action_delay_max'] + time.sleep(random.uniform(min_sec, max_sec)) + + @staticmethod + def _link_text(element): + """ + Read an element's text via textContent rather than Selenium's .text. + + The host links on Bookys live in a collapsed container, so they report as + not-displayed and .text returns an empty string for all of them. + textContent reads the DOM directly and works regardless of visibility. + """ + try: + return (element.get_attribute("textContent") or "").strip() + except Exception: + return "" + + def _kill_popups(self): + """Override window.open on the current document and close any stray tabs.""" + try: + self.driver.execute_script(self._popup_kill_js) + except Exception: + pass + self._close_extra_tabs() + + def _close_extra_tabs(self): + """Close any tab that isn't the main one (defensive - ads can slip a tab in).""" + try: + handles = self.driver.window_handles + if len(handles) <= 1: + return + main = handles[0] + for handle in handles[1:]: + try: + self.driver.switch_to.window(handle) + self.driver.close() + except Exception: + pass + self.driver.switch_to.window(main) + except Exception: + pass + + def _navigate(self, url): + """Navigate directly to a URL and neutralise pop-ups afterwards.""" + if self._stop_requested: + return False + self.driver.get(url) + self._kill_popups() + return True + + def _listing_url(self, page_num): + path = CATEGORIES[self.category]["path"] + base = f"{BASE_DOMAIN}/{path}" + if page_num <= 1: + return base + return f"{base}?page={page_num}" + + # --------------------------------------------------------------- cloudflare + + def wait_for_content(self, timeout=600, poll_interval=2): + """ + Wait until the category listing renders real items, giving the user time + to clear the Cloudflare challenge. Auto-detects success - no button. + + Returns True once items are found, False on timeout/stop. + """ + self._emit("cloudflare_check", {"message": "Waiting for Cloudflare / page to load..."}) + deadline = time.time() + timeout + announced = False + while time.time() < deadline: + if self._stop_requested: + return False + try: + items = self.driver.find_elements(By.CSS_SELECTOR, self.ITEM_SELECTOR) + if items: + self._emit("cloudflare_passed", {"item_count": len(items)}) + return True + except Exception: + pass + + if not announced: + # Only nudge the user once, when content isn't immediately present. + self._emit("cloudflare_waiting", { + "message": "If a Cloudflare check is shown, please tick the " + "checkbox in the browser. Scraping continues automatically." + }) + announced = True + time.sleep(poll_interval) + + self._emit("cloudflare_timeout", {"timeout": timeout}) + return False + + # -------------------------------------------------------------------- write + + def _record_url(self, url, context): + if not url or "1fichier" not in url: + return False + if url in self._seen_urls: + self._emit("link_duplicate", {"url": url, **context}) + return False + self._seen_urls.add(url) + self.collected_urls.append(url) + try: + self.output_file.parent.mkdir(parents=True, exist_ok=True) + with open(self.output_file, "a", encoding="utf-8") as f: + f.write(url + "\n") + except Exception as e: + self._emit("write_error", {"error": str(e), "url": url}) + self._emit("link_found", {"url": url, "total": len(self.collected_urls), **context}) + return True + + # ------------------------------------------------------------------- detail + + def _process_book(self, book_url, page_num, book_index, total_books): + """Open a book detail page and harvest its 1fichier link(s).""" + self._emit("book_started", { + "url": book_url, "page_number": page_num, + "book_index": book_index, "total_books": total_books, + }) + if not self._navigate(book_url): + return 0 + + self._delay() + title = "" + try: + title = self.driver.find_element(By.CSS_SELECTOR, "h1").text.strip() + except Exception: + pass + + # Collect intermediate /dl/ hrefs for 1fichier hosts only. + # A page typically lists several hosts (1fichier, DailyUploads, Filefox...); + # everything except 1fichier is deliberately ignored. + dl_hrefs = [] + hosts_seen = [] + try: + host_links = self.driver.find_elements(By.CSS_SELECTOR, self.HOST_LINK_SELECTOR) + if not host_links: + # Fallback if the site changes its classes: any /dl/ link. + host_links = self.driver.find_elements(By.CSS_SELECTOR, "a[href*='/dl/']") + + for link in host_links: + try: + text = self._link_text(link) + if text: + hosts_seen.append(text) + if "1fichier" in text.lower(): + href = link.get_attribute("href") + if href: + dl_hrefs.append(href) + except Exception: + continue + except Exception as e: + self._emit("book_error", {"url": book_url, "error": str(e)}) + return 0 + + self._emit("hosts_listed", {"url": book_url, "hosts": hosts_seen, + "fichier_count": len(dl_hrefs)}) + + found = 0 + for dl_href in dl_hrefs: + if self._stop_requested: + break + # The /dl/ page carries the direct 1fichier URL as an anchor href. + if not self._navigate(dl_href): + continue + self._delay() + final_url = None + try: + # Primary: any anchor already pointing at 1fichier. This is the + # most direct signal and doesn't depend on the button's wording. + for a in self.driver.find_elements(By.CSS_SELECTOR, "a[href*='1fichier']"): + href = a.get_attribute("href") + if href: + final_url = href + break + + # Fallback: locate the call-to-action by its text (textContent, + # not .text - the element may report as not displayed). + if not final_url: + for a in self.driver.find_elements(By.TAG_NAME, "a"): + try: + text = self._link_text(a).lower() + if "cliquez ici" in text or "chargement" in text: + href = a.get_attribute("href") + if href and "1fichier" in href: + final_url = href + break + except Exception: + continue + except Exception as e: + self._emit("book_error", {"url": dl_href, "error": str(e)}) + + if final_url and self._record_url(final_url, {"title": title, "book_url": book_url}): + found += 1 + + self._emit("book_completed", { + "url": book_url, "title": title, "links_found": found, + "page_number": page_num, "book_index": book_index, + }) + return found + + # -------------------------------------------------------------------- scrape + + def scrape(self, start_page=1, end_page=1): + """Walk pages start_page..end_page and harvest 1fichier URLs.""" + if self._stop_requested: + return {"success": False, "reason": "Cancelled before starting"} + + total_pages = end_page - start_page + 1 + self._emit("scraping_started", { + "start_page": start_page, "end_page": end_page, + "total_pages": total_pages, + "category": CATEGORIES[self.category]["name"], + "output_file": str(self.output_file), + }) + + errors = [] + pages_done = 0 + + for page_num in range(start_page, end_page + 1): + if self._stop_requested: + break + + page_url = self._listing_url(page_num) + self._emit("page_started", { + "page_number": page_num, + "page_index": page_num - start_page + 1, + "total_pages": total_pages, + "url": page_url, + }) + + if not self._navigate(page_url): + continue + + # First page load may show Cloudflare; wait it out (auto-detect). + if not self.wait_for_content(): + errors.append(f"Timed out waiting for content on page {page_num}") + self._emit("page_error", {"page_number": page_num, + "error": "content did not load"}) + break + + try: + book_urls = [a.get_attribute("href") for a + in self.driver.find_elements(By.CSS_SELECTOR, self.ITEM_SELECTOR)] + book_urls = [u for u in book_urls if u] + except Exception as e: + errors.append(f"Page {page_num}: {e}") + self._emit("page_error", {"page_number": page_num, "error": str(e)}) + continue + + self._emit("page_items_found", {"page_number": page_num, + "item_count": len(book_urls)}) + + for i, book_url in enumerate(book_urls, 1): + if self._stop_requested: + break + try: + self._process_book(book_url, page_num, i, len(book_urls)) + except Exception as e: + errors.append(f"Book {book_url}: {e}") + self._emit("book_error", {"url": book_url, "error": str(e)}) + self._delay() + + pages_done += 1 + self._emit("page_completed", {"page_number": page_num, + "items_processed": len(book_urls)}) + + # Short break between pages. + if page_num < end_page and not self._stop_requested: + time.sleep(random.uniform(self.timing['page_break_min'], + self.timing['page_break_max'])) + + summary = { + "success": not self._stop_requested and not errors, + "cancelled": self._stop_requested, + "total_pages_processed": pages_done, + "total_links_found": len(self.collected_urls), + "output_file": str(self.output_file), + "errors": errors, + } + self._emit("scraping_completed", summary) + return summary + + def close(self): + try: + self.driver.quit() + self._emit("scraper_closed", {}) + except Exception as e: + self._emit("scraper_close_error", {"error": str(e)}) diff --git a/core/bookys_thread.py b/core/bookys_thread.py new file mode 100644 index 0000000..7056886 --- /dev/null +++ b/core/bookys_thread.py @@ -0,0 +1,104 @@ +""" +QThread wrapper for BookysScraper, translating callback events into Qt signals. +""" + +from PyQt5.QtCore import QThread, pyqtSignal +from .bookys_scraper import BookysScraper + + +class BookysThread(QThread): + """Runs BookysScraper off the GUI thread and emits progress signals.""" + + # High-level lifecycle + scraping_started = pyqtSignal(dict) + scraping_completed = pyqtSignal(dict) + error_occurred = pyqtSignal(str) + + # Cloudflare handshake + cloudflare_waiting = pyqtSignal(str) # message + cloudflare_passed = pyqtSignal(int) # item_count + cloudflare_timeout = pyqtSignal(int) # timeout seconds + + # Progress + page_started = pyqtSignal(int, int, int, str) # page_number, index, total, url + page_items_found = pyqtSignal(int, int) # page_number, item_count + page_completed = pyqtSignal(int, int) # page_number, items_processed + book_started = pyqtSignal(int, int, int, str) # page_number, book_index, total, url + book_completed = pyqtSignal(str, int) # title, links_found + link_found = pyqtSignal(str, int) # url, running_total + + # Catch-all textual status for the log + status_update = pyqtSignal(str) + + def __init__(self, category="bd", start_page=1, end_page=1, + output_file=None, timing_config=None): + super().__init__() + self.category = category + self.start_page = start_page + self.end_page = end_page + self.output_file = output_file + self.timing_config = timing_config + self.scraper = None + self._is_running = False + + def run(self): + try: + self._is_running = True + self.scraper = BookysScraper( + progress_callback=self._handle_progress, + category=self.category, + output_file=self.output_file, + timing_config=self.timing_config, + ) + summary = self.scraper.scrape(self.start_page, self.end_page) + self.scraping_completed.emit(summary) + except Exception as e: + self.error_occurred.emit(f"Unexpected error: {e}") + finally: + if self.scraper: + self.scraper.close() + self._is_running = False + + def _handle_progress(self, event_type, data): + try: + if event_type == "scraping_started": + self.scraping_started.emit(data) + elif event_type == "scraping_completed": + # Emitted from run() as well; skip here to avoid a double signal. + pass + elif event_type in ("cloudflare_waiting", "cloudflare_check"): + self.cloudflare_waiting.emit(data.get("message", "Waiting for page...")) + elif event_type == "cloudflare_passed": + self.cloudflare_passed.emit(data.get("item_count", 0)) + elif event_type == "cloudflare_timeout": + self.cloudflare_timeout.emit(data.get("timeout", 0)) + elif event_type == "page_started": + self.page_started.emit( + data.get("page_number", 1), data.get("page_index", 1), + data.get("total_pages", 1), data.get("url", "")) + elif event_type == "page_items_found": + self.page_items_found.emit(data.get("page_number", 1), + data.get("item_count", 0)) + elif event_type == "page_completed": + self.page_completed.emit(data.get("page_number", 1), + data.get("items_processed", 0)) + elif event_type == "book_started": + self.book_started.emit( + data.get("page_number", 1), data.get("book_index", 1), + data.get("total_books", 1), data.get("url", "")) + elif event_type == "book_completed": + self.book_completed.emit(data.get("title", ""), + data.get("links_found", 0)) + elif event_type == "link_found": + self.link_found.emit(data.get("url", ""), data.get("total", 0)) + else: + self.status_update.emit(f"{event_type}: {data}") + except Exception as e: + self.error_occurred.emit(f"Signal emission error: {e}") + + def request_stop(self): + if self.scraper: + self.scraper.request_stop() + + def is_running(self): + return self._is_running and self.isRunning() diff --git a/gui/bookys_window.py b/gui/bookys_window.py new file mode 100644 index 0000000..e16f5ac --- /dev/null +++ b/gui/bookys_window.py @@ -0,0 +1,405 @@ +""" +Main window for the Bookys (bookys-ebooks.com) scraper. + +Harvests direct 1fichier download URLs for a category and writes them to a .txt +file. Kept entirely separate from the EBoek window so the EBoek flow is untouched. +""" + +import os +import sys +import subprocess +from pathlib import Path + +from PyQt5.QtWidgets import ( + QMainWindow, QWidget, QVBoxLayout, QHBoxLayout, QGridLayout, + QPushButton, QLabel, QSpinBox, QTextEdit, QGroupBox, QComboBox, + QLineEdit, QProgressBar, QMessageBox, QFileDialog, QApplication, QAction +) +from PyQt5.QtCore import Qt + +project_root = Path(__file__).parent.parent +sys.path.insert(0, str(project_root)) + +from core.credentials import CredentialManager +from core.bookys_thread import BookysThread +from core.bookys_scraper import CATEGORIES, DEFAULT_PROFILE_DIR +from utils.validators import validate_page_range, format_error_message + + +class BookysWindow(QMainWindow): + """Interface for scraping 1fichier links from bookys-ebooks.com.""" + + def __init__(self): + super().__init__() + self.credential_manager = CredentialManager() + self.scraper_thread = None + + # Bookys settings live under their own key in the shared config. + all_settings = self.credential_manager.load_app_settings() or {} + self.settings = all_settings.get('bookys', {}) + + self.init_ui() + self.apply_light_theme() + + # ------------------------------------------------------------------- set-up + + def init_ui(self): + self.setWindowTitle("Bookys Scraper — 1fichier link harvester") + self.setMinimumSize(760, 620) + self.resize(900, 720) + + self.create_menu_bar() + + central = QWidget() + self.setCentralWidget(central) + layout = QVBoxLayout(central) + layout.setSpacing(16) + layout.setContentsMargins(20, 20, 20, 20) + + self.create_config_section(layout) + self.create_output_section(layout) + self.create_control_section(layout) + self.create_log_section(layout) + + self.statusBar().showMessage("Ready") + + def apply_light_theme(self): + self.setStyleSheet("QMainWindow { background-color: #ffffff; }") + + def create_menu_bar(self): + menubar = self.menuBar() + + file_menu = menubar.addMenu('File') + open_out = QAction('Open Output Folder', self) + open_out.triggered.connect(self.open_output_folder) + file_menu.addAction(open_out) + file_menu.addSeparator() + exit_action = QAction('Exit', self) + exit_action.triggered.connect(self.close) + file_menu.addAction(exit_action) + + mode_menu = menubar.addMenu('Mode') + to_eboek = QAction('Switch to EBoek', self) + to_eboek.triggered.connect(self._switch_to_eboek) + mode_menu.addAction(to_eboek) + + browser_menu = menubar.addMenu('Browser') + reset_profile = QAction('Reset Browser Profile', self) + reset_profile.triggered.connect(self.reset_browser_profile) + browser_menu.addAction(reset_profile) + + help_menu = menubar.addMenu('Help') + about_action = QAction('About', self) + about_action.triggered.connect(self.show_about) + help_menu.addAction(about_action) + + def create_config_section(self, parent_layout): + group = QGroupBox("Scraping Configuration") + grid = QGridLayout(group) + + grid.addWidget(QLabel("Category:"), 0, 0) + self.category_combo = QComboBox() + self._category_keys = list(CATEGORIES.keys()) + for key in self._category_keys: + self.category_combo.addItem(CATEGORIES[key]["name"], key) + saved_cat = self.settings.get('category', 'bd') + if saved_cat in self._category_keys: + self.category_combo.setCurrentIndex(self._category_keys.index(saved_cat)) + grid.addWidget(self.category_combo, 0, 1, 1, 3) + + grid.addWidget(QLabel("Start Page:"), 1, 0) + self.start_page_spin = QSpinBox() + self.start_page_spin.setRange(1, 99999) + self.start_page_spin.setValue(self.settings.get('start_page', 1)) + grid.addWidget(self.start_page_spin, 1, 1) + + grid.addWidget(QLabel("End Page:"), 1, 2) + self.end_page_spin = QSpinBox() + self.end_page_spin.setRange(1, 99999) + self.end_page_spin.setValue(self.settings.get('end_page', 1)) + grid.addWidget(self.end_page_spin, 1, 3) + + hint = QLabel("💡 The browser opens visibly. If Cloudflare appears, tick " + "its checkbox — scraping resumes automatically.") + hint.setWordWrap(True) + hint.setStyleSheet("color: #666;") + grid.addWidget(hint, 2, 0, 1, 4) + + parent_layout.addWidget(group) + + def create_output_section(self, parent_layout): + group = QGroupBox("Output File (.txt)") + layout = QHBoxLayout(group) + + default_out = self.settings.get( + 'output_file', str(Path.home() / "Downloads" / "bookys_1fichier_links.txt")) + self.output_edit = QLineEdit(default_out) + layout.addWidget(self.output_edit) + + browse_btn = QPushButton("Browse…") + browse_btn.clicked.connect(self.browse_output) + layout.addWidget(browse_btn) + + parent_layout.addWidget(group) + + def create_control_section(self, parent_layout): + group = QGroupBox("Status & Controls") + layout = QVBoxLayout(group) + + row = QHBoxLayout() + info = QVBoxLayout() + self.status_label = QLabel("Ready to start.") + self.status_label.setStyleSheet("font-weight: bold; color: #2E8B57;") + info.addWidget(self.status_label) + + self.links_label = QLabel("Links found: 0") + info.addWidget(self.links_label) + + self.progress_bar = QProgressBar() + self.progress_bar.setVisible(False) + info.addWidget(self.progress_bar) + + row.addLayout(info) + row.addStretch() + + buttons = QVBoxLayout() + self.start_btn = QPushButton("Start Scraping") + self.start_btn.clicked.connect(self.start_scraping) + buttons.addWidget(self.start_btn) + + self.stop_btn = QPushButton("Stop") + self.stop_btn.clicked.connect(self.stop_scraping) + self.stop_btn.setEnabled(False) + buttons.addWidget(self.stop_btn) + + row.addLayout(buttons) + layout.addLayout(row) + parent_layout.addWidget(group) + + def create_log_section(self, parent_layout): + group = QGroupBox("Activity Log") + layout = QVBoxLayout(group) + self.log_view = QTextEdit() + self.log_view.setReadOnly(True) + layout.addWidget(self.log_view) + parent_layout.addWidget(group) + + # ------------------------------------------------------------------ actions + + def log(self, message): + self.log_view.append(message) + + def browse_output(self): + path, _ = QFileDialog.getSaveFileName( + self, "Choose output file", self.output_edit.text(), + "Text files (*.txt);;All files (*.*)") + if path: + self.output_edit.setText(path) + + def _current_category(self): + return self.category_combo.currentData() or 'bd' + + def save_settings(self): + all_settings = self.credential_manager.load_app_settings() or {} + all_settings['bookys'] = { + 'category': self._current_category(), + 'start_page': self.start_page_spin.value(), + 'end_page': self.end_page_spin.value(), + 'output_file': self.output_edit.text(), + } + self.credential_manager.save_app_settings(all_settings) + self.settings = all_settings['bookys'] + + def start_scraping(self): + start_page = self.start_page_spin.value() + end_page = self.end_page_spin.value() + + validation = validate_page_range(start_page, end_page) + if not validation['valid']: + QMessageBox.warning(self, "Invalid Page Range", + format_error_message(validation['errors'])) + return + + output_file = self.output_edit.text().strip() + if not output_file: + QMessageBox.warning(self, "No Output File", + "Please choose where to save the links (.txt).") + return + + self.save_settings() + self.log_view.clear() + self.log(f"Starting Bookys scrape — {CATEGORIES[self._current_category()]['name']}, " + f"pages {start_page}–{end_page}") + self.log(f"Writing links to: {output_file}") + + self.scraper_thread = BookysThread( + category=self._current_category(), + start_page=start_page, + end_page=end_page, + output_file=output_file, + ) + self._connect_signals() + self.scraper_thread.start() + + self.start_btn.setEnabled(False) + self.stop_btn.setEnabled(True) + self.progress_bar.setVisible(True) + self.progress_bar.setRange(0, 0) # indeterminate until first page completes + self.status_label.setText("Opening browser…") + self.status_label.setStyleSheet("font-weight: bold; color: #FF8C00;") + + def stop_scraping(self): + if self.scraper_thread and self.scraper_thread.is_running(): + self.scraper_thread.request_stop() + self.log("Stop requested — finishing current item…") + self.stop_btn.setEnabled(False) + + def _connect_signals(self): + t = self.scraper_thread + t.cloudflare_waiting.connect(self.on_cloudflare_waiting) + t.cloudflare_passed.connect(self.on_cloudflare_passed) + t.cloudflare_timeout.connect(self.on_cloudflare_timeout) + t.page_started.connect(self.on_page_started) + t.page_items_found.connect(self.on_page_items_found) + t.page_completed.connect(self.on_page_completed) + t.book_started.connect(self.on_book_started) + t.book_completed.connect(self.on_book_completed) + t.link_found.connect(self.on_link_found) + t.scraping_completed.connect(self.on_scraping_completed) + t.error_occurred.connect(lambda m: self.log(f"ERROR: {m}")) + + # ------------------------------------------------------------------- signals + + def on_cloudflare_waiting(self, message): + self.status_label.setText("Waiting for Cloudflare…") + self.status_label.setStyleSheet("font-weight: bold; color: #FF8C00;") + self.log(f"⏳ {message}") + + def on_cloudflare_passed(self, item_count): + self.log(f"✅ Page loaded ({item_count} items).") + self.status_label.setText("Scraping…") + + def on_cloudflare_timeout(self, timeout): + self.log(f"⚠️ Timed out after {timeout}s waiting for the page/Cloudflare.") + + def on_page_started(self, page_number, page_index, total_pages, url): + self.log(f"— Page {page_number} ({page_index}/{total_pages})") + if total_pages > 0: + self.progress_bar.setRange(0, total_pages) + self.progress_bar.setValue(page_index - 1) + + def on_page_items_found(self, page_number, item_count): + self.log(f" Found {item_count} items on page {page_number}.") + + def on_page_completed(self, page_number, items_processed): + self.log(f" Page {page_number} done ({items_processed} items).") + self.progress_bar.setValue(self.progress_bar.value() + 1) + + def on_book_started(self, page_number, book_index, total_books, url): + self.status_label.setText(f"Page {page_number}: item {book_index}/{total_books}") + + def on_book_completed(self, title, links_found): + label = title if title else "(untitled)" + self.log(f" • {label} — {links_found} 1fichier link(s)") + + def on_link_found(self, url, total): + self.links_label.setText(f"Links found: {total}") + + def on_scraping_completed(self, summary): + self.start_btn.setEnabled(True) + self.stop_btn.setEnabled(False) + self.progress_bar.setRange(0, 1) + self.progress_bar.setValue(1) + + total = summary.get('total_links_found', 0) + if summary.get('cancelled'): + self.status_label.setText("Cancelled") + self.status_label.setStyleSheet("font-weight: bold; color: #FF6B35;") + elif summary.get('success'): + self.status_label.setText("Completed") + self.status_label.setStyleSheet("font-weight: bold; color: #2E8B57;") + else: + self.status_label.setText("Completed with errors") + self.status_label.setStyleSheet("font-weight: bold; color: #f44336;") + + self.log(f"Done. {total} link(s) written to {summary.get('output_file', '')}") + errors = summary.get('errors') or [] + if errors: + self.log(f"{len(errors)} error(s) occurred:") + for e in errors[:10]: + self.log(f" ! {e}") + + # --------------------------------------------------------------------- misc + + def open_output_folder(self): + folder = Path(self.output_edit.text()).parent + try: + if sys.platform == "win32": + os.startfile(folder) + elif sys.platform == "darwin": + subprocess.run(["open", str(folder)]) + else: + subprocess.run(["xdg-open", str(folder)]) + except Exception as e: + QMessageBox.information(self, "Output Folder", + f"Links are saved to:\n{folder}\n\n" + f"Could not open folder automatically: {e}") + + def reset_browser_profile(self): + """Delete the persistent Chrome profile (forces a fresh Cloudflare pass).""" + if self.scraper_thread and self.scraper_thread.is_running(): + QMessageBox.warning(self, "Scraping in Progress", + "Stop the current scrape before resetting the profile.") + return + + reply = QMessageBox.question( + self, "Reset Browser Profile", + f"Delete the saved browser profile?\n\n{DEFAULT_PROFILE_DIR}\n\n" + "You'll need to pass the Cloudflare check once more on the next run. " + "Use this if the browser stops loading pages correctly.", + QMessageBox.Yes | QMessageBox.No) + + if reply != QMessageBox.Yes: + return + + import shutil + try: + if DEFAULT_PROFILE_DIR.exists(): + shutil.rmtree(DEFAULT_PROFILE_DIR) + self.log("Browser profile reset.") + QMessageBox.information(self, "Profile Reset", + "The browser profile has been cleared.") + except Exception as e: + QMessageBox.warning(self, "Reset Failed", f"Could not reset profile:\n{e}") + + def show_about(self): + QMessageBox.about(self, "About Bookys Scraper", + "Bookys Scraper\n\n" + "Harvests direct 1fichier download URLs from " + "bookys-ebooks.com and saves them to a .txt file.\n\n" + "• Visible browser (Cloudflare-friendly)\n" + "• Auto-detects when the challenge clears\n" + "• Ignores pop-up ads\n\n" + "Built with Python and PyQt5.") + + def _switch_to_eboek(self): + app = QApplication.instance() + if hasattr(app, 'show_eboek'): + app.show_eboek() + + def closeEvent(self, event): + if self.scraper_thread and self.scraper_thread.is_running(): + reply = QMessageBox.question( + self, "Scraping in Progress", + "Scraping is running. Stop and continue?", + QMessageBox.Yes | QMessageBox.No) + if reply == QMessageBox.Yes: + self.scraper_thread.request_stop() + self.scraper_thread.wait(3000) + event.accept() + else: + event.ignore() + return + else: + self.save_settings() + event.accept() diff --git a/gui/main_window.py b/gui/main_window.py index efc86d2..ae66235 100644 --- a/gui/main_window.py +++ b/gui/main_window.py @@ -124,6 +124,12 @@ class MainWindow(QMainWindow): exit_action.triggered.connect(self.close) file_menu.addAction(exit_action) + # Mode menu - switch between the EBoek and Bookys scrapers + mode_menu = menubar.addMenu('Mode') + to_bookys_action = QAction('Switch to Bookys', self) + to_bookys_action.triggered.connect(self._switch_to_bookys) + mode_menu.addAction(to_bookys_action) + # Settings menu settings_menu = menubar.addMenu('Settings') @@ -138,6 +144,13 @@ class MainWindow(QMainWindow): about_action.triggered.connect(self.show_about) help_menu.addAction(about_action) + def _switch_to_bookys(self): + """Switch to the Bookys scraper window.""" + from PyQt5.QtWidgets import QApplication + app = QApplication.instance() + if hasattr(app, 'show_bookys'): + app.show_bookys() + def create_credential_section(self, parent_layout): """Create the credential management section.""" group = QGroupBox("Account Credentials") diff --git a/gui/startup_dialog.py b/gui/startup_dialog.py new file mode 100644 index 0000000..5aaa41b --- /dev/null +++ b/gui/startup_dialog.py @@ -0,0 +1,58 @@ +""" +Startup chooser: pick which site to scrape when the app boots. +""" + +from PyQt5.QtWidgets import ( + QDialog, QVBoxLayout, QHBoxLayout, QPushButton, QLabel +) +from PyQt5.QtCore import Qt + + +class StartupDialog(QDialog): + """Small modal shown on boot to choose EBoek or Bookys.""" + + EBOEK = "eboek" + BOOKYS = "bookys" + + def __init__(self, parent=None): + super().__init__(parent) + self.choice = None + self._init_ui() + + def _init_ui(self): + self.setWindowTitle("Choose Scraper") + self.setModal(True) + self.setMinimumWidth(420) + + layout = QVBoxLayout(self) + layout.setSpacing(18) + layout.setContentsMargins(28, 28, 28, 28) + + title = QLabel("Which site do you want to scrape?") + title.setStyleSheet("font-size: 16px; font-weight: bold;") + title.setAlignment(Qt.AlignCenter) + layout.addWidget(title) + + subtitle = QLabel("You can switch anytime from the menu bar.") + subtitle.setAlignment(Qt.AlignCenter) + subtitle.setStyleSheet("color: #666;") + layout.addWidget(subtitle) + + buttons = QHBoxLayout() + buttons.setSpacing(14) + + eboek_btn = QPushButton("EBoek.info\n\nDownload comic files") + eboek_btn.setMinimumHeight(90) + eboek_btn.clicked.connect(lambda: self._choose(self.EBOEK)) + buttons.addWidget(eboek_btn) + + bookys_btn = QPushButton("Bookys\n\nHarvest 1fichier links → .txt") + bookys_btn.setMinimumHeight(90) + bookys_btn.clicked.connect(lambda: self._choose(self.BOOKYS)) + buttons.addWidget(bookys_btn) + + layout.addLayout(buttons) + + def _choose(self, choice): + self.choice = choice + self.accept() diff --git a/gui_main.py b/gui_main.py index 2ff8930..f7b75c9 100644 --- a/gui_main.py +++ b/gui_main.py @@ -59,7 +59,8 @@ class EBoekScraperApp(QApplication): # Handle exceptions sys.excepthook = self.handle_exception - self.main_window = None + self.main_window = None # EBoek window + self.bookys_window = None # Bookys window def set_application_icon(self): """Set the application icon if available.""" @@ -101,9 +102,16 @@ class EBoekScraperApp(QApplication): # Check system requirements self.check_requirements() - # Create and show main window - self.main_window = MainWindow() - self.main_window.show() + # Ask which site to scrape, then show the matching window. + from gui.startup_dialog import StartupDialog + chooser = StartupDialog() + chooser.exec_() + + if chooser.choice == StartupDialog.BOOKYS: + self.show_bookys() + else: + # Default to EBoek (also covers the dialog being closed). + self.show_eboek() return True @@ -124,6 +132,27 @@ class EBoekScraperApp(QApplication): return False + def show_eboek(self): + """Show the EBoek window, hiding the Bookys one if present.""" + if self.main_window is None: + self.main_window = MainWindow() + if self.bookys_window is not None: + self.bookys_window.hide() + self.main_window.show() + self.main_window.raise_() + self.main_window.activateWindow() + + def show_bookys(self): + """Show the Bookys window, hiding the EBoek one if present.""" + from gui.bookys_window import BookysWindow + if self.bookys_window is None: + self.bookys_window = BookysWindow() + if self.main_window is not None: + self.main_window.hide() + self.bookys_window.show() + self.bookys_window.raise_() + self.bookys_window.activateWindow() + def check_requirements(self): """Check system requirements and dependencies.""" errors = [] diff --git a/scripts/build_exe.bat b/scripts/build_exe.bat index 7f057b0..99cf3a7 100644 --- a/scripts/build_exe.bat +++ b/scripts/build_exe.bat @@ -43,6 +43,10 @@ python -m pyinstaller --onefile --windowed --name "EBoek_Scraper" ^ --hidden-import "gui.login_dialog" ^ --hidden-import "gui.progress_dialog" ^ --hidden-import "utils.validators" ^ + --hidden-import "core.bookys_scraper" ^ + --hidden-import "core.bookys_thread" ^ + --hidden-import "gui.bookys_window" ^ + --hidden-import "gui.startup_dialog" ^ --exclude-module "tkinter" ^ --exclude-module "matplotlib" ^ ..\gui_main.py diff --git a/scripts/build_executable.py b/scripts/build_executable.py index b49dd35..cc8e871 100644 --- a/scripts/build_executable.py +++ b/scripts/build_executable.py @@ -92,6 +92,11 @@ def main(): "--hidden-import", "gui.login_dialog", "--hidden-import", "gui.progress_dialog", "--hidden-import", "utils.validators", + # Bookys scraper modules (imported lazily, so PyInstaller can't see them) + "--hidden-import", "core.bookys_scraper", + "--hidden-import", "core.bookys_thread", + "--hidden-import", "gui.bookys_window", + "--hidden-import", "gui.startup_dialog", # Exclude unnecessary modules to reduce size "--exclude-module", "tkinter", "--exclude-module", "matplotlib", diff --git a/scripts/eboek_scraper.spec b/scripts/eboek_scraper.spec index 602f38d..05d2bd0 100644 --- a/scripts/eboek_scraper.spec +++ b/scripts/eboek_scraper.spec @@ -6,7 +6,7 @@ a = Analysis( pathex=[], binaries=[], datas=[], - hiddenimports=['PyQt5.QtCore', 'PyQt5.QtGui', 'PyQt5.QtWidgets', 'selenium', 'selenium.webdriver', 'selenium.webdriver.chrome', 'selenium.webdriver.chrome.options', 'selenium.webdriver.common.by', 'core.scraper', 'core.scraper_thread', 'core.credentials', 'gui.main_window', 'gui.login_dialog', 'gui.progress_dialog', 'utils.validators'], + hiddenimports=['PyQt5.QtCore', 'PyQt5.QtGui', 'PyQt5.QtWidgets', 'selenium', 'selenium.webdriver', 'selenium.webdriver.chrome', 'selenium.webdriver.chrome.options', 'selenium.webdriver.common.by', 'core.scraper', 'core.scraper_thread', 'core.credentials', 'gui.main_window', 'gui.login_dialog', 'gui.progress_dialog', 'utils.validators', 'core.bookys_scraper', 'core.bookys_thread', 'gui.bookys_window', 'gui.startup_dialog'], hookspath=[], hooksconfig={}, runtime_hooks=[],