6 Commits

Author SHA1 Message Date
Louis Mylle
37a6e2ad09 feat: Add Comics and Manga categories to Bookys scraper
Both share the BD page structure (20 items/page, ?page=N, .bys-item links),
so this is just two new CATEGORIES entries; the UI dropdown and URL builder
pick them up automatically. Note the site uses singular "comic" in the path.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01NVJc2XuTvqBHSuqotetseS
2026-07-28 22:27:36 +02:00
louis
9f548309f8 feat: Add simplified build script for creating executables
Added build.py to simplify the process of building standalone executables
with PyInstaller. Also added .gitignore to exclude build artifacts.

Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
2026-07-19 21:24:05 +02:00
Louis Mylle
251dd77e49 chore: Commit remaining local settings and compiled bytecode
Includes the Python bytecode caches (already tracked in this repo) and the
local Claude Code permission allowlist.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01NVJc2XuTvqBHSuqotetseS
2026-07-19 21:05:54 +02:00
Louis Mylle
c6a69c6ada feat: Add Bookys scraper alongside EBoek with startup mode chooser
Adds a second scraper for bookys-ebooks.com that harvests direct 1fichier
download URLs into a .txt file. The existing EBoek scraper core is unchanged;
only the GUI gained a Mode menu entry to switch between the two.

Bookys specifics:
- Visible (never headless) browser, since the site is behind Cloudflare.
  The challenge is auto-detected: the scraper polls for real listing items
  and continues on its own once it clears, no button needed.
- Persistent Chrome profile in ~/.eboek_scraper/bookys_chrome_profile so the
  Cloudflare clearance cookie survives between runs. Removed the hardcoded
  user-agent override, which claimed Chrome/120 against a real Chrome/150 and
  invalidated that cookie on every launch.
- Pop-up ads are avoided by never clicking host links: hrefs are read and
  navigated to directly, with window.open neutered via CDP as a backstop.
- Host links are read with textContent rather than Selenium's .text, which
  returns empty for these elements because they sit in a collapsed container.
- Only 1fichier hosts are followed; other hosts on a page are ignored.

Build configs list the new modules as hidden imports in all four places.
gui_main imports the Bookys window lazily, so PyInstaller's static analysis
would otherwise miss it and the .exe would crash on switching modes.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01NVJc2XuTvqBHSuqotetseS
2026-07-19 20:05:49 +02:00
Louis Mylle
4b65cbbe84 Merge branch 'main' of https://git.xbastards.com/LouisMylle/eboek.info-scraper 2026-01-10 17:36:26 +01:00
Louis Mylle
610a20d12d feat: Enhance scraper timing configuration with user-defined settings and UI adjustments 2026-01-10 17:36:09 +01:00
31 changed files with 1685 additions and 74 deletions

BIN
.DS_Store vendored

Binary file not shown.

View File

@@ -5,7 +5,18 @@
"Bash(python:*)", "Bash(python:*)",
"Bash(build_exe.bat)", "Bash(build_exe.bat)",
"Bash(cmd /c \"cd scripts && build_exe.bat\")", "Bash(cmd /c \"cd scripts && build_exe.bat\")",
"Bash(dir:*)" "Bash(dir:*)",
"Bash(python3 -c \"import PyQt5\")",
"Bash(python3 -c \"import selenium; print\\('selenium', selenium.__version__\\)\")",
"Bash(python3 -m pip --version)",
"Bash(curl -s https://pypi.org/pypi/PyQt5/json)",
"Bash(python3 -c ' *)",
"Bash(python3 -m venv .venv)",
"Bash(.venv/bin/pip install *)",
"Bash(.venv/bin/python -c ' *)",
"Bash(.venv/bin/python gui_main.py)",
"Skill(claude-in-chrome)",
"Bash(pkill -f \"gui_main.py\")"
] ]
} }
} }

37
.gitignore vendored Normal file
View File

@@ -0,0 +1,37 @@
# Python
__pycache__/
*.py[cod]
*$py.class
*.so
.Python
*.egg-info/
dist/
build/
*.egg
.pytest_cache/
# PyInstaller
*.spec
# Virtual Environment
venv/
env/
ENV/
# IDE
.vscode/
.idea/
*.swp
*.swo
*~
# OS
.DS_Store
Thumbs.db
# Credentials
credentials.json
*.log
# Claude
.claude/settings.local.json

View File

@@ -6,7 +6,7 @@ a = Analysis(
pathex=[], pathex=[],
binaries=[], binaries=[],
datas=[], datas=[],
hiddenimports=['PyQt5.QtCore', 'PyQt5.QtGui', 'PyQt5.QtWidgets', 'selenium', 'selenium.webdriver', 'selenium.webdriver.chrome', 'core.scraper', 'core.scraper_thread', 'core.credentials', 'gui.main_window', 'gui.login_dialog', 'gui.progress_dialog', 'utils.validators'], hiddenimports=['PyQt5.QtCore', 'PyQt5.QtGui', 'PyQt5.QtWidgets', 'selenium', 'selenium.webdriver', 'selenium.webdriver.chrome', 'core.scraper', 'core.scraper_thread', 'core.credentials', 'gui.main_window', 'gui.login_dialog', 'gui.progress_dialog', 'utils.validators', 'core.bookys_scraper', 'core.bookys_thread', 'gui.bookys_window', 'gui.startup_dialog'],
hookspath=[], hookspath=[],
hooksconfig={}, hooksconfig={},
runtime_hooks=[], runtime_hooks=[],

129
build.py Normal file
View File

@@ -0,0 +1,129 @@
#!/usr/bin/env python3
"""
Simple build wrapper for EBoek.info Scraper
Creates a standalone executable using PyInstaller
"""
import os
import sys
import subprocess
import shutil
from pathlib import Path
def main():
"""Main build process."""
print("=" * 50)
print("EBoek.info Scraper - Executable Builder")
print("=" * 50)
print()
project_root = Path(__file__).parent
os.chdir(project_root)
print(f"Working from: {project_root}")
print()
# Check if PyInstaller is installed
try:
import PyInstaller
print(f"PyInstaller {PyInstaller.__version__} found")
except ImportError:
print("Installing PyInstaller...")
subprocess.run([sys.executable, "-m", "pip", "install", "pyinstaller"], check=True)
# Clean previous builds
print()
for directory in ["dist", "build"]:
if Path(directory).exists():
shutil.rmtree(directory)
print(f"Cleaned {directory}/")
# Determine platform-specific settings
is_windows = sys.platform.startswith('win')
app_name = "EBoek_Scraper"
# Build command
build_cmd = [
sys.executable, "-m", "PyInstaller",
"--onefile",
"--windowed",
"--name", app_name,
# Hidden imports for PyQt5
"--hidden-import", "PyQt5.QtCore",
"--hidden-import", "PyQt5.QtGui",
"--hidden-import", "PyQt5.QtWidgets",
# Hidden imports for Selenium
"--hidden-import", "selenium",
"--hidden-import", "selenium.webdriver",
"--hidden-import", "selenium.webdriver.chrome",
"--hidden-import", "selenium.webdriver.chrome.options",
"--hidden-import", "selenium.webdriver.common.by",
# Hidden imports for our modules
"--hidden-import", "core.scraper",
"--hidden-import", "core.scraper_thread",
"--hidden-import", "core.credentials",
"--hidden-import", "gui.main_window",
"--hidden-import", "gui.login_dialog",
"--hidden-import", "gui.progress_dialog",
"--hidden-import", "utils.validators",
# Bookys scraper modules
"--hidden-import", "core.bookys_scraper",
"--hidden-import", "core.bookys_thread",
"--hidden-import", "gui.bookys_window",
"--hidden-import", "gui.startup_dialog",
# Exclude unnecessary modules
"--exclude-module", "tkinter",
"--exclude-module", "matplotlib",
"--exclude-module", "numpy",
"--exclude-module", "pandas",
# Main script
"gui_main.py"
]
print()
print("Building executable...")
print("This may take a few minutes...")
print()
# Run PyInstaller
result = subprocess.run(build_cmd)
if result.returncode != 0:
print("Build failed!")
sys.exit(1)
print("Build completed successfully!")
# Check if executable was created
exe_name = "EBoek_Scraper.exe" if is_windows else "EBoek_Scraper"
exe_path = Path("dist") / exe_name
if not exe_path.exists():
print(f"Error: Executable not found at {exe_path}")
sys.exit(1)
# Display results
file_size = exe_path.stat().st_size / (1024 * 1024) # MB
print()
print("=" * 50)
print("BUILD SUCCESSFUL!")
print("=" * 50)
print(f"Executable location: {exe_path}")
print(f"File size: {file_size:.1f} MB")
print()
if is_windows:
print("Windows Instructions:")
print(f" Run: {exe_path}")
print(" Windows may show security warning on first run")
print(" Click 'More Info' -> 'Run Anyway' if prompted")
else:
print("macOS/Linux Instructions:")
print(f" Run: ./{exe_path}")
print()
print("Ready for distribution!")
if __name__ == "__main__":
main()

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

461
core/bookys_scraper.py Normal file
View File

@@ -0,0 +1,461 @@
"""
Bookys-ebooks.com scraper.
Unlike the EBoek scraper (which downloads files), this scraper harvests the
direct 1fichier.com download URLs for each item in a category and writes them,
one per line, to a plain text file.
Key differences from the EBoek scraper:
* Never runs headless - the site is behind Cloudflare and the user must pass
the "I am not a robot" challenge manually. The scraper auto-detects when the
challenge clears and continues on its own.
* Never clicks host/download links. The site spawns pop-up ads on click, so we
read the href and navigate to it directly (driver.get). Pop-ups are also
neutralised by overriding window.open on every document.
* Only follows 1fichier hosts (per requirements), ignoring other file hosts.
"""
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.options import Options
import time
import random
from pathlib import Path
import urllib3
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
# Category slug -> listing path on the site. All share the same page structure
# (20 items per page, ?page=N pagination, .bys-item links), so adding one is
# just a new entry here - the rest of the scraper and the UI adapt automatically.
CATEGORIES = {
"bd": {
"name": "Bandes dessinées (BD)",
"path": "bandes-dessinees/bd",
},
"comic": {
"name": "Comics",
"path": "bandes-dessinees/comic", # note: singular "comic" on the site
},
"manga": {
"name": "Manga",
"path": "bandes-dessinees/manga",
},
}
BASE_DOMAIN = "https://www7.bookys-ebooks.com"
# Persistent Chrome profile for the Bookys scraper. Keeping this between runs is
# what stops Cloudflare from re-challenging on every launch.
DEFAULT_PROFILE_DIR = Path.home() / ".eboek_scraper" / "bookys_chrome_profile"
class BookysScraper:
"""Harvests 1fichier download URLs from bookys-ebooks.com category listings."""
# Selectors, kept together so a future site change is easy to patch.
ITEM_SELECTOR = ".bys-items-container a.bys-item"
HOST_LINK_SELECTOR = "a.bys-link.bys-host"
def __init__(self, progress_callback=None, category="bd", output_file=None,
timing_config=None, profile_dir=None):
"""
Args:
progress_callback (callable): callback(event_type: str, data: dict)
category (str): category slug (see CATEGORIES)
output_file (str|Path): path to the .txt file to write URLs into
timing_config (dict): optional timing overrides
profile_dir (str|Path): persistent Chrome profile directory. Keeping a
profile between runs preserves the Cloudflare clearance cookie, so
the challenge doesn't reappear on every launch.
"""
self.progress_callback = progress_callback
self._stop_requested = False
self.category = category if category in CATEGORIES else "bd"
self.output_file = Path(output_file) if output_file else (
Path.home() / "Downloads" / "bookys_1fichier_links.txt"
)
# Dedicated profile - deliberately NOT the user's everyday Chrome profile,
# which Chrome refuses to open while a normal Chrome window is running.
self.profile_dir = Path(profile_dir) if profile_dir else DEFAULT_PROFILE_DIR
self.timing = timing_config or {}
self._setup_timing_defaults()
# Collected URLs (also written to disk as we go) and a dedupe set.
self.collected_urls = []
self._seen_urls = set()
# Popup-neutralising script injected into every new document.
self._popup_kill_js = "window.open = function(){ return null; };"
chrome_options = Options()
# NOTE: never headless - Cloudflare needs a real, visible browser.
# Persistent profile: this is what keeps the Cloudflare clearance cookie
# alive between runs, so the challenge isn't shown on every single launch.
try:
self.profile_dir.mkdir(parents=True, exist_ok=True)
except Exception:
pass
chrome_options.add_argument(f'--user-data-dir={self.profile_dir}')
chrome_options.add_argument('--profile-directory=Default')
chrome_options.add_argument('--ignore-ssl-errors')
chrome_options.add_argument('--ignore-certificate-errors')
chrome_options.add_argument('--disable-blink-features=AutomationControlled')
chrome_options.add_experimental_option("excludeSwitches",
["enable-automation", "enable-logging"])
chrome_options.add_experimental_option('useAutomationExtension', False)
chrome_options.add_argument('--log-level=3')
# No user-agent override on purpose: a spoofed UA that disagrees with the
# real Chrome build invalidates the clearance cookie and re-triggers the
# challenge. Letting Chrome send its genuine UA is both safer and stealthier.
try:
self.driver = webdriver.Chrome(options=chrome_options)
except Exception as e:
# Most common cause: the profile is already locked by another run.
self._emit("scraper_init_failed", {
"error": str(e),
"profile_dir": str(self.profile_dir),
"hint": "If a previous scraper Chrome window is still open, close it "
"and try again (the profile can only be used by one at a time).",
})
raise
self.driver.execute_script(
"Object.defineProperty(navigator, 'webdriver', {get: () => undefined})"
)
# Kill window.open before any page script runs, on every navigation.
try:
self.driver.execute_cdp_cmd(
'Page.addScriptToEvaluateOnNewDocument',
{'source': self._popup_kill_js}
)
except Exception:
# Non-Chromium drivers won't support CDP; per-page fallback still runs.
pass
self._emit("scraper_initialized", {"category": self.category,
"output_file": str(self.output_file)})
# ------------------------------------------------------------------ helpers
def _emit(self, event_type, data):
if self.progress_callback:
try:
self.progress_callback(event_type, data)
except Exception:
pass
def request_stop(self):
"""Ask the scraper to stop gracefully at the next checkpoint."""
self._stop_requested = True
self._emit("stop_requested", {})
def _setup_timing_defaults(self):
defaults = {
'action_delay_min': 0.8,
'action_delay_max': 2.5,
'page_break_min': 3,
'page_break_max': 8,
}
for key, value in defaults.items():
self.timing.setdefault(key, value)
def _delay(self, min_sec=None, max_sec=None):
if self._stop_requested:
return
if min_sec is None:
min_sec = self.timing['action_delay_min']
if max_sec is None:
max_sec = self.timing['action_delay_max']
time.sleep(random.uniform(min_sec, max_sec))
@staticmethod
def _link_text(element):
"""
Read an element's text via textContent rather than Selenium's .text.
The host links on Bookys live in a collapsed container, so they report as
not-displayed and .text returns an empty string for all of them.
textContent reads the DOM directly and works regardless of visibility.
"""
try:
return (element.get_attribute("textContent") or "").strip()
except Exception:
return ""
def _kill_popups(self):
"""Override window.open on the current document and close any stray tabs."""
try:
self.driver.execute_script(self._popup_kill_js)
except Exception:
pass
self._close_extra_tabs()
def _close_extra_tabs(self):
"""Close any tab that isn't the main one (defensive - ads can slip a tab in)."""
try:
handles = self.driver.window_handles
if len(handles) <= 1:
return
main = handles[0]
for handle in handles[1:]:
try:
self.driver.switch_to.window(handle)
self.driver.close()
except Exception:
pass
self.driver.switch_to.window(main)
except Exception:
pass
def _navigate(self, url):
"""Navigate directly to a URL and neutralise pop-ups afterwards."""
if self._stop_requested:
return False
self.driver.get(url)
self._kill_popups()
return True
def _listing_url(self, page_num):
path = CATEGORIES[self.category]["path"]
base = f"{BASE_DOMAIN}/{path}"
if page_num <= 1:
return base
return f"{base}?page={page_num}"
# --------------------------------------------------------------- cloudflare
def wait_for_content(self, timeout=600, poll_interval=2):
"""
Wait until the category listing renders real items, giving the user time
to clear the Cloudflare challenge. Auto-detects success - no button.
Returns True once items are found, False on timeout/stop.
"""
self._emit("cloudflare_check", {"message": "Waiting for Cloudflare / page to load..."})
deadline = time.time() + timeout
announced = False
while time.time() < deadline:
if self._stop_requested:
return False
try:
items = self.driver.find_elements(By.CSS_SELECTOR, self.ITEM_SELECTOR)
if items:
self._emit("cloudflare_passed", {"item_count": len(items)})
return True
except Exception:
pass
if not announced:
# Only nudge the user once, when content isn't immediately present.
self._emit("cloudflare_waiting", {
"message": "If a Cloudflare check is shown, please tick the "
"checkbox in the browser. Scraping continues automatically."
})
announced = True
time.sleep(poll_interval)
self._emit("cloudflare_timeout", {"timeout": timeout})
return False
# -------------------------------------------------------------------- write
def _record_url(self, url, context):
if not url or "1fichier" not in url:
return False
if url in self._seen_urls:
self._emit("link_duplicate", {"url": url, **context})
return False
self._seen_urls.add(url)
self.collected_urls.append(url)
try:
self.output_file.parent.mkdir(parents=True, exist_ok=True)
with open(self.output_file, "a", encoding="utf-8") as f:
f.write(url + "\n")
except Exception as e:
self._emit("write_error", {"error": str(e), "url": url})
self._emit("link_found", {"url": url, "total": len(self.collected_urls), **context})
return True
# ------------------------------------------------------------------- detail
def _process_book(self, book_url, page_num, book_index, total_books):
"""Open a book detail page and harvest its 1fichier link(s)."""
self._emit("book_started", {
"url": book_url, "page_number": page_num,
"book_index": book_index, "total_books": total_books,
})
if not self._navigate(book_url):
return 0
self._delay()
title = ""
try:
title = self.driver.find_element(By.CSS_SELECTOR, "h1").text.strip()
except Exception:
pass
# Collect intermediate /dl/<id> hrefs for 1fichier hosts only.
# A page typically lists several hosts (1fichier, DailyUploads, Filefox...);
# everything except 1fichier is deliberately ignored.
dl_hrefs = []
hosts_seen = []
try:
host_links = self.driver.find_elements(By.CSS_SELECTOR, self.HOST_LINK_SELECTOR)
if not host_links:
# Fallback if the site changes its classes: any /dl/ link.
host_links = self.driver.find_elements(By.CSS_SELECTOR, "a[href*='/dl/']")
for link in host_links:
try:
text = self._link_text(link)
if text:
hosts_seen.append(text)
if "1fichier" in text.lower():
href = link.get_attribute("href")
if href:
dl_hrefs.append(href)
except Exception:
continue
except Exception as e:
self._emit("book_error", {"url": book_url, "error": str(e)})
return 0
self._emit("hosts_listed", {"url": book_url, "hosts": hosts_seen,
"fichier_count": len(dl_hrefs)})
found = 0
for dl_href in dl_hrefs:
if self._stop_requested:
break
# The /dl/ page carries the direct 1fichier URL as an anchor href.
if not self._navigate(dl_href):
continue
self._delay()
final_url = None
try:
# Primary: any anchor already pointing at 1fichier. This is the
# most direct signal and doesn't depend on the button's wording.
for a in self.driver.find_elements(By.CSS_SELECTOR, "a[href*='1fichier']"):
href = a.get_attribute("href")
if href:
final_url = href
break
# Fallback: locate the call-to-action by its text (textContent,
# not .text - the element may report as not displayed).
if not final_url:
for a in self.driver.find_elements(By.TAG_NAME, "a"):
try:
text = self._link_text(a).lower()
if "cliquez ici" in text or "chargement" in text:
href = a.get_attribute("href")
if href and "1fichier" in href:
final_url = href
break
except Exception:
continue
except Exception as e:
self._emit("book_error", {"url": dl_href, "error": str(e)})
if final_url and self._record_url(final_url, {"title": title, "book_url": book_url}):
found += 1
self._emit("book_completed", {
"url": book_url, "title": title, "links_found": found,
"page_number": page_num, "book_index": book_index,
})
return found
# -------------------------------------------------------------------- scrape
def scrape(self, start_page=1, end_page=1):
"""Walk pages start_page..end_page and harvest 1fichier URLs."""
if self._stop_requested:
return {"success": False, "reason": "Cancelled before starting"}
total_pages = end_page - start_page + 1
self._emit("scraping_started", {
"start_page": start_page, "end_page": end_page,
"total_pages": total_pages,
"category": CATEGORIES[self.category]["name"],
"output_file": str(self.output_file),
})
errors = []
pages_done = 0
for page_num in range(start_page, end_page + 1):
if self._stop_requested:
break
page_url = self._listing_url(page_num)
self._emit("page_started", {
"page_number": page_num,
"page_index": page_num - start_page + 1,
"total_pages": total_pages,
"url": page_url,
})
if not self._navigate(page_url):
continue
# First page load may show Cloudflare; wait it out (auto-detect).
if not self.wait_for_content():
errors.append(f"Timed out waiting for content on page {page_num}")
self._emit("page_error", {"page_number": page_num,
"error": "content did not load"})
break
try:
book_urls = [a.get_attribute("href") for a
in self.driver.find_elements(By.CSS_SELECTOR, self.ITEM_SELECTOR)]
book_urls = [u for u in book_urls if u]
except Exception as e:
errors.append(f"Page {page_num}: {e}")
self._emit("page_error", {"page_number": page_num, "error": str(e)})
continue
self._emit("page_items_found", {"page_number": page_num,
"item_count": len(book_urls)})
for i, book_url in enumerate(book_urls, 1):
if self._stop_requested:
break
try:
self._process_book(book_url, page_num, i, len(book_urls))
except Exception as e:
errors.append(f"Book {book_url}: {e}")
self._emit("book_error", {"url": book_url, "error": str(e)})
self._delay()
pages_done += 1
self._emit("page_completed", {"page_number": page_num,
"items_processed": len(book_urls)})
# Short break between pages.
if page_num < end_page and not self._stop_requested:
time.sleep(random.uniform(self.timing['page_break_min'],
self.timing['page_break_max']))
summary = {
"success": not self._stop_requested and not errors,
"cancelled": self._stop_requested,
"total_pages_processed": pages_done,
"total_links_found": len(self.collected_urls),
"output_file": str(self.output_file),
"errors": errors,
}
self._emit("scraping_completed", summary)
return summary
def close(self):
try:
self.driver.quit()
self._emit("scraper_closed", {})
except Exception as e:
self._emit("scraper_close_error", {"error": str(e)})

104
core/bookys_thread.py Normal file
View File

@@ -0,0 +1,104 @@
"""
QThread wrapper for BookysScraper, translating callback events into Qt signals.
"""
from PyQt5.QtCore import QThread, pyqtSignal
from .bookys_scraper import BookysScraper
class BookysThread(QThread):
"""Runs BookysScraper off the GUI thread and emits progress signals."""
# High-level lifecycle
scraping_started = pyqtSignal(dict)
scraping_completed = pyqtSignal(dict)
error_occurred = pyqtSignal(str)
# Cloudflare handshake
cloudflare_waiting = pyqtSignal(str) # message
cloudflare_passed = pyqtSignal(int) # item_count
cloudflare_timeout = pyqtSignal(int) # timeout seconds
# Progress
page_started = pyqtSignal(int, int, int, str) # page_number, index, total, url
page_items_found = pyqtSignal(int, int) # page_number, item_count
page_completed = pyqtSignal(int, int) # page_number, items_processed
book_started = pyqtSignal(int, int, int, str) # page_number, book_index, total, url
book_completed = pyqtSignal(str, int) # title, links_found
link_found = pyqtSignal(str, int) # url, running_total
# Catch-all textual status for the log
status_update = pyqtSignal(str)
def __init__(self, category="bd", start_page=1, end_page=1,
output_file=None, timing_config=None):
super().__init__()
self.category = category
self.start_page = start_page
self.end_page = end_page
self.output_file = output_file
self.timing_config = timing_config
self.scraper = None
self._is_running = False
def run(self):
try:
self._is_running = True
self.scraper = BookysScraper(
progress_callback=self._handle_progress,
category=self.category,
output_file=self.output_file,
timing_config=self.timing_config,
)
summary = self.scraper.scrape(self.start_page, self.end_page)
self.scraping_completed.emit(summary)
except Exception as e:
self.error_occurred.emit(f"Unexpected error: {e}")
finally:
if self.scraper:
self.scraper.close()
self._is_running = False
def _handle_progress(self, event_type, data):
try:
if event_type == "scraping_started":
self.scraping_started.emit(data)
elif event_type == "scraping_completed":
# Emitted from run() as well; skip here to avoid a double signal.
pass
elif event_type in ("cloudflare_waiting", "cloudflare_check"):
self.cloudflare_waiting.emit(data.get("message", "Waiting for page..."))
elif event_type == "cloudflare_passed":
self.cloudflare_passed.emit(data.get("item_count", 0))
elif event_type == "cloudflare_timeout":
self.cloudflare_timeout.emit(data.get("timeout", 0))
elif event_type == "page_started":
self.page_started.emit(
data.get("page_number", 1), data.get("page_index", 1),
data.get("total_pages", 1), data.get("url", ""))
elif event_type == "page_items_found":
self.page_items_found.emit(data.get("page_number", 1),
data.get("item_count", 0))
elif event_type == "page_completed":
self.page_completed.emit(data.get("page_number", 1),
data.get("items_processed", 0))
elif event_type == "book_started":
self.book_started.emit(
data.get("page_number", 1), data.get("book_index", 1),
data.get("total_books", 1), data.get("url", ""))
elif event_type == "book_completed":
self.book_completed.emit(data.get("title", ""),
data.get("links_found", 0))
elif event_type == "link_found":
self.link_found.emit(data.get("url", ""), data.get("total", 0))
else:
self.status_update.emit(f"{event_type}: {data}")
except Exception as e:
self.error_occurred.emit(f"Signal emission error: {e}")
def request_stop(self):
if self.scraper:
self.scraper.request_stop()
def is_running(self):
return self._is_running and self.isRunning()

View File

@@ -257,7 +257,17 @@ class CredentialManager:
'download_path': str(Path.home() / "Downloads"), 'download_path': str(Path.home() / "Downloads"),
'default_start_page': 1, 'default_start_page': 1,
'default_end_page': 1, 'default_end_page': 1,
'scraping_mode': 0 # 0=All Comics, 1=Latest Comics 'scraping_mode': 0, # 0=All Comics, 1=Latest Comics
# Timing configuration defaults
'action_delay_min': 0.5, # Minimum delay between actions (seconds)
'action_delay_max': 2.0, # Maximum delay between actions (seconds)
'page_break_chance': 70, # Percentage chance of taking a break between pages
'page_break_min': 15, # Minimum page break duration (seconds)
'page_break_max': 45, # Maximum page break duration (seconds)
'batch_break_interval': 5, # Take a break every N comics
'batch_break_min': 3, # Minimum batch break duration (seconds)
'batch_break_max': 7, # Maximum batch break duration (seconds)
'typing_delay': 0.1, # Delay between character typing (seconds)
} }
def export_settings(self, export_path): def export_settings(self, export_path):

View File

@@ -25,7 +25,7 @@ class Scraper:
callback mechanisms for progress updates to a GUI application. callback mechanisms for progress updates to a GUI application.
""" """
def __init__(self, headless=False, progress_callback=None, scraping_mode=0): def __init__(self, headless=False, progress_callback=None, scraping_mode=0, timing_config=None):
""" """
Initialize the scraper with optional GUI callback support. Initialize the scraper with optional GUI callback support.
@@ -34,11 +34,16 @@ class Scraper:
progress_callback (callable): Optional callback function for progress updates progress_callback (callable): Optional callback function for progress updates
Callback signature: callback(event_type: str, data: dict) Callback signature: callback(event_type: str, data: dict)
scraping_mode (int): Scraping mode (0=All Comics, 1=Latest Comics) scraping_mode (int): Scraping mode (0=All Comics, 1=Latest Comics)
timing_config (dict): Timing configuration settings
""" """
self.progress_callback = progress_callback self.progress_callback = progress_callback
self._stop_requested = False self._stop_requested = False
self.scraping_mode = scraping_mode self.scraping_mode = scraping_mode
# Set up timing configuration with defaults
self.timing = timing_config or {}
self._setup_timing_defaults()
# Set up Chrome options with anti-detection measures # Set up Chrome options with anti-detection measures
chrome_options = Options() chrome_options = Options()
if headless: if headless:
@@ -103,16 +108,42 @@ class Scraper:
self._stop_requested = True self._stop_requested = True
self._emit_progress("stop_requested", {}) self._emit_progress("stop_requested", {})
def human_delay(self, min_sec=0.5, max_sec=2): def _setup_timing_defaults(self):
"""Set up timing configuration with default values."""
defaults = {
'action_delay_min': 0.5,
'action_delay_max': 2.0,
'page_break_chance': 70,
'page_break_min': 15,
'page_break_max': 45,
'batch_break_interval': 5,
'batch_break_min': 3,
'batch_break_max': 7,
'typing_delay': 0.1,
}
# Fill in any missing values with defaults
for key, default_value in defaults.items():
if key not in self.timing:
self.timing[key] = default_value
def human_delay(self, min_sec=None, max_sec=None):
""" """
Simulate human-like delay with cancellation support. Simulate human-like delay with cancellation support.
Args: Args:
min_sec (float): Minimum delay time min_sec (float): Minimum delay time (uses config default if None)
max_sec (float): Maximum delay time max_sec (float): Maximum delay time (uses config default if None)
""" """
if self._stop_requested: if self._stop_requested:
return return
# Use configured timing if no specific values provided
if min_sec is None:
min_sec = self.timing['action_delay_min']
if max_sec is None:
max_sec = self.timing['action_delay_max']
delay_time = random.uniform(min_sec, max_sec) delay_time = random.uniform(min_sec, max_sec)
self._emit_progress("delay_started", {"duration": delay_time}) self._emit_progress("delay_started", {"duration": delay_time})
time.sleep(delay_time) time.sleep(delay_time)
@@ -129,7 +160,8 @@ class Scraper:
if self._stop_requested: if self._stop_requested:
return return
element.send_keys(char) element.send_keys(char)
time.sleep(random.uniform(0.05, 0.15)) typing_delay = self.timing['typing_delay']
time.sleep(random.uniform(typing_delay * 0.5, typing_delay * 1.5))
def navigate(self, url): def navigate(self, url):
""" """
@@ -330,8 +362,9 @@ class Scraper:
# Take a break between pages (more likely and longer) # Take a break between pages (more likely and longer)
if page_num > start_page: if page_num > start_page:
if random.random() < 0.7: # 70% chance of break break_chance = self.timing['page_break_chance'] / 100.0
break_time = random.uniform(15, 45) # 15-45 seconds if random.random() < break_chance:
break_time = random.uniform(self.timing['page_break_min'], self.timing['page_break_max'])
self._emit_progress("page_break_started", { self._emit_progress("page_break_started", {
"duration": break_time, "duration": break_time,
"page_number": page_num "page_number": page_num
@@ -446,9 +479,10 @@ class Scraper:
"comic_index": i "comic_index": i
}) })
# Take a longer break every 5 comics # Take a longer break every N comics (configurable)
if i % 5 == 0 and i < len(comic_urls): batch_interval = self.timing['batch_break_interval']
break_time = random.uniform(3, 7) if i % batch_interval == 0 and i < len(comic_urls):
break_time = random.uniform(self.timing['batch_break_min'], self.timing['batch_break_max'])
self._emit_progress("comic_batch_break", { self._emit_progress("comic_batch_break", {
"duration": break_time, "duration": break_time,
"comics_processed": i "comics_processed": i

View File

@@ -58,7 +58,7 @@ class ScraperThread(QThread):
comic_batch_break = pyqtSignal(float, int) # duration, comics_processed comic_batch_break = pyqtSignal(float, int) # duration, comics_processed
download_delay = pyqtSignal(float, int) # duration, remaining_downloads download_delay = pyqtSignal(float, int) # duration, remaining_downloads
def __init__(self, username, password, start_page, end_page, scraping_mode=0, headless=True): def __init__(self, username, password, start_page, end_page, scraping_mode=0, headless=True, timing_config=None):
""" """
Initialize the scraper thread. Initialize the scraper thread.
@@ -69,6 +69,7 @@ class ScraperThread(QThread):
end_page (int): Ending page number end_page (int): Ending page number
scraping_mode (int): Scraping mode (0=All Comics, 1=Latest Comics) scraping_mode (int): Scraping mode (0=All Comics, 1=Latest Comics)
headless (bool): Whether to run Chrome in headless mode headless (bool): Whether to run Chrome in headless mode
timing_config (dict): Timing configuration settings
""" """
super().__init__() super().__init__()
self.username = username self.username = username
@@ -77,6 +78,7 @@ class ScraperThread(QThread):
self.end_page = end_page self.end_page = end_page
self.scraping_mode = scraping_mode self.scraping_mode = scraping_mode
self.headless = headless self.headless = headless
self.timing_config = timing_config
self.scraper = None self.scraper = None
self._is_running = False self._is_running = False
@@ -92,7 +94,8 @@ class ScraperThread(QThread):
self.scraper = Scraper( self.scraper = Scraper(
headless=self.headless, headless=self.headless,
progress_callback=self._handle_scraper_progress, progress_callback=self._handle_scraper_progress,
scraping_mode=self.scraping_mode scraping_mode=self.scraping_mode,
timing_config=self.timing_config
) )
# Perform login # Perform login

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

405
gui/bookys_window.py Normal file
View File

@@ -0,0 +1,405 @@
"""
Main window for the Bookys (bookys-ebooks.com) scraper.
Harvests direct 1fichier download URLs for a category and writes them to a .txt
file. Kept entirely separate from the EBoek window so the EBoek flow is untouched.
"""
import os
import sys
import subprocess
from pathlib import Path
from PyQt5.QtWidgets import (
QMainWindow, QWidget, QVBoxLayout, QHBoxLayout, QGridLayout,
QPushButton, QLabel, QSpinBox, QTextEdit, QGroupBox, QComboBox,
QLineEdit, QProgressBar, QMessageBox, QFileDialog, QApplication, QAction
)
from PyQt5.QtCore import Qt
project_root = Path(__file__).parent.parent
sys.path.insert(0, str(project_root))
from core.credentials import CredentialManager
from core.bookys_thread import BookysThread
from core.bookys_scraper import CATEGORIES, DEFAULT_PROFILE_DIR
from utils.validators import validate_page_range, format_error_message
class BookysWindow(QMainWindow):
"""Interface for scraping 1fichier links from bookys-ebooks.com."""
def __init__(self):
super().__init__()
self.credential_manager = CredentialManager()
self.scraper_thread = None
# Bookys settings live under their own key in the shared config.
all_settings = self.credential_manager.load_app_settings() or {}
self.settings = all_settings.get('bookys', {})
self.init_ui()
self.apply_light_theme()
# ------------------------------------------------------------------- set-up
def init_ui(self):
self.setWindowTitle("Bookys Scraper — 1fichier link harvester")
self.setMinimumSize(760, 620)
self.resize(900, 720)
self.create_menu_bar()
central = QWidget()
self.setCentralWidget(central)
layout = QVBoxLayout(central)
layout.setSpacing(16)
layout.setContentsMargins(20, 20, 20, 20)
self.create_config_section(layout)
self.create_output_section(layout)
self.create_control_section(layout)
self.create_log_section(layout)
self.statusBar().showMessage("Ready")
def apply_light_theme(self):
self.setStyleSheet("QMainWindow { background-color: #ffffff; }")
def create_menu_bar(self):
menubar = self.menuBar()
file_menu = menubar.addMenu('File')
open_out = QAction('Open Output Folder', self)
open_out.triggered.connect(self.open_output_folder)
file_menu.addAction(open_out)
file_menu.addSeparator()
exit_action = QAction('Exit', self)
exit_action.triggered.connect(self.close)
file_menu.addAction(exit_action)
mode_menu = menubar.addMenu('Mode')
to_eboek = QAction('Switch to EBoek', self)
to_eboek.triggered.connect(self._switch_to_eboek)
mode_menu.addAction(to_eboek)
browser_menu = menubar.addMenu('Browser')
reset_profile = QAction('Reset Browser Profile', self)
reset_profile.triggered.connect(self.reset_browser_profile)
browser_menu.addAction(reset_profile)
help_menu = menubar.addMenu('Help')
about_action = QAction('About', self)
about_action.triggered.connect(self.show_about)
help_menu.addAction(about_action)
def create_config_section(self, parent_layout):
group = QGroupBox("Scraping Configuration")
grid = QGridLayout(group)
grid.addWidget(QLabel("Category:"), 0, 0)
self.category_combo = QComboBox()
self._category_keys = list(CATEGORIES.keys())
for key in self._category_keys:
self.category_combo.addItem(CATEGORIES[key]["name"], key)
saved_cat = self.settings.get('category', 'bd')
if saved_cat in self._category_keys:
self.category_combo.setCurrentIndex(self._category_keys.index(saved_cat))
grid.addWidget(self.category_combo, 0, 1, 1, 3)
grid.addWidget(QLabel("Start Page:"), 1, 0)
self.start_page_spin = QSpinBox()
self.start_page_spin.setRange(1, 99999)
self.start_page_spin.setValue(self.settings.get('start_page', 1))
grid.addWidget(self.start_page_spin, 1, 1)
grid.addWidget(QLabel("End Page:"), 1, 2)
self.end_page_spin = QSpinBox()
self.end_page_spin.setRange(1, 99999)
self.end_page_spin.setValue(self.settings.get('end_page', 1))
grid.addWidget(self.end_page_spin, 1, 3)
hint = QLabel("💡 The browser opens visibly. If Cloudflare appears, tick "
"its checkbox — scraping resumes automatically.")
hint.setWordWrap(True)
hint.setStyleSheet("color: #666;")
grid.addWidget(hint, 2, 0, 1, 4)
parent_layout.addWidget(group)
def create_output_section(self, parent_layout):
group = QGroupBox("Output File (.txt)")
layout = QHBoxLayout(group)
default_out = self.settings.get(
'output_file', str(Path.home() / "Downloads" / "bookys_1fichier_links.txt"))
self.output_edit = QLineEdit(default_out)
layout.addWidget(self.output_edit)
browse_btn = QPushButton("Browse…")
browse_btn.clicked.connect(self.browse_output)
layout.addWidget(browse_btn)
parent_layout.addWidget(group)
def create_control_section(self, parent_layout):
group = QGroupBox("Status & Controls")
layout = QVBoxLayout(group)
row = QHBoxLayout()
info = QVBoxLayout()
self.status_label = QLabel("Ready to start.")
self.status_label.setStyleSheet("font-weight: bold; color: #2E8B57;")
info.addWidget(self.status_label)
self.links_label = QLabel("Links found: 0")
info.addWidget(self.links_label)
self.progress_bar = QProgressBar()
self.progress_bar.setVisible(False)
info.addWidget(self.progress_bar)
row.addLayout(info)
row.addStretch()
buttons = QVBoxLayout()
self.start_btn = QPushButton("Start Scraping")
self.start_btn.clicked.connect(self.start_scraping)
buttons.addWidget(self.start_btn)
self.stop_btn = QPushButton("Stop")
self.stop_btn.clicked.connect(self.stop_scraping)
self.stop_btn.setEnabled(False)
buttons.addWidget(self.stop_btn)
row.addLayout(buttons)
layout.addLayout(row)
parent_layout.addWidget(group)
def create_log_section(self, parent_layout):
group = QGroupBox("Activity Log")
layout = QVBoxLayout(group)
self.log_view = QTextEdit()
self.log_view.setReadOnly(True)
layout.addWidget(self.log_view)
parent_layout.addWidget(group)
# ------------------------------------------------------------------ actions
def log(self, message):
self.log_view.append(message)
def browse_output(self):
path, _ = QFileDialog.getSaveFileName(
self, "Choose output file", self.output_edit.text(),
"Text files (*.txt);;All files (*.*)")
if path:
self.output_edit.setText(path)
def _current_category(self):
return self.category_combo.currentData() or 'bd'
def save_settings(self):
all_settings = self.credential_manager.load_app_settings() or {}
all_settings['bookys'] = {
'category': self._current_category(),
'start_page': self.start_page_spin.value(),
'end_page': self.end_page_spin.value(),
'output_file': self.output_edit.text(),
}
self.credential_manager.save_app_settings(all_settings)
self.settings = all_settings['bookys']
def start_scraping(self):
start_page = self.start_page_spin.value()
end_page = self.end_page_spin.value()
validation = validate_page_range(start_page, end_page)
if not validation['valid']:
QMessageBox.warning(self, "Invalid Page Range",
format_error_message(validation['errors']))
return
output_file = self.output_edit.text().strip()
if not output_file:
QMessageBox.warning(self, "No Output File",
"Please choose where to save the links (.txt).")
return
self.save_settings()
self.log_view.clear()
self.log(f"Starting Bookys scrape — {CATEGORIES[self._current_category()]['name']}, "
f"pages {start_page}{end_page}")
self.log(f"Writing links to: {output_file}")
self.scraper_thread = BookysThread(
category=self._current_category(),
start_page=start_page,
end_page=end_page,
output_file=output_file,
)
self._connect_signals()
self.scraper_thread.start()
self.start_btn.setEnabled(False)
self.stop_btn.setEnabled(True)
self.progress_bar.setVisible(True)
self.progress_bar.setRange(0, 0) # indeterminate until first page completes
self.status_label.setText("Opening browser…")
self.status_label.setStyleSheet("font-weight: bold; color: #FF8C00;")
def stop_scraping(self):
if self.scraper_thread and self.scraper_thread.is_running():
self.scraper_thread.request_stop()
self.log("Stop requested — finishing current item…")
self.stop_btn.setEnabled(False)
def _connect_signals(self):
t = self.scraper_thread
t.cloudflare_waiting.connect(self.on_cloudflare_waiting)
t.cloudflare_passed.connect(self.on_cloudflare_passed)
t.cloudflare_timeout.connect(self.on_cloudflare_timeout)
t.page_started.connect(self.on_page_started)
t.page_items_found.connect(self.on_page_items_found)
t.page_completed.connect(self.on_page_completed)
t.book_started.connect(self.on_book_started)
t.book_completed.connect(self.on_book_completed)
t.link_found.connect(self.on_link_found)
t.scraping_completed.connect(self.on_scraping_completed)
t.error_occurred.connect(lambda m: self.log(f"ERROR: {m}"))
# ------------------------------------------------------------------- signals
def on_cloudflare_waiting(self, message):
self.status_label.setText("Waiting for Cloudflare…")
self.status_label.setStyleSheet("font-weight: bold; color: #FF8C00;")
self.log(f"{message}")
def on_cloudflare_passed(self, item_count):
self.log(f"✅ Page loaded ({item_count} items).")
self.status_label.setText("Scraping…")
def on_cloudflare_timeout(self, timeout):
self.log(f"⚠️ Timed out after {timeout}s waiting for the page/Cloudflare.")
def on_page_started(self, page_number, page_index, total_pages, url):
self.log(f"— Page {page_number} ({page_index}/{total_pages})")
if total_pages > 0:
self.progress_bar.setRange(0, total_pages)
self.progress_bar.setValue(page_index - 1)
def on_page_items_found(self, page_number, item_count):
self.log(f" Found {item_count} items on page {page_number}.")
def on_page_completed(self, page_number, items_processed):
self.log(f" Page {page_number} done ({items_processed} items).")
self.progress_bar.setValue(self.progress_bar.value() + 1)
def on_book_started(self, page_number, book_index, total_books, url):
self.status_label.setText(f"Page {page_number}: item {book_index}/{total_books}")
def on_book_completed(self, title, links_found):
label = title if title else "(untitled)"
self.log(f"{label}{links_found} 1fichier link(s)")
def on_link_found(self, url, total):
self.links_label.setText(f"Links found: {total}")
def on_scraping_completed(self, summary):
self.start_btn.setEnabled(True)
self.stop_btn.setEnabled(False)
self.progress_bar.setRange(0, 1)
self.progress_bar.setValue(1)
total = summary.get('total_links_found', 0)
if summary.get('cancelled'):
self.status_label.setText("Cancelled")
self.status_label.setStyleSheet("font-weight: bold; color: #FF6B35;")
elif summary.get('success'):
self.status_label.setText("Completed")
self.status_label.setStyleSheet("font-weight: bold; color: #2E8B57;")
else:
self.status_label.setText("Completed with errors")
self.status_label.setStyleSheet("font-weight: bold; color: #f44336;")
self.log(f"Done. {total} link(s) written to {summary.get('output_file', '')}")
errors = summary.get('errors') or []
if errors:
self.log(f"{len(errors)} error(s) occurred:")
for e in errors[:10]:
self.log(f" ! {e}")
# --------------------------------------------------------------------- misc
def open_output_folder(self):
folder = Path(self.output_edit.text()).parent
try:
if sys.platform == "win32":
os.startfile(folder)
elif sys.platform == "darwin":
subprocess.run(["open", str(folder)])
else:
subprocess.run(["xdg-open", str(folder)])
except Exception as e:
QMessageBox.information(self, "Output Folder",
f"Links are saved to:\n{folder}\n\n"
f"Could not open folder automatically: {e}")
def reset_browser_profile(self):
"""Delete the persistent Chrome profile (forces a fresh Cloudflare pass)."""
if self.scraper_thread and self.scraper_thread.is_running():
QMessageBox.warning(self, "Scraping in Progress",
"Stop the current scrape before resetting the profile.")
return
reply = QMessageBox.question(
self, "Reset Browser Profile",
f"Delete the saved browser profile?\n\n{DEFAULT_PROFILE_DIR}\n\n"
"You'll need to pass the Cloudflare check once more on the next run. "
"Use this if the browser stops loading pages correctly.",
QMessageBox.Yes | QMessageBox.No)
if reply != QMessageBox.Yes:
return
import shutil
try:
if DEFAULT_PROFILE_DIR.exists():
shutil.rmtree(DEFAULT_PROFILE_DIR)
self.log("Browser profile reset.")
QMessageBox.information(self, "Profile Reset",
"The browser profile has been cleared.")
except Exception as e:
QMessageBox.warning(self, "Reset Failed", f"Could not reset profile:\n{e}")
def show_about(self):
QMessageBox.about(self, "About Bookys Scraper",
"Bookys Scraper\n\n"
"Harvests direct 1fichier download URLs from "
"bookys-ebooks.com and saves them to a .txt file.\n\n"
"• Visible browser (Cloudflare-friendly)\n"
"• Auto-detects when the challenge clears\n"
"• Ignores pop-up ads\n\n"
"Built with Python and PyQt5.")
def _switch_to_eboek(self):
app = QApplication.instance()
if hasattr(app, 'show_eboek'):
app.show_eboek()
def closeEvent(self, event):
if self.scraper_thread and self.scraper_thread.is_running():
reply = QMessageBox.question(
self, "Scraping in Progress",
"Scraping is running. Stop and continue?",
QMessageBox.Yes | QMessageBox.No)
if reply == QMessageBox.Yes:
self.scraper_thread.request_stop()
self.scraper_thread.wait(3000)
event.accept()
else:
event.ignore()
return
else:
self.save_settings()
event.accept()

View File

@@ -6,9 +6,9 @@ import sys
from pathlib import Path from pathlib import Path
from PyQt5.QtWidgets import ( from PyQt5.QtWidgets import (
QMainWindow, QWidget, QVBoxLayout, QHBoxLayout, QGridLayout, QMainWindow, QWidget, QVBoxLayout, QHBoxLayout, QGridLayout,
QPushButton, QLabel, QSpinBox, QTextEdit, QGroupBox, QPushButton, QLabel, QSpinBox, QDoubleSpinBox, QTextEdit, QGroupBox,
QCheckBox, QProgressBar, QMessageBox, QFileDialog, QMenuBar, QMenu, QAction, QCheckBox, QProgressBar, QMessageBox, QFileDialog, QMenuBar, QMenu, QAction,
QComboBox QComboBox, QFrame, QScrollArea, QSizePolicy
) )
from PyQt5.QtCore import Qt, QTimer, pyqtSignal from PyQt5.QtCore import Qt, QTimer, pyqtSignal
from PyQt5.QtGui import QFont, QIcon from PyQt5.QtGui import QFont, QIcon
@@ -54,33 +54,55 @@ class MainWindow(QMainWindow):
self.app_settings = self.credential_manager.get_default_settings() self.app_settings = self.credential_manager.get_default_settings()
self.init_ui() self.init_ui()
self.apply_light_theme()
self.update_credential_status() self.update_credential_status()
def init_ui(self): def init_ui(self):
"""Initialize the user interface.""" """Initialize the user interface."""
self.setWindowTitle("EBoek.info Scraper") self.setWindowTitle("EBoek.info Scraper")
self.setMinimumSize(600, 500) self.setMinimumSize(900, 650)
self.resize(700, 600) self.resize(1200, 750)
# Create menu bar # Create menu bar
self.create_menu_bar() self.create_menu_bar()
# Create central widget # Create central widget with scroll area for better responsiveness
central_widget = QWidget() scroll_area = QScrollArea()
self.setCentralWidget(central_widget) scroll_widget = QWidget()
scroll_area.setWidget(scroll_widget)
scroll_area.setWidgetResizable(True)
scroll_area.setHorizontalScrollBarPolicy(Qt.ScrollBarAsNeeded)
scroll_area.setVerticalScrollBarPolicy(Qt.ScrollBarAsNeeded)
self.setCentralWidget(scroll_area)
# Main layout # Main layout with better spacing
layout = QVBoxLayout(central_widget) layout = QVBoxLayout(scroll_widget)
layout.setSpacing(20)
layout.setContentsMargins(20, 20, 20, 20)
# Create sections # Create sections in a more organized way
self.create_credential_section(layout) self.create_credential_section(layout)
self.create_scraping_section(layout) self.create_scraping_section(layout)
self.create_timing_section(layout)
self.create_status_section(layout) self.create_status_section(layout)
self.create_control_section(layout) self.create_control_section(layout)
# Add stretch to push content to top
layout.addStretch()
# Status bar # Status bar
self.statusBar().showMessage("Ready") self.statusBar().showMessage("Ready")
def apply_light_theme(self):
"""Apply minimal light theme styling with completely default system controls."""
# Very minimal stylesheet - only style the main containers, no form controls
light_stylesheet = """
QMainWindow {
background-color: #ffffff;
}
"""
self.setStyleSheet(light_stylesheet)
def create_menu_bar(self): def create_menu_bar(self):
"""Create the menu bar.""" """Create the menu bar."""
menubar = self.menuBar() menubar = self.menuBar()
@@ -102,6 +124,12 @@ class MainWindow(QMainWindow):
exit_action.triggered.connect(self.close) exit_action.triggered.connect(self.close)
file_menu.addAction(exit_action) file_menu.addAction(exit_action)
# Mode menu - switch between the EBoek and Bookys scrapers
mode_menu = menubar.addMenu('Mode')
to_bookys_action = QAction('Switch to Bookys', self)
to_bookys_action.triggered.connect(self._switch_to_bookys)
mode_menu.addAction(to_bookys_action)
# Settings menu # Settings menu
settings_menu = menubar.addMenu('Settings') settings_menu = menubar.addMenu('Settings')
@@ -116,16 +144,28 @@ class MainWindow(QMainWindow):
about_action.triggered.connect(self.show_about) about_action.triggered.connect(self.show_about)
help_menu.addAction(about_action) help_menu.addAction(about_action)
def _switch_to_bookys(self):
"""Switch to the Bookys scraper window."""
from PyQt5.QtWidgets import QApplication
app = QApplication.instance()
if hasattr(app, 'show_bookys'):
app.show_bookys()
def create_credential_section(self, parent_layout): def create_credential_section(self, parent_layout):
"""Create the credential management section.""" """Create the credential management section."""
group = QGroupBox("Credentials") group = QGroupBox("Account Credentials")
layout = QHBoxLayout(group) layout = QHBoxLayout(group)
# Status with icon
status_layout = QHBoxLayout()
self.credential_status_label = QLabel("No credentials configured") self.credential_status_label = QLabel("No credentials configured")
layout.addWidget(self.credential_status_label) self.credential_status_label.setStyleSheet("font-size: 13px;")
status_layout.addWidget(self.credential_status_label)
layout.addLayout(status_layout)
layout.addStretch() layout.addStretch()
# Button with default system styling
self.change_credentials_btn = QPushButton("Change Credentials") self.change_credentials_btn = QPushButton("Change Credentials")
self.change_credentials_btn.clicked.connect(self.show_login_dialog) self.change_credentials_btn.clicked.connect(self.show_login_dialog)
layout.addWidget(self.change_credentials_btn) layout.addWidget(self.change_credentials_btn)
@@ -135,51 +175,71 @@ class MainWindow(QMainWindow):
def create_scraping_section(self, parent_layout): def create_scraping_section(self, parent_layout):
"""Create the scraping configuration section.""" """Create the scraping configuration section."""
group = QGroupBox("Scraping Configuration") group = QGroupBox("Scraping Configuration")
layout = QGridLayout(group) main_layout = QVBoxLayout(group)
# Create horizontal layout for better space usage
top_section = QHBoxLayout()
# Left side - Mode selection
mode_group = QGroupBox("Scraping Mode")
mode_layout = QVBoxLayout(mode_group)
# Scraping mode selection
layout.addWidget(QLabel("Mode:"), 0, 0)
self.mode_combo = QComboBox() self.mode_combo = QComboBox()
self.mode_combo.addItems([ self.mode_combo.addItems([
"All Comics (stripverhalen-alle)", "All Comics (Complete Archive)",
"Latest Comics (laatste)" "Latest Comics (Recent Additions)"
]) ])
self.mode_combo.setCurrentIndex(self.app_settings.get('scraping_mode', 0)) self.mode_combo.setCurrentIndex(self.app_settings.get('scraping_mode', 0))
self.mode_combo.setToolTip("Select which page type to scrape")
self.mode_combo.currentIndexChanged.connect(self.on_mode_changed) self.mode_combo.currentIndexChanged.connect(self.on_mode_changed)
layout.addWidget(self.mode_combo, 0, 1, 1, 3) mode_layout.addWidget(self.mode_combo)
# Page range selection # Mode description label
layout.addWidget(QLabel("Start Page:"), 1, 0) self.mode_description_label = QLabel("")
self.mode_description_label.setWordWrap(True)
mode_layout.addWidget(self.mode_description_label)
mode_group.setMaximumWidth(400)
# Right side - Page range and options
config_group = QGroupBox("Configuration")
config_layout = QGridLayout(config_group)
# Page range
config_layout.addWidget(QLabel("Start Page:"), 0, 0)
self.start_page_spin = QSpinBox() self.start_page_spin = QSpinBox()
self.start_page_spin.setMinimum(1) self.start_page_spin.setMinimum(1)
self.start_page_spin.setMaximum(9999) self.start_page_spin.setMaximum(9999)
self.start_page_spin.setValue(self.app_settings.get('default_start_page', 1)) self.start_page_spin.setValue(self.app_settings.get('default_start_page', 1))
layout.addWidget(self.start_page_spin, 1, 1) config_layout.addWidget(self.start_page_spin, 0, 1)
layout.addWidget(QLabel("End Page:"), 1, 2) config_layout.addWidget(QLabel("End Page:"), 0, 2)
self.end_page_spin = QSpinBox() self.end_page_spin = QSpinBox()
self.end_page_spin.setMinimum(1) self.end_page_spin.setMinimum(1)
self.end_page_spin.setMaximum(9999) self.end_page_spin.setMaximum(9999)
self.end_page_spin.setValue(self.app_settings.get('default_end_page', 1)) self.end_page_spin.setValue(self.app_settings.get('default_end_page', 1))
layout.addWidget(self.end_page_spin, 1, 3) config_layout.addWidget(self.end_page_spin, 0, 3)
# Mode description label
self.mode_description_label = QLabel("")
self.mode_description_label.setStyleSheet("color: #666; font-size: 11px; font-style: italic;")
self.mode_description_label.setWordWrap(True)
layout.addWidget(self.mode_description_label, 2, 0, 1, 4)
# Options # Options
self.headless_checkbox = QCheckBox("Headless Mode") self.headless_checkbox = QCheckBox("Headless Mode")
self.headless_checkbox.setChecked(self.app_settings.get('headless_mode', True)) self.headless_checkbox.setChecked(self.app_settings.get('headless_mode', True))
self.headless_checkbox.setToolTip("Run browser in background (recommended)") config_layout.addWidget(self.headless_checkbox, 1, 0, 1, 2)
layout.addWidget(self.headless_checkbox, 3, 0, 1, 2)
self.verbose_checkbox = QCheckBox("Verbose Logging") self.verbose_checkbox = QCheckBox("Verbose Logging")
self.verbose_checkbox.setChecked(self.app_settings.get('verbose_logging', False)) self.verbose_checkbox.setChecked(self.app_settings.get('verbose_logging', False))
self.verbose_checkbox.setToolTip("Show detailed progress information") config_layout.addWidget(self.verbose_checkbox, 1, 2, 1, 2)
layout.addWidget(self.verbose_checkbox, 3, 2, 1, 2)
# Help texts
page_help = QLabel("💡 Start with 1-3 pages to test")
config_layout.addWidget(page_help, 2, 0, 1, 4)
browser_help = QLabel("🖥️ Headless = background mode (recommended) • Verbose = detailed logs")
config_layout.addWidget(browser_help, 3, 0, 1, 4)
# Add to horizontal layout
top_section.addWidget(mode_group)
top_section.addWidget(config_group)
main_layout.addLayout(top_section)
# Update mode description # Update mode description
self.update_mode_description() self.update_mode_description()
@@ -196,49 +256,286 @@ class MainWindow(QMainWindow):
mode_index = self.mode_combo.currentIndex() mode_index = self.mode_combo.currentIndex()
if mode_index == 0: # All Comics if mode_index == 0: # All Comics
description = ("Scrapes all comics from the 'stripverhalen-alle' page. " description = ("🗃️ <b>All Comics (Complete Archive)</b><br>"
"This is the original scraping mode with complete comic archives.") "Scrapes from 'stripverhalen-alle' page containing the full comic archive. "
"Best for systematic collection of all available comics. "
"Large page counts available (1000+ pages).")
elif mode_index == 1: # Latest Comics elif mode_index == 1: # Latest Comics
description = ("Scrapes latest comics from the 'laatste' page. " description = ("🆕 <b>Latest Comics (Recent Additions)</b><br>"
"This mode gets the most recently added comics with page parameter support.") "Scrapes from 'laatste' page with most recently added comics. "
"Perfect for staying up-to-date with new releases. "
"Smaller page counts but fresh content.")
else: else:
description = "" description = ""
self.mode_description_label.setText(description) self.mode_description_label.setText(description)
def create_timing_section(self, parent_layout):
"""Create the timing configuration section."""
group = QGroupBox("Timing Configuration")
main_layout = QVBoxLayout(group)
# Create horizontal layout for better space usage
top_layout = QHBoxLayout()
# Left side - Quick presets
preset_group = QGroupBox("Quick Presets")
preset_layout = QVBoxLayout(preset_group)
preset_info = QLabel("Choose a predefined timing profile:")
preset_layout.addWidget(preset_info)
preset_buttons_layout = QVBoxLayout()
fast_btn = QPushButton("Fast Mode")
fast_btn.clicked.connect(lambda: self.apply_timing_preset("fast"))
preset_buttons_layout.addWidget(fast_btn)
fast_desc = QLabel("Minimal delays • Faster scraping • Higher detection risk")
fast_desc.setWordWrap(True)
preset_buttons_layout.addWidget(fast_desc)
balanced_btn = QPushButton("Balanced Mode (Recommended)")
balanced_btn.clicked.connect(lambda: self.apply_timing_preset("balanced"))
preset_buttons_layout.addWidget(balanced_btn)
balanced_desc = QLabel("Default settings • Good balance • Recommended for most users")
balanced_desc.setWordWrap(True)
preset_buttons_layout.addWidget(balanced_desc)
stealth_btn = QPushButton("Stealth Mode")
stealth_btn.clicked.connect(lambda: self.apply_timing_preset("stealth"))
preset_buttons_layout.addWidget(stealth_btn)
stealth_desc = QLabel("Maximum delays • Very human-like • Slower but undetectable")
stealth_desc.setWordWrap(True)
preset_buttons_layout.addWidget(stealth_desc)
preset_layout.addLayout(preset_buttons_layout)
preset_layout.addStretch()
preset_group.setMinimumWidth(350) # Changed from setMaximumWidth to setMinimumWidth
# Right side - Manual controls in a more compact layout
manual_group = QGroupBox("Manual Configuration")
manual_layout = QGridLayout(manual_group)
# Action delays
manual_layout.addWidget(QLabel("Action Delays (sec):"), 0, 0)
self.action_delay_min_spin = self.create_double_spinbox(0.1, 10.0, 0.1,
self.app_settings.get('action_delay_min', 0.5))
manual_layout.addWidget(self.action_delay_min_spin, 0, 1)
manual_layout.addWidget(QLabel("to"), 0, 2)
self.action_delay_max_spin = self.create_double_spinbox(0.1, 10.0, 0.1,
self.app_settings.get('action_delay_max', 2.0))
manual_layout.addWidget(self.action_delay_max_spin, 0, 3)
# Page breaks
manual_layout.addWidget(QLabel("Page Break Chance:"), 1, 0)
self.page_break_chance_spin = QSpinBox()
self.page_break_chance_spin.setRange(0, 100)
self.page_break_chance_spin.setValue(self.app_settings.get('page_break_chance', 70))
self.page_break_chance_spin.setSuffix("%")
manual_layout.addWidget(self.page_break_chance_spin, 1, 1)
manual_layout.addWidget(QLabel("Duration:"), 1, 2)
page_duration_layout = QHBoxLayout()
self.page_break_min_spin = QSpinBox()
self.page_break_min_spin.setRange(5, 300)
self.page_break_min_spin.setValue(self.app_settings.get('page_break_min', 15))
page_duration_layout.addWidget(self.page_break_min_spin)
page_duration_layout.addWidget(QLabel("-"))
self.page_break_max_spin = QSpinBox()
self.page_break_max_spin.setRange(5, 300)
self.page_break_max_spin.setValue(self.app_settings.get('page_break_max', 45))
self.page_break_max_spin.setSuffix("s")
page_duration_layout.addWidget(self.page_break_max_spin)
manual_layout.addLayout(page_duration_layout, 1, 3)
# Batch breaks
manual_layout.addWidget(QLabel("Batch Break Every:"), 2, 0)
self.batch_break_interval_spin = QSpinBox()
self.batch_break_interval_spin.setRange(1, 50)
self.batch_break_interval_spin.setValue(self.app_settings.get('batch_break_interval', 5))
self.batch_break_interval_spin.setSuffix(" comics")
manual_layout.addWidget(self.batch_break_interval_spin, 2, 1)
manual_layout.addWidget(QLabel("Duration:"), 2, 2)
batch_duration_layout = QHBoxLayout()
self.batch_break_min_spin = QSpinBox()
self.batch_break_min_spin.setRange(1, 60)
self.batch_break_min_spin.setValue(self.app_settings.get('batch_break_min', 3))
batch_duration_layout.addWidget(self.batch_break_min_spin)
batch_duration_layout.addWidget(QLabel("-"))
self.batch_break_max_spin = QSpinBox()
self.batch_break_max_spin.setRange(1, 60)
self.batch_break_max_spin.setValue(self.app_settings.get('batch_break_max', 7))
self.batch_break_max_spin.setSuffix("s")
batch_duration_layout.addWidget(self.batch_break_max_spin)
manual_layout.addLayout(batch_duration_layout, 2, 3)
# Typing speed
manual_layout.addWidget(QLabel("Typing Speed:"), 3, 0)
self.typing_delay_spin = self.create_double_spinbox(0.01, 1.0, 0.01,
self.app_settings.get('typing_delay', 0.1))
self.typing_delay_spin.setSuffix(" sec/char")
manual_layout.addWidget(self.typing_delay_spin, 3, 1)
# Bottom row with reset and help buttons
button_layout = QHBoxLayout()
self.reset_timing_btn = QPushButton("Reset to Balanced")
self.reset_timing_btn.clicked.connect(self.reset_timing_defaults)
button_layout.addWidget(self.reset_timing_btn)
help_btn = QPushButton("Help")
help_btn.clicked.connect(self.show_timing_help_dialog)
button_layout.addWidget(help_btn)
button_layout.addStretch()
manual_layout.addLayout(button_layout, 4, 0, 1, 4)
# Add to horizontal layout with proper proportions
top_layout.addWidget(preset_group, 1) # Give preset group more space
top_layout.addWidget(manual_group, 1) # Equal space for manual group
main_layout.addLayout(top_layout)
# Set the timing section to expand properly
group.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Preferred)
parent_layout.addWidget(group)
def create_double_spinbox(self, min_val, max_val, step, value):
"""Create a double precision spinbox with specified parameters."""
spinbox = QDoubleSpinBox()
spinbox.setRange(min_val, max_val)
spinbox.setSingleStep(step)
spinbox.setDecimals(1)
spinbox.setValue(value)
return spinbox
def show_timing_help_dialog(self):
"""Show a compact help dialog for timing configuration."""
dialog = QMessageBox(self)
dialog.setWindowTitle("Timing Configuration Help")
dialog.setIcon(QMessageBox.Information)
help_text = """
<b>Timing Configuration Guide</b><br><br>
<b>🎯 Purpose:</b> Simulate human-like browsing to avoid automated detection<br><br>
<b>⚡ Action Delays:</b> Time between clicks and scrolls<br>
• Lower = Faster scraping, higher detection risk<br>
• Higher = Slower scraping, more realistic behavior<br><br>
<b>⏸️ Page Breaks:</b> Random pauses between pages<br>
• Simulates reading time and human fatigue<br>
• 70% chance with 15-45s duration is realistic<br><br>
<b>📚 Batch Breaks:</b> Pauses after processing multiple comics<br>
• Every 5 comics with 3-7s breaks simulates attention shifts<br><br>
<b>⌨️ Typing Speed:</b> Character delay when entering credentials<br>
• 0.1 sec = Average typing speed (40 WPM)<br><br>
<b>💡 Recommendations:</b><br>
• <b>Fast Mode:</b> Testing or when speed matters more than stealth<br>
• <b>Balanced Mode:</b> Recommended for regular use<br>
• <b>Stealth Mode:</b> Maximum safety but slower operation
"""
dialog.setText(help_text)
dialog.setStandardButtons(QMessageBox.Ok)
dialog.exec_()
def apply_timing_preset(self, preset_type):
"""Apply a timing preset configuration."""
if preset_type == "fast":
# Fast & Aggressive - minimal delays
self.action_delay_min_spin.setValue(0.1)
self.action_delay_max_spin.setValue(0.5)
self.page_break_chance_spin.setValue(20)
self.page_break_min_spin.setValue(5)
self.page_break_max_spin.setValue(10)
self.batch_break_interval_spin.setValue(15)
self.batch_break_min_spin.setValue(1)
self.batch_break_max_spin.setValue(2)
self.typing_delay_spin.setValue(0.05)
elif preset_type == "balanced":
# Balanced - default recommended settings
self.action_delay_min_spin.setValue(0.5)
self.action_delay_max_spin.setValue(2.0)
self.page_break_chance_spin.setValue(70)
self.page_break_min_spin.setValue(15)
self.page_break_max_spin.setValue(45)
self.batch_break_interval_spin.setValue(5)
self.batch_break_min_spin.setValue(3)
self.batch_break_max_spin.setValue(7)
self.typing_delay_spin.setValue(0.1)
elif preset_type == "stealth":
# Stealth - maximum human-like behavior
self.action_delay_min_spin.setValue(1.0)
self.action_delay_max_spin.setValue(3.0)
self.page_break_chance_spin.setValue(90)
self.page_break_min_spin.setValue(30)
self.page_break_max_spin.setValue(90)
self.batch_break_interval_spin.setValue(3)
self.batch_break_min_spin.setValue(5)
self.batch_break_max_spin.setValue(15)
self.typing_delay_spin.setValue(0.15)
# Save the new settings
self.save_current_settings()
# Show feedback
self.statusBar().showMessage(f"Applied {preset_type.title()} timing preset", 3000)
def reset_timing_defaults(self):
"""Reset all timing settings to safe defaults."""
self.apply_timing_preset("balanced")
def create_status_section(self, parent_layout): def create_status_section(self, parent_layout):
"""Create the status display section.""" """Create the status display section."""
group = QGroupBox("Status") group = QGroupBox("Status & Controls")
layout = QVBoxLayout(group) layout = QVBoxLayout(group)
# Status display
status_layout = QHBoxLayout()
status_info_layout = QVBoxLayout()
self.status_label = QLabel("Ready to start scraping...") self.status_label = QLabel("Ready to start scraping...")
self.status_label.setStyleSheet("font-weight: bold; color: #2E8B57;") self.status_label.setStyleSheet("font-weight: bold; color: #2E8B57; font-size: 13px;")
layout.addWidget(self.status_label) status_info_layout.addWidget(self.status_label)
# Progress bar # Progress bar
self.progress_bar = QProgressBar() self.progress_bar = QProgressBar()
self.progress_bar.setVisible(False) self.progress_bar.setVisible(False)
layout.addWidget(self.progress_bar) status_info_layout.addWidget(self.progress_bar)
parent_layout.addWidget(group) status_layout.addLayout(status_info_layout)
status_layout.addStretch()
# Control buttons
def create_control_section(self, parent_layout): button_layout = QVBoxLayout()
"""Create the control buttons section."""
layout = QHBoxLayout()
self.start_btn = QPushButton("Start Scraping") self.start_btn = QPushButton("Start Scraping")
self.start_btn.clicked.connect(self.start_scraping) self.start_btn.clicked.connect(self.start_scraping)
self.start_btn.setStyleSheet("QPushButton { background-color: #4CAF50; color: white; font-weight: bold; padding: 8px; }") button_layout.addWidget(self.start_btn)
layout.addWidget(self.start_btn)
layout.addStretch()
self.downloads_btn = QPushButton("Open Downloads Folder") self.downloads_btn = QPushButton("Open Downloads Folder")
self.downloads_btn.clicked.connect(self.open_downloads_folder) self.downloads_btn.clicked.connect(self.open_downloads_folder)
layout.addWidget(self.downloads_btn) button_layout.addWidget(self.downloads_btn)
parent_layout.addLayout(layout) status_layout.addLayout(button_layout)
layout.addLayout(status_layout)
parent_layout.addWidget(group)
def create_control_section(self, parent_layout):
"""Create the control buttons section - now integrated into status section."""
pass # This is now handled by create_status_section
def update_credential_status(self): def update_credential_status(self):
"""Update the credential status display.""" """Update the credential status display."""
@@ -289,13 +586,27 @@ class MainWindow(QMainWindow):
self.log_message(f"Starting scraping: {mode_name} mode, pages {start_page} to {end_page}") self.log_message(f"Starting scraping: {mode_name} mode, pages {start_page} to {end_page}")
# Create and start scraper thread # Create and start scraper thread
# Collect current timing configuration
timing_config = {
'action_delay_min': self.action_delay_min_spin.value(),
'action_delay_max': self.action_delay_max_spin.value(),
'page_break_chance': self.page_break_chance_spin.value(),
'page_break_min': self.page_break_min_spin.value(),
'page_break_max': self.page_break_max_spin.value(),
'batch_break_interval': self.batch_break_interval_spin.value(),
'batch_break_min': self.batch_break_min_spin.value(),
'batch_break_max': self.batch_break_max_spin.value(),
'typing_delay': self.typing_delay_spin.value(),
}
self.scraper_thread = ScraperThread( self.scraper_thread = ScraperThread(
username=credentials['username'], username=credentials['username'],
password=credentials['password'], password=credentials['password'],
start_page=start_page, start_page=start_page,
end_page=end_page, end_page=end_page,
scraping_mode=mode_index, scraping_mode=mode_index,
headless=self.headless_checkbox.isChecked() headless=self.headless_checkbox.isChecked(),
timing_config=timing_config
) )
# Connect signals # Connect signals
@@ -398,6 +709,16 @@ class MainWindow(QMainWindow):
'default_start_page': self.start_page_spin.value(), 'default_start_page': self.start_page_spin.value(),
'default_end_page': self.end_page_spin.value(), 'default_end_page': self.end_page_spin.value(),
'scraping_mode': self.mode_combo.currentIndex(), 'scraping_mode': self.mode_combo.currentIndex(),
# Timing settings
'action_delay_min': self.action_delay_min_spin.value(),
'action_delay_max': self.action_delay_max_spin.value(),
'page_break_chance': self.page_break_chance_spin.value(),
'page_break_min': self.page_break_min_spin.value(),
'page_break_max': self.page_break_max_spin.value(),
'batch_break_interval': self.batch_break_interval_spin.value(),
'batch_break_min': self.batch_break_min_spin.value(),
'batch_break_max': self.batch_break_max_spin.value(),
'typing_delay': self.typing_delay_spin.value(),
} }
settings.update(self.app_settings) # Keep other settings settings.update(self.app_settings) # Keep other settings
self.credential_manager.save_app_settings(settings) self.credential_manager.save_app_settings(settings)

58
gui/startup_dialog.py Normal file
View File

@@ -0,0 +1,58 @@
"""
Startup chooser: pick which site to scrape when the app boots.
"""
from PyQt5.QtWidgets import (
QDialog, QVBoxLayout, QHBoxLayout, QPushButton, QLabel
)
from PyQt5.QtCore import Qt
class StartupDialog(QDialog):
"""Small modal shown on boot to choose EBoek or Bookys."""
EBOEK = "eboek"
BOOKYS = "bookys"
def __init__(self, parent=None):
super().__init__(parent)
self.choice = None
self._init_ui()
def _init_ui(self):
self.setWindowTitle("Choose Scraper")
self.setModal(True)
self.setMinimumWidth(420)
layout = QVBoxLayout(self)
layout.setSpacing(18)
layout.setContentsMargins(28, 28, 28, 28)
title = QLabel("Which site do you want to scrape?")
title.setStyleSheet("font-size: 16px; font-weight: bold;")
title.setAlignment(Qt.AlignCenter)
layout.addWidget(title)
subtitle = QLabel("You can switch anytime from the menu bar.")
subtitle.setAlignment(Qt.AlignCenter)
subtitle.setStyleSheet("color: #666;")
layout.addWidget(subtitle)
buttons = QHBoxLayout()
buttons.setSpacing(14)
eboek_btn = QPushButton("EBoek.info\n\nDownload comic files")
eboek_btn.setMinimumHeight(90)
eboek_btn.clicked.connect(lambda: self._choose(self.EBOEK))
buttons.addWidget(eboek_btn)
bookys_btn = QPushButton("Bookys\n\nHarvest 1fichier links → .txt")
bookys_btn.setMinimumHeight(90)
bookys_btn.clicked.connect(lambda: self._choose(self.BOOKYS))
buttons.addWidget(bookys_btn)
layout.addLayout(buttons)
def _choose(self, choice):
self.choice = choice
self.accept()

View File

@@ -59,7 +59,8 @@ class EBoekScraperApp(QApplication):
# Handle exceptions # Handle exceptions
sys.excepthook = self.handle_exception sys.excepthook = self.handle_exception
self.main_window = None self.main_window = None # EBoek window
self.bookys_window = None # Bookys window
def set_application_icon(self): def set_application_icon(self):
"""Set the application icon if available.""" """Set the application icon if available."""
@@ -101,9 +102,16 @@ class EBoekScraperApp(QApplication):
# Check system requirements # Check system requirements
self.check_requirements() self.check_requirements()
# Create and show main window # Ask which site to scrape, then show the matching window.
self.main_window = MainWindow() from gui.startup_dialog import StartupDialog
self.main_window.show() chooser = StartupDialog()
chooser.exec_()
if chooser.choice == StartupDialog.BOOKYS:
self.show_bookys()
else:
# Default to EBoek (also covers the dialog being closed).
self.show_eboek()
return True return True
@@ -124,6 +132,27 @@ class EBoekScraperApp(QApplication):
return False return False
def show_eboek(self):
"""Show the EBoek window, hiding the Bookys one if present."""
if self.main_window is None:
self.main_window = MainWindow()
if self.bookys_window is not None:
self.bookys_window.hide()
self.main_window.show()
self.main_window.raise_()
self.main_window.activateWindow()
def show_bookys(self):
"""Show the Bookys window, hiding the EBoek one if present."""
from gui.bookys_window import BookysWindow
if self.bookys_window is None:
self.bookys_window = BookysWindow()
if self.main_window is not None:
self.main_window.hide()
self.bookys_window.show()
self.bookys_window.raise_()
self.bookys_window.activateWindow()
def check_requirements(self): def check_requirements(self):
"""Check system requirements and dependencies.""" """Check system requirements and dependencies."""
errors = [] errors = []

View File

@@ -43,6 +43,10 @@ python -m pyinstaller --onefile --windowed --name "EBoek_Scraper" ^
--hidden-import "gui.login_dialog" ^ --hidden-import "gui.login_dialog" ^
--hidden-import "gui.progress_dialog" ^ --hidden-import "gui.progress_dialog" ^
--hidden-import "utils.validators" ^ --hidden-import "utils.validators" ^
--hidden-import "core.bookys_scraper" ^
--hidden-import "core.bookys_thread" ^
--hidden-import "gui.bookys_window" ^
--hidden-import "gui.startup_dialog" ^
--exclude-module "tkinter" ^ --exclude-module "tkinter" ^
--exclude-module "matplotlib" ^ --exclude-module "matplotlib" ^
..\gui_main.py ..\gui_main.py

View File

@@ -92,6 +92,11 @@ def main():
"--hidden-import", "gui.login_dialog", "--hidden-import", "gui.login_dialog",
"--hidden-import", "gui.progress_dialog", "--hidden-import", "gui.progress_dialog",
"--hidden-import", "utils.validators", "--hidden-import", "utils.validators",
# Bookys scraper modules (imported lazily, so PyInstaller can't see them)
"--hidden-import", "core.bookys_scraper",
"--hidden-import", "core.bookys_thread",
"--hidden-import", "gui.bookys_window",
"--hidden-import", "gui.startup_dialog",
# Exclude unnecessary modules to reduce size # Exclude unnecessary modules to reduce size
"--exclude-module", "tkinter", "--exclude-module", "tkinter",
"--exclude-module", "matplotlib", "--exclude-module", "matplotlib",

View File

@@ -6,7 +6,7 @@ a = Analysis(
pathex=[], pathex=[],
binaries=[], binaries=[],
datas=[], datas=[],
hiddenimports=['PyQt5.QtCore', 'PyQt5.QtGui', 'PyQt5.QtWidgets', 'selenium', 'selenium.webdriver', 'selenium.webdriver.chrome', 'selenium.webdriver.chrome.options', 'selenium.webdriver.common.by', 'core.scraper', 'core.scraper_thread', 'core.credentials', 'gui.main_window', 'gui.login_dialog', 'gui.progress_dialog', 'utils.validators'], hiddenimports=['PyQt5.QtCore', 'PyQt5.QtGui', 'PyQt5.QtWidgets', 'selenium', 'selenium.webdriver', 'selenium.webdriver.chrome', 'selenium.webdriver.chrome.options', 'selenium.webdriver.common.by', 'core.scraper', 'core.scraper_thread', 'core.credentials', 'gui.main_window', 'gui.login_dialog', 'gui.progress_dialog', 'utils.validators', 'core.bookys_scraper', 'core.bookys_thread', 'gui.bookys_window', 'gui.startup_dialog'],
hookspath=[], hookspath=[],
hooksconfig={}, hooksconfig={},
runtime_hooks=[], runtime_hooks=[],

Binary file not shown.