feat: Add installation scripts for Windows and Unix-based systems
- Created `install_and_run.bat` for Windows installation and setup. - Created `install_and_run.sh` for Unix-based systems installation and setup. - Removed `main.py` as it is no longer needed. - Updated `requirements.txt` to specify package versions and added PyQt5. - Deleted `start.bat` as it is redundant. - Added unit tests for core functionality and scraping modes. - Implemented input validation utilities in `utils/validators.py`. - Added support for dual scraping modes in the scraper.
This commit is contained in:
Louis Mylle
1
gui/__init__.py
Normal file
1
gui/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
# GUI components for the scraper application
|
||||
317
gui/login_dialog.py
Normal file
317
gui/login_dialog.py
Normal file
@@ -0,0 +1,317 @@
|
||||
"""
|
||||
Login dialog for EBoek.info credential input.
|
||||
"""
|
||||
|
||||
from PyQt5.QtWidgets import (
|
||||
QDialog, QVBoxLayout, QHBoxLayout, QGridLayout,
|
||||
QPushButton, QLabel, QLineEdit, QCheckBox, QMessageBox, QProgressBar
|
||||
)
|
||||
from PyQt5.QtCore import Qt, QTimer, QThread, pyqtSignal
|
||||
from PyQt5.QtGui import QFont
|
||||
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
# Add the project root directory to Python path
|
||||
project_root = Path(__file__).parent.parent
|
||||
sys.path.insert(0, str(project_root))
|
||||
|
||||
from utils.validators import validate_username, validate_password, format_error_message
|
||||
|
||||
|
||||
class LoginTestThread(QThread):
|
||||
"""Thread for testing login credentials without blocking the UI."""
|
||||
|
||||
login_result = pyqtSignal(bool, str) # success, message
|
||||
|
||||
def __init__(self, username, password):
|
||||
super().__init__()
|
||||
self.username = username
|
||||
self.password = password
|
||||
|
||||
def run(self):
|
||||
"""Test the login credentials."""
|
||||
try:
|
||||
# Import here to avoid circular imports and ensure GUI responsiveness
|
||||
from core.scraper import Scraper
|
||||
|
||||
# Create a scraper instance for testing
|
||||
scraper = Scraper(headless=True)
|
||||
|
||||
# Attempt login
|
||||
success = scraper.login(self.username, self.password)
|
||||
|
||||
# Clean up
|
||||
scraper.close()
|
||||
|
||||
if success:
|
||||
self.login_result.emit(True, "Login successful!")
|
||||
else:
|
||||
self.login_result.emit(False, "Login failed. Please check your credentials.")
|
||||
|
||||
except Exception as e:
|
||||
self.login_result.emit(False, f"Error testing login: {str(e)}")
|
||||
|
||||
|
||||
class LoginDialog(QDialog):
|
||||
"""
|
||||
Dialog for entering EBoek.info login credentials.
|
||||
|
||||
Provides fields for username and password input, with options to save
|
||||
credentials and test them before saving.
|
||||
"""
|
||||
|
||||
def __init__(self, parent=None, credential_manager=None):
|
||||
super().__init__(parent)
|
||||
self.credential_manager = credential_manager
|
||||
self.test_thread = None
|
||||
|
||||
self.init_ui()
|
||||
self.load_existing_credentials()
|
||||
|
||||
def init_ui(self):
|
||||
"""Initialize the user interface."""
|
||||
self.setWindowTitle("EBoek.info Login")
|
||||
self.setModal(True)
|
||||
self.setFixedSize(400, 300)
|
||||
|
||||
layout = QVBoxLayout(self)
|
||||
|
||||
# Title
|
||||
title_label = QLabel("EBoek.info Credentials")
|
||||
title_font = QFont()
|
||||
title_font.setPointSize(14)
|
||||
title_font.setBold(True)
|
||||
title_label.setFont(title_font)
|
||||
title_label.setAlignment(Qt.AlignCenter)
|
||||
layout.addWidget(title_label)
|
||||
|
||||
layout.addSpacing(10)
|
||||
|
||||
# Credentials form
|
||||
form_layout = QGridLayout()
|
||||
|
||||
form_layout.addWidget(QLabel("Username:"), 0, 0)
|
||||
self.username_input = QLineEdit()
|
||||
self.username_input.setPlaceholderText("Enter your EBoek.info username")
|
||||
form_layout.addWidget(self.username_input, 0, 1)
|
||||
|
||||
form_layout.addWidget(QLabel("Password:"), 1, 0)
|
||||
self.password_input = QLineEdit()
|
||||
self.password_input.setEchoMode(QLineEdit.Password)
|
||||
self.password_input.setPlaceholderText("Enter your password")
|
||||
form_layout.addWidget(self.password_input, 1, 1)
|
||||
|
||||
layout.addLayout(form_layout)
|
||||
|
||||
layout.addSpacing(10)
|
||||
|
||||
# Options
|
||||
self.remember_checkbox = QCheckBox("Save credentials for future use")
|
||||
self.remember_checkbox.setChecked(True)
|
||||
layout.addWidget(self.remember_checkbox)
|
||||
|
||||
layout.addSpacing(5)
|
||||
|
||||
# Info text
|
||||
info_label = QLabel(
|
||||
"Note: Credentials are stored securely on your computer "
|
||||
"for convenience. You can clear them anytime from the Settings menu."
|
||||
)
|
||||
info_label.setWordWrap(True)
|
||||
info_label.setStyleSheet("color: #666; font-size: 10px;")
|
||||
layout.addWidget(info_label)
|
||||
|
||||
layout.addSpacing(15)
|
||||
|
||||
# Test progress (hidden initially)
|
||||
self.test_progress = QProgressBar()
|
||||
self.test_progress.setVisible(False)
|
||||
layout.addWidget(self.test_progress)
|
||||
|
||||
self.test_status_label = QLabel("")
|
||||
self.test_status_label.setVisible(False)
|
||||
layout.addWidget(self.test_status_label)
|
||||
|
||||
# Buttons
|
||||
button_layout = QHBoxLayout()
|
||||
|
||||
self.test_btn = QPushButton("Test Login")
|
||||
self.test_btn.clicked.connect(self.test_login)
|
||||
button_layout.addWidget(self.test_btn)
|
||||
|
||||
button_layout.addStretch()
|
||||
|
||||
self.ok_btn = QPushButton("OK")
|
||||
self.ok_btn.clicked.connect(self.accept_credentials)
|
||||
self.ok_btn.setDefault(True)
|
||||
button_layout.addWidget(self.ok_btn)
|
||||
|
||||
self.cancel_btn = QPushButton("Cancel")
|
||||
self.cancel_btn.clicked.connect(self.reject)
|
||||
button_layout.addWidget(self.cancel_btn)
|
||||
|
||||
layout.addLayout(button_layout)
|
||||
|
||||
# Connect Enter key to OK button
|
||||
self.username_input.returnPressed.connect(self.password_input.setFocus)
|
||||
self.password_input.returnPressed.connect(self.accept_credentials)
|
||||
|
||||
def load_existing_credentials(self):
|
||||
"""Load existing credentials if available."""
|
||||
if self.credential_manager:
|
||||
username = self.credential_manager.get_saved_username()
|
||||
if username:
|
||||
self.username_input.setText(username)
|
||||
# Focus password field if username is pre-filled
|
||||
self.password_input.setFocus()
|
||||
else:
|
||||
self.username_input.setFocus()
|
||||
|
||||
def validate_input(self):
|
||||
"""
|
||||
Validate the entered credentials.
|
||||
|
||||
Returns:
|
||||
tuple: (is_valid, errors_list)
|
||||
"""
|
||||
username = self.username_input.text().strip()
|
||||
password = self.password_input.text()
|
||||
|
||||
username_validation = validate_username(username)
|
||||
password_validation = validate_password(password)
|
||||
|
||||
all_errors = []
|
||||
all_errors.extend(username_validation.get('errors', []))
|
||||
all_errors.extend(password_validation.get('errors', []))
|
||||
|
||||
return len(all_errors) == 0, all_errors
|
||||
|
||||
def test_login(self):
|
||||
"""Test the login credentials."""
|
||||
# First validate input
|
||||
is_valid, errors = self.validate_input()
|
||||
if not is_valid:
|
||||
QMessageBox.warning(self, "Invalid Input", format_error_message(errors))
|
||||
return
|
||||
|
||||
# Disable UI elements during test
|
||||
self.test_btn.setEnabled(False)
|
||||
self.ok_btn.setEnabled(False)
|
||||
self.username_input.setEnabled(False)
|
||||
self.password_input.setEnabled(False)
|
||||
|
||||
# Show progress
|
||||
self.test_progress.setVisible(True)
|
||||
self.test_progress.setRange(0, 0) # Indeterminate progress
|
||||
self.test_status_label.setText("Testing login credentials...")
|
||||
self.test_status_label.setVisible(True)
|
||||
|
||||
# Start test thread
|
||||
username = self.username_input.text().strip()
|
||||
password = self.password_input.text()
|
||||
|
||||
self.test_thread = LoginTestThread(username, password)
|
||||
self.test_thread.login_result.connect(self.on_test_completed)
|
||||
self.test_thread.start()
|
||||
|
||||
def on_test_completed(self, success, message):
|
||||
"""Handle test completion."""
|
||||
# Re-enable UI elements
|
||||
self.test_btn.setEnabled(True)
|
||||
self.ok_btn.setEnabled(True)
|
||||
self.username_input.setEnabled(True)
|
||||
self.password_input.setEnabled(True)
|
||||
|
||||
# Hide progress
|
||||
self.test_progress.setVisible(False)
|
||||
|
||||
# Show result
|
||||
if success:
|
||||
self.test_status_label.setText("✓ " + message)
|
||||
self.test_status_label.setStyleSheet("color: #2E8B57; font-weight: bold;")
|
||||
else:
|
||||
self.test_status_label.setText("✗ " + message)
|
||||
self.test_status_label.setStyleSheet("color: #f44336; font-weight: bold;")
|
||||
|
||||
# Auto-hide status after 5 seconds
|
||||
QTimer.singleShot(5000, lambda: self.test_status_label.setVisible(False))
|
||||
|
||||
# Clean up thread
|
||||
self.test_thread = None
|
||||
|
||||
def accept_credentials(self):
|
||||
"""Accept and save the credentials."""
|
||||
# Validate input
|
||||
is_valid, errors = self.validate_input()
|
||||
if not is_valid:
|
||||
QMessageBox.warning(self, "Invalid Input", format_error_message(errors))
|
||||
return
|
||||
|
||||
username = self.username_input.text().strip()
|
||||
password = self.password_input.text()
|
||||
remember = self.remember_checkbox.isChecked()
|
||||
|
||||
# Save credentials if manager is available
|
||||
if self.credential_manager:
|
||||
if remember:
|
||||
success = self.credential_manager.save_credentials(username, password, remember=True)
|
||||
if not success:
|
||||
QMessageBox.warning(
|
||||
self, "Save Error",
|
||||
"Could not save credentials. They will be used for this session only."
|
||||
)
|
||||
else:
|
||||
# Clear any existing saved credentials if user unchecked remember
|
||||
self.credential_manager.clear_credentials()
|
||||
|
||||
# Accept the dialog
|
||||
self.accept()
|
||||
|
||||
def get_credentials(self):
|
||||
"""
|
||||
Get the entered credentials.
|
||||
|
||||
Returns:
|
||||
dict: Dictionary with 'username', 'password', and 'remember' keys
|
||||
"""
|
||||
return {
|
||||
'username': self.username_input.text().strip(),
|
||||
'password': self.password_input.text(),
|
||||
'remember': self.remember_checkbox.isChecked()
|
||||
}
|
||||
|
||||
def closeEvent(self, event):
|
||||
"""Handle dialog close event."""
|
||||
# Make sure test thread is stopped
|
||||
if self.test_thread and self.test_thread.isRunning():
|
||||
self.test_thread.quit()
|
||||
self.test_thread.wait(1000) # Wait up to 1 second
|
||||
|
||||
event.accept()
|
||||
|
||||
def reject(self):
|
||||
"""Handle dialog rejection (Cancel button)."""
|
||||
# Stop test thread if running
|
||||
if self.test_thread and self.test_thread.isRunning():
|
||||
self.test_thread.quit()
|
||||
self.test_thread.wait(1000)
|
||||
|
||||
super().reject()
|
||||
|
||||
|
||||
def show_login_dialog(parent=None, credential_manager=None):
|
||||
"""
|
||||
Convenience function to show login dialog and get credentials.
|
||||
|
||||
Args:
|
||||
parent: Parent widget
|
||||
credential_manager: CredentialManager instance
|
||||
|
||||
Returns:
|
||||
dict or None: Credentials if dialog accepted, None if cancelled
|
||||
"""
|
||||
dialog = LoginDialog(parent, credential_manager)
|
||||
if dialog.exec_() == QDialog.Accepted:
|
||||
return dialog.get_credentials()
|
||||
return None
|
||||
510
gui/main_window.py
Normal file
510
gui/main_window.py
Normal file
@@ -0,0 +1,510 @@
|
||||
"""
|
||||
Main application window for the EBoek.info scraper GUI.
|
||||
"""
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from PyQt5.QtWidgets import (
|
||||
QMainWindow, QWidget, QVBoxLayout, QHBoxLayout, QGridLayout,
|
||||
QPushButton, QLabel, QSpinBox, QTextEdit, QGroupBox,
|
||||
QCheckBox, QProgressBar, QMessageBox, QFileDialog, QMenuBar, QMenu, QAction,
|
||||
QComboBox
|
||||
)
|
||||
from PyQt5.QtCore import Qt, QTimer, pyqtSignal
|
||||
from PyQt5.QtGui import QFont, QIcon
|
||||
|
||||
# Import our custom modules
|
||||
import os
|
||||
import sys
|
||||
|
||||
# Add the project root directory to Python path so we can import our modules
|
||||
project_root = Path(__file__).parent.parent
|
||||
sys.path.insert(0, str(project_root))
|
||||
|
||||
from core.credentials import CredentialManager
|
||||
from core.scraper_thread import ScraperThread
|
||||
from utils.validators import validate_page_range, format_error_message
|
||||
from gui.login_dialog import LoginDialog
|
||||
from gui.progress_dialog import ProgressDialog
|
||||
|
||||
|
||||
class MainWindow(QMainWindow):
|
||||
"""
|
||||
Main application window for the EBoek.info scraper.
|
||||
|
||||
This window provides the primary interface for:
|
||||
- Managing credentials
|
||||
- Setting scraping parameters
|
||||
- Starting/stopping scraping operations
|
||||
- Monitoring progress and logs
|
||||
"""
|
||||
|
||||
# Custom signals
|
||||
scraping_requested = pyqtSignal(str, str, int, int, bool) # username, password, start_page, end_page, headless
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.credential_manager = CredentialManager()
|
||||
self.scraper_thread = None
|
||||
self.progress_dialog = None
|
||||
|
||||
# Load application settings
|
||||
self.app_settings = self.credential_manager.load_app_settings()
|
||||
if not self.app_settings:
|
||||
self.app_settings = self.credential_manager.get_default_settings()
|
||||
|
||||
self.init_ui()
|
||||
self.update_credential_status()
|
||||
|
||||
def init_ui(self):
|
||||
"""Initialize the user interface."""
|
||||
self.setWindowTitle("EBoek.info Scraper")
|
||||
self.setMinimumSize(600, 500)
|
||||
self.resize(700, 600)
|
||||
|
||||
# Create menu bar
|
||||
self.create_menu_bar()
|
||||
|
||||
# Create central widget
|
||||
central_widget = QWidget()
|
||||
self.setCentralWidget(central_widget)
|
||||
|
||||
# Main layout
|
||||
layout = QVBoxLayout(central_widget)
|
||||
|
||||
# Create sections
|
||||
self.create_credential_section(layout)
|
||||
self.create_scraping_section(layout)
|
||||
self.create_status_section(layout)
|
||||
self.create_control_section(layout)
|
||||
|
||||
# Status bar
|
||||
self.statusBar().showMessage("Ready")
|
||||
|
||||
def create_menu_bar(self):
|
||||
"""Create the menu bar."""
|
||||
menubar = self.menuBar()
|
||||
|
||||
# File menu
|
||||
file_menu = menubar.addMenu('File')
|
||||
|
||||
export_action = QAction('Export Settings', self)
|
||||
export_action.triggered.connect(self.export_settings)
|
||||
file_menu.addAction(export_action)
|
||||
|
||||
import_action = QAction('Import Settings', self)
|
||||
import_action.triggered.connect(self.import_settings)
|
||||
file_menu.addAction(import_action)
|
||||
|
||||
file_menu.addSeparator()
|
||||
|
||||
exit_action = QAction('Exit', self)
|
||||
exit_action.triggered.connect(self.close)
|
||||
file_menu.addAction(exit_action)
|
||||
|
||||
# Settings menu
|
||||
settings_menu = menubar.addMenu('Settings')
|
||||
|
||||
clear_creds_action = QAction('Clear Saved Credentials', self)
|
||||
clear_creds_action.triggered.connect(self.clear_credentials)
|
||||
settings_menu.addAction(clear_creds_action)
|
||||
|
||||
# Help menu
|
||||
help_menu = menubar.addMenu('Help')
|
||||
|
||||
about_action = QAction('About', self)
|
||||
about_action.triggered.connect(self.show_about)
|
||||
help_menu.addAction(about_action)
|
||||
|
||||
def create_credential_section(self, parent_layout):
|
||||
"""Create the credential management section."""
|
||||
group = QGroupBox("Credentials")
|
||||
layout = QHBoxLayout(group)
|
||||
|
||||
self.credential_status_label = QLabel("No credentials configured")
|
||||
layout.addWidget(self.credential_status_label)
|
||||
|
||||
layout.addStretch()
|
||||
|
||||
self.change_credentials_btn = QPushButton("Change Credentials")
|
||||
self.change_credentials_btn.clicked.connect(self.show_login_dialog)
|
||||
layout.addWidget(self.change_credentials_btn)
|
||||
|
||||
parent_layout.addWidget(group)
|
||||
|
||||
def create_scraping_section(self, parent_layout):
|
||||
"""Create the scraping configuration section."""
|
||||
group = QGroupBox("Scraping Configuration")
|
||||
layout = QGridLayout(group)
|
||||
|
||||
# Scraping mode selection
|
||||
layout.addWidget(QLabel("Mode:"), 0, 0)
|
||||
self.mode_combo = QComboBox()
|
||||
self.mode_combo.addItems([
|
||||
"All Comics (stripverhalen-alle)",
|
||||
"Latest Comics (laatste)"
|
||||
])
|
||||
self.mode_combo.setCurrentIndex(self.app_settings.get('scraping_mode', 0))
|
||||
self.mode_combo.setToolTip("Select which page type to scrape")
|
||||
self.mode_combo.currentIndexChanged.connect(self.on_mode_changed)
|
||||
layout.addWidget(self.mode_combo, 0, 1, 1, 3)
|
||||
|
||||
# Page range selection
|
||||
layout.addWidget(QLabel("Start Page:"), 1, 0)
|
||||
self.start_page_spin = QSpinBox()
|
||||
self.start_page_spin.setMinimum(1)
|
||||
self.start_page_spin.setMaximum(9999)
|
||||
self.start_page_spin.setValue(self.app_settings.get('default_start_page', 1))
|
||||
layout.addWidget(self.start_page_spin, 1, 1)
|
||||
|
||||
layout.addWidget(QLabel("End Page:"), 1, 2)
|
||||
self.end_page_spin = QSpinBox()
|
||||
self.end_page_spin.setMinimum(1)
|
||||
self.end_page_spin.setMaximum(9999)
|
||||
self.end_page_spin.setValue(self.app_settings.get('default_end_page', 1))
|
||||
layout.addWidget(self.end_page_spin, 1, 3)
|
||||
|
||||
# Mode description label
|
||||
self.mode_description_label = QLabel("")
|
||||
self.mode_description_label.setStyleSheet("color: #666; font-size: 11px; font-style: italic;")
|
||||
self.mode_description_label.setWordWrap(True)
|
||||
layout.addWidget(self.mode_description_label, 2, 0, 1, 4)
|
||||
|
||||
# Options
|
||||
self.headless_checkbox = QCheckBox("Headless Mode")
|
||||
self.headless_checkbox.setChecked(self.app_settings.get('headless_mode', True))
|
||||
self.headless_checkbox.setToolTip("Run browser in background (recommended)")
|
||||
layout.addWidget(self.headless_checkbox, 3, 0, 1, 2)
|
||||
|
||||
self.verbose_checkbox = QCheckBox("Verbose Logging")
|
||||
self.verbose_checkbox.setChecked(self.app_settings.get('verbose_logging', False))
|
||||
self.verbose_checkbox.setToolTip("Show detailed progress information")
|
||||
layout.addWidget(self.verbose_checkbox, 3, 2, 1, 2)
|
||||
|
||||
# Update mode description
|
||||
self.update_mode_description()
|
||||
|
||||
parent_layout.addWidget(group)
|
||||
|
||||
def on_mode_changed(self):
|
||||
"""Handle scraping mode selection change."""
|
||||
self.update_mode_description()
|
||||
self.save_current_settings()
|
||||
|
||||
def update_mode_description(self):
|
||||
"""Update the mode description text based on current selection."""
|
||||
mode_index = self.mode_combo.currentIndex()
|
||||
|
||||
if mode_index == 0: # All Comics
|
||||
description = ("Scrapes all comics from the 'stripverhalen-alle' page. "
|
||||
"This is the original scraping mode with complete comic archives.")
|
||||
elif mode_index == 1: # Latest Comics
|
||||
description = ("Scrapes latest comics from the 'laatste' page. "
|
||||
"This mode gets the most recently added comics with page parameter support.")
|
||||
else:
|
||||
description = ""
|
||||
|
||||
self.mode_description_label.setText(description)
|
||||
|
||||
def create_status_section(self, parent_layout):
|
||||
"""Create the status display section."""
|
||||
group = QGroupBox("Status")
|
||||
layout = QVBoxLayout(group)
|
||||
|
||||
self.status_label = QLabel("Ready to start scraping...")
|
||||
self.status_label.setStyleSheet("font-weight: bold; color: #2E8B57;")
|
||||
layout.addWidget(self.status_label)
|
||||
|
||||
# Progress bar
|
||||
self.progress_bar = QProgressBar()
|
||||
self.progress_bar.setVisible(False)
|
||||
layout.addWidget(self.progress_bar)
|
||||
|
||||
parent_layout.addWidget(group)
|
||||
|
||||
|
||||
def create_control_section(self, parent_layout):
|
||||
"""Create the control buttons section."""
|
||||
layout = QHBoxLayout()
|
||||
|
||||
self.start_btn = QPushButton("Start Scraping")
|
||||
self.start_btn.clicked.connect(self.start_scraping)
|
||||
self.start_btn.setStyleSheet("QPushButton { background-color: #4CAF50; color: white; font-weight: bold; padding: 8px; }")
|
||||
layout.addWidget(self.start_btn)
|
||||
|
||||
layout.addStretch()
|
||||
|
||||
self.downloads_btn = QPushButton("Open Downloads Folder")
|
||||
self.downloads_btn.clicked.connect(self.open_downloads_folder)
|
||||
layout.addWidget(self.downloads_btn)
|
||||
|
||||
parent_layout.addLayout(layout)
|
||||
|
||||
def update_credential_status(self):
|
||||
"""Update the credential status display."""
|
||||
username = self.credential_manager.get_saved_username()
|
||||
if username:
|
||||
self.credential_status_label.setText(f"Logged in as: {username}")
|
||||
self.credential_status_label.setStyleSheet("color: #2E8B57; font-weight: bold;")
|
||||
else:
|
||||
self.credential_status_label.setText("No credentials configured")
|
||||
self.credential_status_label.setStyleSheet("color: #FF6B35; font-weight: bold;")
|
||||
|
||||
def show_login_dialog(self):
|
||||
"""Show the login dialog for credential input."""
|
||||
dialog = LoginDialog(self, self.credential_manager)
|
||||
if dialog.exec_() == dialog.Accepted:
|
||||
self.update_credential_status()
|
||||
self.log_message("Credentials updated successfully.")
|
||||
|
||||
def start_scraping(self):
|
||||
"""Start the scraping process."""
|
||||
# Validate credentials
|
||||
credentials = self.credential_manager.load_credentials()
|
||||
if not credentials:
|
||||
QMessageBox.warning(self, "No Credentials",
|
||||
"Please configure your EBoek.info credentials first.")
|
||||
self.show_login_dialog()
|
||||
return
|
||||
|
||||
# Validate page range
|
||||
start_page = self.start_page_spin.value()
|
||||
end_page = self.end_page_spin.value()
|
||||
|
||||
validation = validate_page_range(start_page, end_page)
|
||||
if not validation['valid']:
|
||||
QMessageBox.warning(self, "Invalid Page Range",
|
||||
format_error_message(validation['errors']))
|
||||
return
|
||||
|
||||
# Save current settings
|
||||
self.save_current_settings()
|
||||
|
||||
# Get scraping mode
|
||||
mode_index = self.mode_combo.currentIndex()
|
||||
mode_names = ["All Comics", "Latest Comics"]
|
||||
mode_name = mode_names[mode_index] if mode_index < len(mode_names) else "Unknown"
|
||||
|
||||
# Start scraping
|
||||
self.log_message(f"Starting scraping: {mode_name} mode, pages {start_page} to {end_page}")
|
||||
|
||||
# Create and start scraper thread
|
||||
self.scraper_thread = ScraperThread(
|
||||
username=credentials['username'],
|
||||
password=credentials['password'],
|
||||
start_page=start_page,
|
||||
end_page=end_page,
|
||||
scraping_mode=mode_index,
|
||||
headless=self.headless_checkbox.isChecked()
|
||||
)
|
||||
|
||||
# Connect signals
|
||||
self.connect_scraper_signals()
|
||||
|
||||
# Show progress dialog
|
||||
self.progress_dialog = ProgressDialog(self, self.scraper_thread)
|
||||
self.progress_dialog.show()
|
||||
|
||||
# Start the thread
|
||||
self.scraper_thread.start()
|
||||
|
||||
# Update UI state
|
||||
self.start_btn.setEnabled(False)
|
||||
self.status_label.setText("Scraping in progress...")
|
||||
self.status_label.setStyleSheet("font-weight: bold; color: #FF8C00;")
|
||||
|
||||
def connect_scraper_signals(self):
|
||||
"""Connect signals from the scraper thread to UI updates."""
|
||||
if not self.scraper_thread:
|
||||
return
|
||||
|
||||
# Login signals
|
||||
self.scraper_thread.login_started.connect(self.on_login_started)
|
||||
self.scraper_thread.login_success.connect(self.on_login_success)
|
||||
self.scraper_thread.login_failed.connect(self.on_login_failed)
|
||||
|
||||
# Scraping completion
|
||||
self.scraper_thread.scraping_completed.connect(self.on_scraping_completed)
|
||||
|
||||
# Status updates
|
||||
self.scraper_thread.status_update.connect(self.log_message)
|
||||
self.scraper_thread.error_occurred.connect(self.on_error_occurred)
|
||||
|
||||
# Page progress
|
||||
self.scraper_thread.page_started.connect(self.on_page_started)
|
||||
self.scraper_thread.page_completed.connect(self.on_page_completed)
|
||||
|
||||
def on_login_started(self, username):
|
||||
"""Handle login started event."""
|
||||
self.log_message(f"Logging in as {username}...")
|
||||
|
||||
def on_login_success(self, username):
|
||||
"""Handle successful login."""
|
||||
self.log_message(f"Login successful for {username}")
|
||||
|
||||
def on_login_failed(self, username, error):
|
||||
"""Handle failed login."""
|
||||
self.log_message(f"Login failed for {username}: {error}")
|
||||
QMessageBox.critical(self, "Login Failed",
|
||||
f"Could not log in as {username}.\n\n{error}\n\nPlease check your credentials.")
|
||||
|
||||
def on_page_started(self, page_number, page_index, total_pages, url):
|
||||
"""Handle page started event."""
|
||||
self.log_message(f"Processing page {page_number} ({page_index}/{total_pages})")
|
||||
|
||||
def on_page_completed(self, page_number, comics_processed):
|
||||
"""Handle page completed event."""
|
||||
self.log_message(f"Page {page_number} completed - {comics_processed} comics processed")
|
||||
|
||||
def on_scraping_completed(self, summary):
|
||||
"""Handle scraping completion."""
|
||||
self.start_btn.setEnabled(True)
|
||||
|
||||
if summary.get('cancelled'):
|
||||
self.status_label.setText("Scraping cancelled")
|
||||
self.status_label.setStyleSheet("font-weight: bold; color: #FF6B35;")
|
||||
self.log_message("Scraping was cancelled by user")
|
||||
elif summary.get('success'):
|
||||
self.status_label.setText("Scraping completed")
|
||||
self.status_label.setStyleSheet("font-weight: bold; color: #2E8B57;")
|
||||
self.log_message(f"Scraping completed! Processed {summary.get('total_comics_processed', 0)} comics, "
|
||||
f"triggered {summary.get('total_downloads_triggered', 0)} downloads")
|
||||
else:
|
||||
self.status_label.setText("Scraping failed")
|
||||
self.status_label.setStyleSheet("font-weight: bold; color: #f44336;")
|
||||
self.log_message("Scraping failed - see errors above")
|
||||
|
||||
# Show summary
|
||||
if summary.get('errors'):
|
||||
error_count = len(summary['errors'])
|
||||
QMessageBox.warning(self, "Scraping Completed with Errors",
|
||||
f"Scraping completed but {error_count} errors occurred.\n"
|
||||
f"Check the log for details.")
|
||||
|
||||
def on_error_occurred(self, error_message):
|
||||
"""Handle error events."""
|
||||
self.log_message(f"ERROR: {error_message}")
|
||||
|
||||
def log_message(self, message):
|
||||
"""Log message - removed from main window, only shown in progress dialog."""
|
||||
# Activity log now only appears in the scraping progress dialog
|
||||
pass
|
||||
|
||||
def save_current_settings(self):
|
||||
"""Save current UI settings to configuration."""
|
||||
settings = {
|
||||
'headless_mode': self.headless_checkbox.isChecked(),
|
||||
'verbose_logging': self.verbose_checkbox.isChecked(),
|
||||
'default_start_page': self.start_page_spin.value(),
|
||||
'default_end_page': self.end_page_spin.value(),
|
||||
'scraping_mode': self.mode_combo.currentIndex(),
|
||||
}
|
||||
settings.update(self.app_settings) # Keep other settings
|
||||
self.credential_manager.save_app_settings(settings)
|
||||
self.app_settings = settings
|
||||
|
||||
def open_downloads_folder(self):
|
||||
"""Open the downloads folder in the system file manager."""
|
||||
downloads_path = Path.home() / "Downloads"
|
||||
|
||||
import sys
|
||||
import subprocess
|
||||
|
||||
try:
|
||||
if sys.platform == "win32":
|
||||
os.startfile(downloads_path)
|
||||
elif sys.platform == "darwin":
|
||||
subprocess.run(["open", str(downloads_path)])
|
||||
else: # linux
|
||||
subprocess.run(["xdg-open", str(downloads_path)])
|
||||
except Exception as e:
|
||||
QMessageBox.information(self, "Downloads Folder",
|
||||
f"Downloads are saved to:\n{downloads_path}\n\n"
|
||||
f"Could not open folder automatically: {e}")
|
||||
|
||||
def clear_credentials(self):
|
||||
"""Clear saved credentials."""
|
||||
reply = QMessageBox.question(self, "Clear Credentials",
|
||||
"Are you sure you want to clear the saved credentials?",
|
||||
QMessageBox.Yes | QMessageBox.No)
|
||||
|
||||
if reply == QMessageBox.Yes:
|
||||
if self.credential_manager.clear_credentials():
|
||||
self.update_credential_status()
|
||||
self.log_message("Saved credentials cleared.")
|
||||
QMessageBox.information(self, "Credentials Cleared",
|
||||
"Saved credentials have been cleared.")
|
||||
else:
|
||||
QMessageBox.warning(self, "Error", "Could not clear credentials.")
|
||||
|
||||
def export_settings(self):
|
||||
"""Export application settings to a file."""
|
||||
file_path, _ = QFileDialog.getSaveFileName(
|
||||
self, "Export Settings",
|
||||
"eboek_scraper_settings.json",
|
||||
"JSON files (*.json);;All files (*.*)"
|
||||
)
|
||||
|
||||
if file_path:
|
||||
if self.credential_manager.export_settings(file_path):
|
||||
QMessageBox.information(self, "Export Successful",
|
||||
f"Settings exported to:\n{file_path}")
|
||||
else:
|
||||
QMessageBox.warning(self, "Export Failed",
|
||||
"Could not export settings.")
|
||||
|
||||
def import_settings(self):
|
||||
"""Import application settings from a file."""
|
||||
file_path, _ = QFileDialog.getOpenFileName(
|
||||
self, "Import Settings",
|
||||
"",
|
||||
"JSON files (*.json);;All files (*.*)"
|
||||
)
|
||||
|
||||
if file_path:
|
||||
if self.credential_manager.import_settings(file_path):
|
||||
self.app_settings = self.credential_manager.load_app_settings()
|
||||
# Update UI with imported settings
|
||||
self.headless_checkbox.setChecked(self.app_settings.get('headless_mode', True))
|
||||
self.verbose_checkbox.setChecked(self.app_settings.get('verbose_logging', False))
|
||||
self.start_page_spin.setValue(self.app_settings.get('default_start_page', 1))
|
||||
self.end_page_spin.setValue(self.app_settings.get('default_end_page', 1))
|
||||
|
||||
QMessageBox.information(self, "Import Successful",
|
||||
f"Settings imported from:\n{file_path}")
|
||||
self.log_message("Settings imported successfully.")
|
||||
else:
|
||||
QMessageBox.warning(self, "Import Failed",
|
||||
"Could not import settings.")
|
||||
|
||||
def show_about(self):
|
||||
"""Show the about dialog."""
|
||||
QMessageBox.about(self, "About EBoek.info Scraper",
|
||||
"EBoek.info Scraper\n\n"
|
||||
"A GUI application for downloading comic strips from eboek.info.\n\n"
|
||||
"Features:\n"
|
||||
"• Automated login and scraping\n"
|
||||
"• Real-time progress monitoring\n"
|
||||
"• Human-like behavior simulation\n"
|
||||
"• Secure credential storage\n\n"
|
||||
"Built with Python and PyQt5.")
|
||||
|
||||
def closeEvent(self, event):
|
||||
"""Handle application close event."""
|
||||
if self.scraper_thread and self.scraper_thread.isRunning():
|
||||
reply = QMessageBox.question(self, "Scraping in Progress",
|
||||
"Scraping is currently in progress. "
|
||||
"Do you want to stop and exit?",
|
||||
QMessageBox.Yes | QMessageBox.No)
|
||||
|
||||
if reply == QMessageBox.Yes:
|
||||
self.scraper_thread.request_stop()
|
||||
# Give it a moment to stop gracefully
|
||||
self.scraper_thread.wait(3000) # Wait up to 3 seconds
|
||||
event.accept()
|
||||
else:
|
||||
event.ignore()
|
||||
else:
|
||||
# Save settings before closing
|
||||
self.save_current_settings()
|
||||
event.accept()
|
||||
477
gui/progress_dialog.py
Normal file
477
gui/progress_dialog.py
Normal file
@@ -0,0 +1,477 @@
|
||||
"""
|
||||
Progress dialog for real-time scraping progress monitoring.
|
||||
"""
|
||||
|
||||
import time
|
||||
from PyQt5.QtWidgets import (
|
||||
QDialog, QVBoxLayout, QHBoxLayout, QGridLayout,
|
||||
QPushButton, QLabel, QProgressBar, QTextEdit, QGroupBox
|
||||
)
|
||||
from PyQt5.QtCore import Qt, QTimer, pyqtSignal
|
||||
from PyQt5.QtGui import QFont
|
||||
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
# Add the project root directory to Python path
|
||||
project_root = Path(__file__).parent.parent
|
||||
sys.path.insert(0, str(project_root))
|
||||
|
||||
|
||||
class ProgressDialog(QDialog):
|
||||
"""
|
||||
Dialog for displaying real-time scraping progress.
|
||||
|
||||
Shows progress bars for pages and comics, current activity status,
|
||||
and a detailed log of operations with the ability to cancel.
|
||||
"""
|
||||
|
||||
# Signals
|
||||
cancel_requested = pyqtSignal()
|
||||
|
||||
def __init__(self, parent=None, scraper_thread=None):
|
||||
super().__init__(parent)
|
||||
self.scraper_thread = scraper_thread
|
||||
self.start_time = time.time()
|
||||
|
||||
# Progress tracking
|
||||
self.total_pages = 0
|
||||
self.current_page = 0
|
||||
self.total_comics_on_page = 0
|
||||
self.current_comic = 0
|
||||
self.total_comics_processed = 0
|
||||
self.total_downloads_triggered = 0
|
||||
|
||||
# Enhanced time tracking for better estimation
|
||||
self.comic_start_times = [] # Track start time of each comic
|
||||
self.comic_durations = [] # Track how long each comic took
|
||||
self.estimated_total_comics = 0 # Estimated total comics across all pages
|
||||
self.last_comic_start = None
|
||||
self.pages_processed = 0
|
||||
|
||||
self.init_ui()
|
||||
self.connect_signals()
|
||||
|
||||
def init_ui(self):
|
||||
"""Initialize the user interface."""
|
||||
self.setWindowTitle("Scraping Progress")
|
||||
self.setMinimumSize(500, 400)
|
||||
self.resize(600, 500)
|
||||
|
||||
layout = QVBoxLayout(self)
|
||||
|
||||
# Overall progress section
|
||||
self.create_overall_progress_section(layout)
|
||||
|
||||
# Current activity section
|
||||
self.create_activity_section(layout)
|
||||
|
||||
# Progress details section
|
||||
self.create_details_section(layout)
|
||||
|
||||
# Log section
|
||||
self.create_log_section(layout)
|
||||
|
||||
# Control buttons
|
||||
self.create_control_section(layout)
|
||||
|
||||
def create_overall_progress_section(self, parent_layout):
|
||||
"""Create the overall progress section."""
|
||||
group = QGroupBox("Overall Progress")
|
||||
layout = QVBoxLayout(group)
|
||||
|
||||
# Page progress
|
||||
self.page_progress_label = QLabel("Initializing...")
|
||||
layout.addWidget(self.page_progress_label)
|
||||
|
||||
self.page_progress_bar = QProgressBar()
|
||||
self.page_progress_bar.setRange(0, 100)
|
||||
layout.addWidget(self.page_progress_bar)
|
||||
|
||||
# Comic progress (current page)
|
||||
self.comic_progress_label = QLabel("Waiting for page data...")
|
||||
layout.addWidget(self.comic_progress_label)
|
||||
|
||||
self.comic_progress_bar = QProgressBar()
|
||||
self.comic_progress_bar.setRange(0, 100)
|
||||
layout.addWidget(self.comic_progress_bar)
|
||||
|
||||
parent_layout.addWidget(group)
|
||||
|
||||
def create_activity_section(self, parent_layout):
|
||||
"""Create the current activity section."""
|
||||
group = QGroupBox("Current Activity")
|
||||
layout = QVBoxLayout(group)
|
||||
|
||||
self.activity_label = QLabel("Starting scraper...")
|
||||
self.activity_label.setStyleSheet("font-weight: bold; color: #2E8B57;")
|
||||
layout.addWidget(self.activity_label)
|
||||
|
||||
# Current item details
|
||||
self.current_item_label = QLabel("")
|
||||
layout.addWidget(self.current_item_label)
|
||||
|
||||
parent_layout.addWidget(group)
|
||||
|
||||
def create_details_section(self, parent_layout):
|
||||
"""Create the progress details section."""
|
||||
group = QGroupBox("Session Details")
|
||||
layout = QGridLayout(group)
|
||||
|
||||
# Time information
|
||||
layout.addWidget(QLabel("Time Elapsed:"), 0, 0)
|
||||
self.elapsed_time_label = QLabel("00:00:00")
|
||||
layout.addWidget(self.elapsed_time_label, 0, 1)
|
||||
|
||||
layout.addWidget(QLabel("Estimated Remaining:"), 0, 2)
|
||||
self.remaining_time_label = QLabel("Calculating...")
|
||||
layout.addWidget(self.remaining_time_label, 0, 3)
|
||||
|
||||
# Progress statistics
|
||||
layout.addWidget(QLabel("Comics Processed:"), 1, 0)
|
||||
self.comics_processed_label = QLabel("0")
|
||||
layout.addWidget(self.comics_processed_label, 1, 1)
|
||||
|
||||
layout.addWidget(QLabel("Downloads Triggered:"), 1, 2)
|
||||
self.downloads_triggered_label = QLabel("0")
|
||||
layout.addWidget(self.downloads_triggered_label, 1, 3)
|
||||
|
||||
parent_layout.addWidget(group)
|
||||
|
||||
# Start timer for elapsed time updates
|
||||
self.timer = QTimer()
|
||||
self.timer.timeout.connect(self.update_elapsed_time)
|
||||
self.timer.start(1000) # Update every second
|
||||
|
||||
def create_log_section(self, parent_layout):
|
||||
"""Create the log display section."""
|
||||
group = QGroupBox("Activity Log")
|
||||
layout = QVBoxLayout(group)
|
||||
|
||||
self.log_text = QTextEdit()
|
||||
self.log_text.setReadOnly(True)
|
||||
self.log_text.setMaximumHeight(150)
|
||||
|
||||
# Set monospace font for logs (cross-platform)
|
||||
font = QFont()
|
||||
font.setFamily("Monaco, Consolas, 'Courier New', monospace") # Cross-platform fallback
|
||||
font.setPointSize(9)
|
||||
font.setStyleHint(QFont.TypeWriter) # Monospace hint
|
||||
self.log_text.setFont(font)
|
||||
|
||||
layout.addWidget(self.log_text)
|
||||
|
||||
parent_layout.addWidget(group)
|
||||
|
||||
def create_control_section(self, parent_layout):
|
||||
"""Create the control buttons section."""
|
||||
layout = QHBoxLayout()
|
||||
|
||||
layout.addStretch()
|
||||
|
||||
self.cancel_btn = QPushButton("Cancel Operation")
|
||||
self.cancel_btn.clicked.connect(self.cancel_scraping)
|
||||
self.cancel_btn.setStyleSheet("QPushButton { background-color: #f44336; color: white; font-weight: bold; padding: 8px; }")
|
||||
layout.addWidget(self.cancel_btn)
|
||||
|
||||
self.close_btn = QPushButton("Close")
|
||||
self.close_btn.clicked.connect(self.accept)
|
||||
self.close_btn.setEnabled(False) # Enabled when scraping completes
|
||||
layout.addWidget(self.close_btn)
|
||||
|
||||
parent_layout.addLayout(layout)
|
||||
|
||||
def connect_signals(self):
|
||||
"""Connect signals from the scraper thread."""
|
||||
if not self.scraper_thread:
|
||||
return
|
||||
|
||||
# Login signals
|
||||
self.scraper_thread.login_started.connect(self.on_login_started)
|
||||
self.scraper_thread.login_success.connect(self.on_login_success)
|
||||
self.scraper_thread.login_failed.connect(self.on_login_failed)
|
||||
|
||||
# Scraping progress
|
||||
self.scraper_thread.scraping_started.connect(self.on_scraping_started)
|
||||
self.scraper_thread.scraping_completed.connect(self.on_scraping_completed)
|
||||
|
||||
# Page progress
|
||||
self.scraper_thread.page_started.connect(self.on_page_started)
|
||||
self.scraper_thread.page_completed.connect(self.on_page_completed)
|
||||
self.scraper_thread.page_comics_found.connect(self.on_page_comics_found)
|
||||
|
||||
# Comic progress
|
||||
self.scraper_thread.comic_started.connect(self.on_comic_started)
|
||||
self.scraper_thread.comic_completed.connect(self.on_comic_completed)
|
||||
self.scraper_thread.comic_title_extracted.connect(self.on_comic_title_extracted)
|
||||
|
||||
# Download progress
|
||||
self.scraper_thread.download_links_found.connect(self.on_download_links_found)
|
||||
self.scraper_thread.download_started.connect(self.on_download_started)
|
||||
self.scraper_thread.download_triggered.connect(self.on_download_triggered)
|
||||
|
||||
# General status
|
||||
self.scraper_thread.status_update.connect(self.log_message)
|
||||
self.scraper_thread.error_occurred.connect(self.on_error_occurred)
|
||||
|
||||
# Timing events
|
||||
self.scraper_thread.page_break_started.connect(self.on_break_started)
|
||||
self.scraper_thread.comic_batch_break.connect(self.on_break_started)
|
||||
|
||||
def cancel_scraping(self):
|
||||
"""Cancel the scraping operation."""
|
||||
if self.scraper_thread:
|
||||
self.log_message("Cancel requested - stopping after current operation...")
|
||||
self.scraper_thread.request_stop()
|
||||
self.cancel_btn.setEnabled(False)
|
||||
self.activity_label.setText("Cancelling...")
|
||||
self.activity_label.setStyleSheet("font-weight: bold; color: #FF6B35;")
|
||||
|
||||
def log_message(self, message):
|
||||
"""Add a message to the log."""
|
||||
import datetime
|
||||
timestamp = datetime.datetime.now().strftime("%H:%M:%S")
|
||||
formatted_message = f"[{timestamp}] {message}"
|
||||
|
||||
self.log_text.append(formatted_message)
|
||||
|
||||
# Auto-scroll to bottom
|
||||
scrollbar = self.log_text.verticalScrollBar()
|
||||
scrollbar.setValue(scrollbar.maximum())
|
||||
|
||||
def update_elapsed_time(self):
|
||||
"""Update the elapsed time display with enhanced estimation."""
|
||||
elapsed = int(time.time() - self.start_time)
|
||||
hours = elapsed // 3600
|
||||
minutes = (elapsed % 3600) // 60
|
||||
seconds = elapsed % 60
|
||||
|
||||
self.elapsed_time_label.setText(f"{hours:02d}:{minutes:02d}:{seconds:02d}")
|
||||
|
||||
# Enhanced time estimation based on comic processing rate
|
||||
self.calculate_realistic_time_estimate(elapsed)
|
||||
|
||||
def calculate_realistic_time_estimate(self, elapsed):
|
||||
"""Calculate realistic time estimate based on comic processing data."""
|
||||
try:
|
||||
# If we have comic duration data, use it for accurate estimation
|
||||
if len(self.comic_durations) >= 2 and self.estimated_total_comics > 0:
|
||||
# Calculate average time per comic from actual data
|
||||
avg_time_per_comic = sum(self.comic_durations) / len(self.comic_durations)
|
||||
comics_remaining = self.estimated_total_comics - self.total_comics_processed
|
||||
|
||||
if comics_remaining > 0:
|
||||
estimated_remaining = int(comics_remaining * avg_time_per_comic)
|
||||
self.format_remaining_time(estimated_remaining)
|
||||
else:
|
||||
self.remaining_time_label.setText("Almost done!")
|
||||
|
||||
# Comic-based estimation when we know total comics but don't have enough duration data
|
||||
elif self.estimated_total_comics > 0 and self.total_comics_processed > 0:
|
||||
# Use current processing rate
|
||||
avg_time_per_comic = elapsed / self.total_comics_processed
|
||||
comics_remaining = self.estimated_total_comics - self.total_comics_processed
|
||||
|
||||
if comics_remaining > 0:
|
||||
estimated_remaining = int(comics_remaining * avg_time_per_comic)
|
||||
self.format_remaining_time(estimated_remaining)
|
||||
else:
|
||||
self.remaining_time_label.setText("Almost done!")
|
||||
|
||||
# Fallback to combined page + comic estimation
|
||||
elif self.total_pages > 0 and self.total_comics_on_page > 0:
|
||||
# Calculate combined progress: pages completed + current page comic progress
|
||||
pages_completed = self.current_page - 1
|
||||
current_page_progress = self.current_comic / self.total_comics_on_page
|
||||
total_progress = (pages_completed + current_page_progress) / self.total_pages
|
||||
|
||||
if total_progress > 0.05: # Only estimate after 5% progress
|
||||
estimated_total = elapsed / total_progress
|
||||
remaining = int(estimated_total - elapsed)
|
||||
if remaining > 0:
|
||||
self.format_remaining_time(remaining)
|
||||
else:
|
||||
self.remaining_time_label.setText("Almost done!")
|
||||
else:
|
||||
self.remaining_time_label.setText("Calculating...")
|
||||
else:
|
||||
self.remaining_time_label.setText("Calculating...")
|
||||
|
||||
except (ZeroDivisionError, ValueError):
|
||||
self.remaining_time_label.setText("Calculating...")
|
||||
|
||||
def format_remaining_time(self, remaining_seconds):
|
||||
"""Format remaining time into readable format."""
|
||||
if remaining_seconds <= 0:
|
||||
self.remaining_time_label.setText("Almost done!")
|
||||
return
|
||||
|
||||
rem_hours = remaining_seconds // 3600
|
||||
rem_minutes = (remaining_seconds % 3600) // 60
|
||||
rem_secs = remaining_seconds % 60
|
||||
|
||||
# Show different formats based on duration
|
||||
if rem_hours > 0:
|
||||
self.remaining_time_label.setText(f"{rem_hours:02d}:{rem_minutes:02d}:{rem_secs:02d}")
|
||||
elif rem_minutes > 0:
|
||||
self.remaining_time_label.setText(f"{rem_minutes:02d}:{rem_secs:02d}")
|
||||
else:
|
||||
self.remaining_time_label.setText(f"{rem_secs} sec")
|
||||
|
||||
def update_progress_bars(self):
|
||||
"""Update progress bars based on current state."""
|
||||
# Page progress
|
||||
if self.total_pages > 0:
|
||||
page_progress = int((self.current_page / self.total_pages) * 100)
|
||||
self.page_progress_bar.setValue(page_progress)
|
||||
self.page_progress_label.setText(f"Page {self.current_page} of {self.total_pages} ({page_progress}%)")
|
||||
|
||||
# Comic progress
|
||||
if self.total_comics_on_page > 0:
|
||||
comic_progress = int((self.current_comic / self.total_comics_on_page) * 100)
|
||||
self.comic_progress_bar.setValue(comic_progress)
|
||||
self.comic_progress_label.setText(f"Comic {self.current_comic} of {self.total_comics_on_page} ({comic_progress}%)")
|
||||
|
||||
def update_statistics(self):
|
||||
"""Update the statistics display."""
|
||||
self.comics_processed_label.setText(str(self.total_comics_processed))
|
||||
self.downloads_triggered_label.setText(str(self.total_downloads_triggered))
|
||||
|
||||
# Event handlers
|
||||
def on_login_started(self, username):
|
||||
"""Handle login started."""
|
||||
self.activity_label.setText(f"Logging in as {username}...")
|
||||
self.log_message(f"Logging in as {username}")
|
||||
|
||||
def on_login_success(self, username):
|
||||
"""Handle successful login."""
|
||||
self.activity_label.setText("Login successful - starting scraper...")
|
||||
self.log_message(f"Login successful for {username}")
|
||||
|
||||
def on_login_failed(self, username, error):
|
||||
"""Handle failed login."""
|
||||
self.activity_label.setText("Login failed")
|
||||
self.activity_label.setStyleSheet("font-weight: bold; color: #f44336;")
|
||||
self.log_message(f"Login failed: {error}")
|
||||
|
||||
def on_scraping_started(self, start_page, end_page, total_pages):
|
||||
"""Handle scraping start."""
|
||||
self.total_pages = total_pages
|
||||
self.current_page = 0
|
||||
self.activity_label.setText(f"Starting scraping: pages {start_page} to {end_page}")
|
||||
self.log_message(f"Starting scraping: pages {start_page} to {end_page}")
|
||||
self.update_progress_bars()
|
||||
|
||||
def on_scraping_completed(self, summary):
|
||||
"""Handle scraping completion."""
|
||||
self.cancel_btn.setEnabled(False)
|
||||
self.close_btn.setEnabled(True)
|
||||
self.timer.stop()
|
||||
|
||||
if summary.get('cancelled'):
|
||||
self.activity_label.setText("Scraping cancelled by user")
|
||||
self.activity_label.setStyleSheet("font-weight: bold; color: #FF6B35;")
|
||||
elif summary.get('success'):
|
||||
self.activity_label.setText("Scraping completed successfully!")
|
||||
self.activity_label.setStyleSheet("font-weight: bold; color: #2E8B57;")
|
||||
else:
|
||||
self.activity_label.setText("Scraping completed with errors")
|
||||
self.activity_label.setStyleSheet("font-weight: bold; color: #f44336;")
|
||||
|
||||
# Update final statistics
|
||||
self.total_comics_processed = summary.get('total_comics_processed', 0)
|
||||
self.total_downloads_triggered = summary.get('total_downloads_triggered', 0)
|
||||
self.update_statistics()
|
||||
|
||||
self.log_message("Scraping operation completed")
|
||||
|
||||
def on_page_started(self, page_number, page_index, total_pages, url):
|
||||
"""Handle page start."""
|
||||
self.current_page = page_index
|
||||
self.current_comic = 0
|
||||
self.total_comics_on_page = 0
|
||||
self.activity_label.setText(f"Processing page {page_number}...")
|
||||
self.current_item_label.setText(f"URL: {url}")
|
||||
self.update_progress_bars()
|
||||
self.log_message(f"Started processing page {page_number}")
|
||||
|
||||
def on_page_completed(self, page_number, comics_processed):
|
||||
"""Handle page completion."""
|
||||
self.pages_processed = self.current_page
|
||||
self.log_message(f"Completed page {page_number} - {comics_processed} comics processed")
|
||||
|
||||
def on_page_comics_found(self, page_number, comic_count):
|
||||
"""Handle comics found on page."""
|
||||
self.total_comics_on_page = comic_count
|
||||
self.current_comic = 0
|
||||
|
||||
# Update estimated total comics based on current data
|
||||
if self.total_pages > 0 and self.current_page > 0:
|
||||
avg_comics_per_page = (self.total_comics_processed + comic_count) / self.current_page
|
||||
self.estimated_total_comics = int(avg_comics_per_page * self.total_pages)
|
||||
|
||||
self.log_message(f"Found {comic_count} comics on page {page_number}")
|
||||
self.update_progress_bars()
|
||||
|
||||
def on_comic_started(self, page_number, comic_index, total_comics, url):
|
||||
"""Handle comic start."""
|
||||
self.current_comic = comic_index
|
||||
self.last_comic_start = time.time() # Track start time for duration calculation
|
||||
self.activity_label.setText(f"Processing comic {comic_index} of {total_comics}...")
|
||||
self.current_item_label.setText(f"URL: {url}")
|
||||
self.update_progress_bars()
|
||||
|
||||
def on_comic_completed(self, title, downloads_triggered, page_number, comic_index):
|
||||
"""Handle comic completion."""
|
||||
# Track timing data for enhanced estimation
|
||||
if self.last_comic_start is not None:
|
||||
comic_duration = time.time() - self.last_comic_start
|
||||
self.comic_durations.append(comic_duration)
|
||||
# Keep only recent durations for adaptive estimation (last 20 comics)
|
||||
if len(self.comic_durations) > 20:
|
||||
self.comic_durations = self.comic_durations[-20:]
|
||||
|
||||
# Update live counters
|
||||
self.total_comics_processed += 1
|
||||
# Note: downloads_triggered counter is now updated in real-time in on_download_triggered
|
||||
self.update_statistics() # This updates the live display
|
||||
self.log_message(f"Completed: {title} ({downloads_triggered} downloads)")
|
||||
|
||||
def on_comic_title_extracted(self, title, url):
|
||||
"""Handle comic title extraction."""
|
||||
self.current_item_label.setText(f"Processing: {title}")
|
||||
|
||||
def on_download_links_found(self, title, download_count):
|
||||
"""Handle download links found."""
|
||||
self.log_message(f"Found {download_count} download links for: {title}")
|
||||
|
||||
def on_download_started(self, file_name, url, index, total):
|
||||
"""Handle download start."""
|
||||
self.activity_label.setText(f"Downloading file {index} of {total}")
|
||||
self.current_item_label.setText(f"File: {file_name}")
|
||||
|
||||
def on_download_triggered(self, url):
|
||||
"""Handle download triggered."""
|
||||
# Update download counter in real-time
|
||||
self.total_downloads_triggered += 1
|
||||
self.update_statistics()
|
||||
|
||||
def on_error_occurred(self, error_message):
|
||||
"""Handle error."""
|
||||
self.log_message(f"ERROR: {error_message}")
|
||||
|
||||
def on_break_started(self, duration, context=None):
|
||||
"""Handle break start."""
|
||||
self.activity_label.setText(f"Taking a break for {duration:.1f} seconds...")
|
||||
self.current_item_label.setText("Human-like delay in progress...")
|
||||
|
||||
def closeEvent(self, event):
|
||||
"""Handle dialog close."""
|
||||
if self.scraper_thread and self.scraper_thread.isRunning():
|
||||
# Don't allow closing while scraping is active
|
||||
event.ignore()
|
||||
else:
|
||||
# Stop timer
|
||||
if hasattr(self, 'timer'):
|
||||
self.timer.stop()
|
||||
event.accept()
|
||||
Reference in New Issue
Block a user