#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""
Phase 1: Key Extraction
Extracts Kindle keys from individual book folders with per-book processing and logging
"""

import os
import sys
import shutil
import subprocess
import json
import threading
import time
from datetime import datetime

# Add modules to path
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

from modules.utils import (
    print_step, print_ok, print_error, print_warn,
    check_extractor_exists, check_write_permissions, rmtree_with_retry,
    get_raw_log_path, append_to_raw_log, cleanup_old_raw_logs,
    display_progress_timer, SCRIPT_VERSION,
    append_to_failed_books, resolve_failed_book_title
)


def find_kindle_exe():
    """Search for Kindle.exe in the two known installation locations"""
    from modules.utils import print_warn
    
    user_home = os.path.expanduser("~")
    
    appdata_kindle = os.path.join(user_home, "AppData", "Local", "Amazon", "Kindle", "application")
    appdata_exe = os.path.join(appdata_kindle, "Kindle.exe")
    temp_marker = os.path.join(appdata_kindle, "TEMP.txt")
    
    program_files_kindle = r"C:\Program Files (x86)\Amazon\Kindle"
    program_files_exe = os.path.join(program_files_kindle, "Kindle.exe")
    
    appdata_exists = os.path.exists(appdata_exe)
    program_files_exists = os.path.exists(program_files_exe)
    
    # Check if there's a leftover temp copy in AppData (has TEMP.txt)
    has_temp_marker = os.path.exists(temp_marker)
    
    # NEVER delete AppData Kindle folder unless it has TEMP.txt marker
    # TEMP.txt means it was created by THIS script as a temp copy
    # Without TEMP.txt, it's the user's real Kindle installation
    
    # Determine source for extraction - prefer Program Files
    if program_files_exists:
        # Program Files exists - use it as source
        if appdata_exists and has_temp_marker:
            print_warn("Found leftover temp Kindle copy in AppData, will clean up after extraction")
        elif appdata_exists:
            print_warn("Found Kindle.exe in both locations - using Program Files")
        return program_files_kindle, True  # Need to create temp copy
    elif appdata_exists and has_temp_marker:
        # Only AppData exists and it's a temp copy (has TEMP.txt)
        # Clean it up and return None (no source to copy from)
        print_warn("Found leftover temp Kindle copy in AppData, cleaning up...")
        try:
            shutil.rmtree(appdata_kindle)
        except:
            pass
        return None, False
    elif appdata_exists:
        # Only AppData exists and it's the real installation (no TEMP.txt)
        # Use it directly without creating temp copy
        return appdata_kindle, False  # No temp copy needed - use real install directly
    else:
        return None, False


def create_temp_kindle_copy(source_dir):
    """Create a temporary copy of Kindle installation in AppData"""
    from modules.utils import print_step, print_ok, print_error
    
    user_home = os.path.expanduser("~")
    dest_dir = os.path.join(user_home, "AppData", "Local", "Amazon", "Kindle", "application")
    temp_marker = os.path.join(dest_dir, "TEMP.txt")
    
    # If source and dest are the same, we need a different approach
    if os.path.normcase(os.path.normpath(source_dir)) == os.path.normcase(os.path.normpath(dest_dir)):
        # This shouldn't happen with our updated logic, but handle it just in case
        print_step("Source and destination are the same - skipping copy")
        return dest_dir
    
    try:
        print_step("Creating temporary Kindle copy...")
        print(f"  Copying from: {source_dir}")
        print(f"  Copying to: {dest_dir}")
        print("  (This may take a minute...)")
        print()
        
        shutil.copytree(source_dir, dest_dir)
        print_ok("Kindle folder copied successfully")
        
        with open(temp_marker, 'w') as f:
            f.write(f"Temporary Kindle copy for key extraction\n")
            f.write(f"Created: {datetime.now()}\n")
            f.write(f"Source: {source_dir}\n")
        print_ok("Marker file created")
        print()
        
        return dest_dir
        
    except Exception as e:
        print_error(f"Failed to create temporary copy: {e}")
        if os.path.exists(dest_dir):
            try:
                shutil.rmtree(dest_dir)
            except:
                pass
        raise


def cleanup_temp_kindle_copy(kindle_dir):
    """Remove the temporary Kindle copy after extraction"""
    from modules.utils import print_step, print_ok, print_error
    
    temp_marker = os.path.join(kindle_dir, "TEMP.txt")
    
    if os.path.exists(temp_marker):
        print_step("Cleaning up temporary Kindle copy...")
        try:
            shutil.rmtree(kindle_dir)
            print_ok("Temporary copy removed successfully")
        except Exception as e:
            print_error(f"Failed to cleanup temporary copy: {e}")
        print()


def load_book_history(working_dir):
    """Load book processing history from history.txt"""
    from modules.utils import print_ok, print_warn
    
    history_file = os.path.join(working_dir, "history.txt")
    processed_asins = set()
    
    if os.path.exists(history_file):
        try:
            with open(history_file, 'r', encoding='utf-8') as f:
                for line in f:
                    asin = line.strip()
                    if asin:
                        processed_asins.add(asin)
            print_ok(f"Loaded history: {len(processed_asins)} book(s) previously processed")
        except Exception as e:
            print_warn(f"Failed to load history file: {e}")
    else:
        print_step("No history file found - this appears to be first run")
    
    return processed_asins


def append_to_history(working_dir, asin):
    """Append ASIN to history.txt after successful extraction"""
    from modules.utils import print_warn
    
    history_file = os.path.join(working_dir, "history.txt")
    
    try:
        os.makedirs(os.path.dirname(history_file), exist_ok=True)
        
        with open(history_file, 'a', encoding='utf-8') as f:
            f.write(f"{asin}\n")
    except Exception as e:
        print_warn(f"Failed to update history file: {e}")


def scan_kindle_content_directory(content_dir):
    """Scan Kindle Content directory for individual book folders"""
    from modules.utils import print_error
    
    book_folders = []
    
    try:
        if not os.path.exists(content_dir):
            print_error(f"Content directory does not exist: {content_dir}")
            return []
        
        for root, dirs, files in os.walk(content_dir):
            book_files = [f for f in files if f.lower().endswith(('.azw', '.kfx', '.kfx-zip', '.azw3'))]
            
            if book_files:
                folder_name = os.path.basename(root)
                
                if '_' in folder_name:
                    asin = folder_name.split('_')[0]
                else:
                    first_book = book_files[0]
                    asin = os.path.splitext(first_book)[0].split('_')[0]
                
                book_title = asin
                book_folders.append((asin, root, book_title))
        
        return book_folders
        
    except Exception as e:
        print_error(f"Error scanning content directory: {e}")
        return []


def extract_keys_from_single_book(extractor_path, kindle_dir, book_folder, temp_key, temp_k4i, asin, book_title, working_dir=None, raw_log_path=None, book_prefix="", cleanup_temp=False):
    """Extract keys from a single book folder
    
    Args:
        cleanup_temp: If True, clean up temp_extraction after extraction. 
                     If False, leave it for batch cleanup later.
    """
    from modules.utils import print_error
    
    script_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
    user_home = os.path.expanduser("~")
    
    if working_dir is None:
        can_write, _ = check_write_permissions(script_dir)
        if can_write:
            working_dir = script_dir
        else:
            working_dir = os.path.join(user_home, "AppData", "Local", "Kindle_Key_Finder")
    
    temp_dir = os.path.join(working_dir, "temp_extraction")
    timer_stopped = None
    temp_book = None  # Track for per-book cleanup in finally block
    
    try:
        os.makedirs(temp_dir, exist_ok=True)
        
        extractor_in_kindle = os.path.join(kindle_dir, os.path.basename(extractor_path))
        if not os.path.exists(extractor_in_kindle):
            shutil.copy2(extractor_path, kindle_dir)
        
        # Create unique book subfolder in temp_extraction
        book_name = os.path.basename(book_folder)
        temp_book = os.path.join(temp_dir, book_name)
        
        # If book folder already exists in temp, remove it first
        if os.path.exists(temp_book):
            success = rmtree_with_retry(temp_book, max_retries=5, retry_delay=0.5)
            if not success:
                return False, None, None, f"Failed to clean existing temp book folder", asin
        
        try:
            shutil.copytree(book_folder, temp_book)
        except (OSError, PermissionError) as e:
            error_details = f"Access denied copying book folder - {str(e)}"
            return False, None, None, error_details, asin
        
        # Timer display thread
        timer_stopped = threading.Event()
        timeout_seconds = 60
        
        def display_extraction_timer():
            start_time = time.time()
            while not timer_stopped.is_set():
                elapsed = int(time.time() - start_time)
                minutes, seconds = divmod(elapsed, 60)
                timeout_mins, timeout_secs = divmod(timeout_seconds, 60)
                timer_str = f" [ {minutes:02d}:{seconds:02d} / {timeout_mins:02d}:{timeout_secs:02d} ]"
                print(f"\r{book_prefix}{timer_str}", end='', flush=True)
                time.sleep(1)
        
        timer_thread = threading.Thread(target=display_extraction_timer, daemon=True)
        timer_thread.start()
        
        # CRITICAL: Add 2-second delay before running extractor
        # The extractor crashes if run immediately - needs time for Kindle.exe memory state
        time.sleep(2)
        
        # Run extractor
        cmd = [extractor_in_kindle, temp_dir, temp_key, temp_k4i]
        
        result = subprocess.run(
            cmd,
            capture_output=True,
            text=True,
            timeout=timeout_seconds,
            cwd=kindle_dir,
            env=os.environ.copy()  # Use current environment explicitly
        )
        
        timer_stopped.set()
        timer_thread.join(timeout=1)
        # Clear the entire line including timer (use enough spaces to cover long titles + timer)
        print("\r" + " " * 150 + "\r", end='', flush=True)
        print(f"{book_prefix}", end='', flush=True)
        
        # Log to raw log if enabled
        if raw_log_path:
            append_to_raw_log(
                raw_log_path,
                cmd,
                result.stdout,
                result.stderr,
                result.returncode,
                book_info=f"{asin} - {book_title}"
            )
        
        # Parse output for DSN and tokens FIRST (before checking return code)
        dsn = None
        tokens = None
        stdout_text = result.stdout
        
        if stdout_text:
            lines = stdout_text.split('\n')
            for line in lines:
                if line.startswith('DSN '):
                    dsn = line.replace('DSN ', '').strip()
                elif line.startswith('Tokens '):
                    tokens_line = line.replace('Tokens ', '').strip()
                    if ',' in tokens_line:
                        tokens = tokens_line.split(',')[0].strip()
                    else:
                        tokens = tokens_line
        
        # Check if key files were generated FIRST (extractor may crash after writing keys)
        # Exit code 3221225477 (0xC0000005) is ACCESS_VIOLATION - extractor crashes but keys may be valid
        key_files_exist = os.path.exists(temp_key) and os.path.exists(temp_k4i)
        
        if not key_files_exist:
            # Only fail if key files weren't created
            if result.returncode != 0:
                # -------------------------------------------------------
                # Diagnose well-known exit codes before reporting a generic
                # failure, so the user (and the startup.log) get an
                # actionable message instead of just a hex crash code.
                #
                # 3221225477 = 0xC0000005 = STATUS_ACCESS_VIOLATION
                #   The extractor hard-codes memory offsets for specific
                #   Kindle versions.  A newer build causes checkExecs() to
                #   dereference an invalid pointer and crash immediately.
                #   KFXKeyExtractor supports 2.8.0 / 2.8.1 / 2.8.2.
                #   KFXArchiver supports  2.8.0 / 2.8.1 / 2.8.2 / 2.8.3.
                #   Kindle 2.8.3  -> switch to alt method (KFXArchiver).
                #   Kindle 2.8.4+ -> downgrade required.
                #
                # NOTE: validate_all_requirements() now catches this
                #   BEFORE Phase 1 runs, so this branch is only reached
                #   if the version check failed silently (e.g. PowerShell
                #   unavailable) or the user bypassed the version gate.
                # -------------------------------------------------------
                ACCESS_VIOLATION = 3221225477  # 0xC0000005
                if result.returncode == ACCESS_VIOLATION:
                    error_msg = (
                        f"Key extraction failed (exit code {result.returncode} - "
                        f"STATUS_ACCESS_VIOLATION). "
                        f"This usually means Kindle for PC is a version not "
                        f"supported by KFXKeyExtractor (supports 2.8.0-2.8.2). "
                        f"KFXArchiver (alt method) supports up to 2.8.3. "
                        f"Check Logs/startup.log for the detected Kindle version."
                    )
                else:
                    error_msg = f"Key extraction failed (exit code {result.returncode})"
            else:
                error_msg = "Key files not generated"
            return False, None, None, error_msg, asin
        
        # Key files exist - extraction was successful even if extractor crashed
        return True, dsn, tokens, None, asin
        
    except subprocess.TimeoutExpired:
        if timer_stopped:
            timer_stopped.set()
        # Clear the entire line including timer (use enough spaces to cover long titles + timer)
        print("\r" + " " * 150 + "\r", end='', flush=True)
        print(f"{book_prefix}", end='', flush=True)
        return False, None, None, f"Timeout after 60 seconds", asin
    except Exception as e:
        return False, None, None, str(e), asin
    finally:
        # Always delete this book's individual subfolder from temp_extraction after
        # the extractor finishes (success, failure, or timeout).
        # Previously this was deferred to a single batch rmtree at end-of-run,
        # but that caused temp_extraction to accumulate ALL book folders throughout
        # the run. The extractor is invoked with the parent temp_dir path, so it
        # scans every accumulated subfolder on each call - becoming progressively
        # slower until it hung (60s timeout) on large libraries (~700+ books).
        # rmtree_with_retry handles the Windows handle-release race after a kill.
        if temp_book and os.path.exists(temp_book):
            rmtree_with_retry(temp_book, max_retries=5, retry_delay=0.5)


def append_keys_to_files(output_key, output_k4i, temp_key, temp_k4i):
    """Append newly extracted keys to existing key files"""
    try:
        if not os.path.exists(temp_key) or not os.path.exists(temp_k4i):
            return False, "Temporary key files not found"
        
        with open(temp_key, 'r') as f:
            new_key_content = f.read().strip()
        
        with open(temp_k4i, 'r') as f:
            new_k4i_content = f.read().strip()
        
        # For kindlekey.txt: Append if not duplicate
        if os.path.exists(output_key):
            with open(output_key, 'r') as f:
                existing_key_content = f.read()
            
            if new_key_content not in existing_key_content:
                with open(output_key, 'a') as f:
                    f.write('\n' + new_key_content)
        else:
            with open(output_key, 'w') as f:
                f.write(new_key_content)
        
        # For kindlekey.k4i: Merge JSON data
        if os.path.exists(output_k4i):
            with open(output_k4i, 'r') as f:
                existing_k4i = json.load(f)
            
            new_k4i = json.loads(new_k4i_content)
            
            for key in ['kindle.account.secrets', 'kindle.account.new_secrets', 'kindle.account.clear_old_secrets']:
                if key in new_k4i:
                    if key not in existing_k4i:
                        existing_k4i[key] = []
                    for item in new_k4i[key]:
                        if item not in existing_k4i[key]:
                            existing_k4i[key].append(item)
            
            if 'DSN' in new_k4i and new_k4i['DSN']:
                existing_k4i['DSN'] = new_k4i['DSN']
            
            if 'kindle.account.tokens' in new_k4i and new_k4i['kindle.account.tokens']:
                existing_k4i['kindle.account.tokens'] = new_k4i['kindle.account.tokens']
            
            with open(output_k4i, 'w') as f:
                json.dump(existing_k4i, f, indent=2)
        else:
            with open(output_k4i, 'w') as f:
                f.write(new_k4i_content)
        
        return True, ""
        
    except Exception as e:
        return False, str(e)


def read_book_types_from_db(content_dir):
    """
    Read book types from book_asset.db to detect comics / manga vs novels.

    Comics and manga are identified by having >= 5 KINDLE_MAIN_ATTACHABLE assets
    (the individual page images that make up a comic/manga book).

    If book_asset.db is missing, inaccessible, or the query fails for any reason,
    the function returns an empty dict and detection is silently skipped - all
    books will then default to EPUB-only conversion.

    Args:
        content_dir: Kindle content directory (same path passed to Phase 1)

    Returns:
        dict  {asin: 'comic_manga'}  - only ASINs confirmed as comics are present.
              An empty dict means detection was unavailable or no comics found.
    """
    import sqlite3

    book_types = {}

    # Search for book_asset.db inside content_dir (it lives directly in the folder)
    db_path = None
    try:
        for root, dirs, files in os.walk(content_dir):
            for f in files:
                if f.lower() == 'book_asset.db':
                    db_path = os.path.join(root, f)
                    break
            if db_path:
                break
    except Exception:
        return book_types  # Cannot even walk the directory – return empty

    if not db_path:
        # book_asset.db not present (custom folder or older Kindle version)
        return book_types

    try:
        conn = sqlite3.connect(db_path, timeout=5)
        cursor = conn.cursor()

        # Count KINDLE_MAIN_ATTACHABLE assets per ASIN.
        # Comics/manga store every page as an individual attachable image;
        # novels have 0 or very few such assets.
        # Threshold of 5 safely separates comics from text-only books.
        cursor.execute("""
            SELECT b.asin,
                   SUM(CASE WHEN a.contentType = 'KINDLE_MAIN_ATTACHABLE' THEN 1 ELSE 0 END)
                       AS attachable_count
            FROM Book b
            LEFT JOIN Asset a ON a.bookId = b.id
            GROUP BY b.id
            HAVING attachable_count >= 5
        """)

        rows = cursor.fetchall()
        conn.close()

        for asin, _count in rows:
            if asin:
                book_types[asin] = 'comic_manga'

    except Exception:
        # DB locked, schema changed, or any other error – silently skip detection
        pass

    return book_types


def fetch_book_title_from_asin(asin):
    """Fetch book title from ASIN using fetch-ebook-metadata command"""
    try:
        result = subprocess.run(
            ['fetch-ebook-metadata', '-I', f'asin:{asin}'],
            capture_output=True,
            text=True,
            timeout=10,
            encoding='utf-8',
            errors='replace'
        )
        
        if result.returncode == 0 and result.stdout:
            for line in result.stdout.split('\n'):
                if line.strip().startswith('Title'):
                    parts = line.split(':', 1)
                    if len(parts) == 2:
                        title = parts[1].strip()
                        return title if title else asin
        
        return asin
        
    except Exception:
        return asin


def run_phase_1(extractor_path, content_dir, output_key, output_k4i, working_dir, config):
    """
    Run Phase 1: Key Extraction
    
    Args:
        extractor_path: Path to KFXKeyExtractor28.exe
        content_dir: Kindle content directory path
        output_key: Path for output kindlekey.txt
        output_k4i: Path for output kindlekey.k4i
        working_dir: Working directory for logs/history
        config: Configuration dict
    
    Returns:
        dict with keys:
            - success: bool
            - dsn: str or None
            - tokens: str or None
            - stats: dict with extraction statistics
    """
    from modules.utils import print_step, print_ok, print_warn, print_error
    
    script_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
    user_home = os.path.expanduser("~")
    
    # Find kindle.exe location
    kindle_dir, needs_temp_copy = find_kindle_exe()
    
    if not kindle_dir:
        raise FileNotFoundError("Kindle for PC installation not found.")
    
    temp_copy_created = False
    
    # Initialize extraction statistics
    extraction_stats = {
        'total': 0,
        'success': 0,
        'failed': 0,
        'skipped': 0,
        'failed_books': [],
        'skipped_books': []
    }
    
    # Load config settings
    fetch_titles = config.get('fetch_book_titles', False) if config else False
    enable_raw_logs = config.get('enable_raw_logs', False) if config else False
    
    # Initialize raw log path if enabled (matching original script logic)
    raw_log_path = None
    if enable_raw_logs:
        raw_log_path = get_raw_log_path(working_dir, 'extraction')
        extraction_logs_dir = os.path.join(working_dir, "Logs", "01_Extraction")
        cleanup_old_raw_logs(extraction_logs_dir, keep_count=10)
    
    try:
        # Create temporary copy if needed
        if needs_temp_copy:
            kindle_dir = create_temp_kindle_copy(kindle_dir)
            temp_copy_created = True
        
        # Detect comic / manga books from book_asset.db (best-effort, empty dict if unavailable)
        book_types = read_book_types_from_db(content_dir)
        if book_types:
            print_ok(f"Book type detection: {len(book_types)} comic/manga book(s) identified")
        # (no message on empty – silently skip when DB is absent)

        # Scan for individual book folders
        print_step("Scanning for book folders...")
        book_folders = scan_kindle_content_directory(content_dir)
        
        if not book_folders:
            print_warn("No book folders found in content directory")
            print_error("No books found in content directory!")
            return {
                'success': False,
                'dsn': None,
                'tokens': None,
                'book_types': {},
                'stats': extraction_stats
            }
        
        extraction_stats['total'] = len(book_folders)
        print_ok(f"Found {len(book_folders)} book folder(s)")
        print()
        
        # Load book history
        processed_asins = load_book_history(working_dir)
        
        # Check if any books were previously processed
        previously_processed = [asin for asin, _, _ in book_folders if asin in processed_asins]
        
        if previously_processed:
            print_step(f"Found {len(previously_processed)} previously processed book(s)")
            print("Options:")
            print("  [A] Process All - Re-process everything")
            print("  [N] Process New Only - Skip previously processed (recommended)")
            print("  [Q] Quit")
            print()
            
            while True:
                choice = input("Your choice (A/N/Q) [N]: ").strip().upper()
                if choice == '':
                    choice = 'N'
                
                if choice == 'N':
                    skipped = [(asin, title, "Previously processed") 
                              for asin, _, title in book_folders 
                              if asin in processed_asins]
                    extraction_stats['skipped_books'] = skipped
                    extraction_stats['skipped'] = len(skipped)
                    
                    book_folders = [(asin, path, title) for asin, path, title in book_folders if asin not in processed_asins]
                    print_ok(f"Processing {len(book_folders)} new book(s)")
                    print()
                    break
                elif choice == 'A':
                    print()
                    print_ok("Will process all books including previously processed")
                    # Clear the history set to allow re-extraction
                    processed_asins.clear()
                    print()
                    break
                elif choice == 'Q':
                    print()
                    return {
                        'success': False,
                        'dsn': None,
                        'tokens': None,
                        'book_types': book_types,
                        'stats': extraction_stats
                    }
                else:
                    print_error("Invalid choice. Please enter A, N, or Q.")
        
        if not book_folders:
            print_ok("No new books to process - all books have been previously processed")
            print()
            return {
                'success': True,
                'dsn': None,
                'tokens': None,
                'book_types': book_types,
                'stats': extraction_stats
            }
        
        # Process each book individually
        print_step(f"Extracting keys from {len(book_folders)} book(s)...")
        print()
        
        # Create temporary key files for per-book extraction
        temp_key = output_key + ".temp"
        temp_k4i = output_k4i + ".temp"
        
        dsn = None
        tokens = None
        
        for idx, (asin, book_folder, book_title) in enumerate(book_folders, 1):
            folder_name = os.path.basename(book_folder)
            
            # Initialize fetched_title (defaults to ASIN if not fetched)
            fetched_title = asin
            
            # Fetch book title if enabled in config
            if fetch_titles:
                fetched_title = fetch_book_title_from_asin(asin)
                book_prefix = f"[{idx}/{len(book_folders)}] {folder_name} - {fetched_title}..."
            else:
                book_prefix = f"[{idx}/{len(book_folders)}] {folder_name}..."
            
            print(f"{book_prefix}", end='', flush=True)
            
            success, book_dsn, book_tokens, error_msg, _ = extract_keys_from_single_book(
                extractor_path, kindle_dir, book_folder, temp_key, temp_k4i, asin, book_title,
                working_dir=working_dir, raw_log_path=raw_log_path, book_prefix=book_prefix
            )
            
            if success:
                print(" [OK] ✓")
                extraction_stats['success'] += 1
                
                if not dsn and book_dsn:
                    dsn = book_dsn
                if not tokens and book_tokens:
                    tokens = book_tokens
                
                # Append keys to main files
                append_success, append_error = append_keys_to_files(output_key, output_k4i, temp_key, temp_k4i)
                if not append_success:
                    print_warn(f" (Warning: Failed to append keys - {append_error})")
                
                # Update history
                append_to_history(working_dir, asin)
                
                # Cleanup temp files
                try:
                    if os.path.exists(temp_key):
                        os.remove(temp_key)
                    if os.path.exists(temp_k4i):
                        os.remove(temp_k4i)
                except Exception:
                    pass
            else:
                print(f" [ERROR] ✗ FAILED")
                extraction_stats['failed'] += 1
                
                # Determine book title for failed books
                if fetch_titles:
                    # Title was already fetched
                    failed_book_title = fetched_title
                else:
                    # Title not fetched - will be resolved by append_to_failed_books
                    failed_book_title = asin
                
                extraction_stats['failed_books'].append((asin, failed_book_title, error_msg if error_msg else "Unknown error"))
                
                # Track failed book in Failed-Books.txt
                append_to_failed_books(working_dir, failed_book_title, asin)
                
                try:
                    if os.path.exists(temp_key):
                        os.remove(temp_key)
                    if os.path.exists(temp_k4i):
                        os.remove(temp_k4i)
                except Exception:
                    pass
        
        print()
        
        # Cleanup temp_extraction folder after ALL books have been processed
        temp_extraction_dir = os.path.join(working_dir, "temp_extraction")
        if os.path.exists(temp_extraction_dir):
            print_step("Cleaning up temporary extraction folder...")
            success = rmtree_with_retry(temp_extraction_dir, max_retries=5, retry_delay=0.5)
            if success:
                print_ok("Temporary extraction folder cleaned up successfully")
            else:
                print_warn("Failed to clean up temporary extraction folder - you may need to delete it manually")
            print()
        
        # Cleanup extractor from Kindle folder
        extractor_in_kindle = os.path.join(kindle_dir, os.path.basename(extractor_path))
        if os.path.exists(extractor_in_kindle):
            os.remove(extractor_in_kindle)
        
        # Show raw log location if enabled
        if enable_raw_logs and raw_log_path:
            print_step(f"Raw debug log saved to:")
            print(f"      {raw_log_path}")
        
        print()
        
        overall_success = extraction_stats['success'] > 0
        
        return {
            'success': overall_success,
            'dsn': dsn,
            'tokens': tokens,
            'book_types': book_types,
            'stats': extraction_stats
        }

    except Exception as e:
        print_error(f"Extractor method failed: {e}")
        return {
            'success': False,
            'dsn': None,
            'tokens': None,
            'book_types': {},
            'stats': extraction_stats
        }
        
    finally:
        if temp_copy_created:
            cleanup_temp_kindle_copy(kindle_dir)
