#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""
Phase 4: KFX to EPUB Conversion
Converts imported books to EPUB format, with:
  - Alternate conversion method (calibre-debug / KFX Input plugin)
  - Early PDF-content detection and PDF extraction fallback
  - Phase 4B retry for timed-out / failed books
  - Optional bonus PDF generation for detected Comics / Manga
"""

import os
import sys
import subprocess
import threading
import time


def query_book_info_from_db(library_path, book_ids):
    """Query Calibre database for book info"""
    import sqlite3
    from modules.utils import print_error

    db_path = os.path.join(library_path, "metadata.db")
    book_info = {}

    try:
        conn = sqlite3.connect(db_path)
        cursor = conn.cursor()

        for book_id in book_ids:
            cursor.execute("SELECT title, path FROM books WHERE id = ?", (book_id,))
            result = cursor.fetchone()

            if result:
                title = result[0]
                path = result[1]

                cursor.execute("""
                    SELECT authors.name
                    FROM authors
                    JOIN books_authors_link ON authors.id = books_authors_link.author
                    WHERE books_authors_link.book = ?
                    LIMIT 1
                """, (book_id,))
                author_result = cursor.fetchone()
                author = author_result[0] if author_result else "Unknown Author"

                # Fetch ASIN from Calibre's identifiers table (stored by DeDRM or metadata reader)
                cursor.execute("""
                    SELECT val FROM identifiers
                    WHERE book = ? AND type IN ('amazon', 'asin', 'mobi-asin')
                    LIMIT 1
                """, (book_id,))
                asin_result = cursor.fetchone()
                asin = asin_result[0] if asin_result else None

                book_info[book_id] = {
                    'title': title,
                    'author': author,
                    'path': path,
                    'asin': asin   # None when Calibre did not store an identifier
                }

        conn.close()
        return book_info

    except Exception as e:
        print_error(f"Failed to query database: {e}")
        return {}


def find_source_file_in_directory(book_dir):
    """Find the source ebook file in the book directory"""
    from modules.utils import print_error

    try:
        if not os.path.exists(book_dir):
            return None, False

        for filename in os.listdir(book_dir):
            lower_name = filename.lower()
            if lower_name.endswith(('.kfx', '.azw', '.azw3', '.kfx-zip', '.mobi')):
                is_kfx_zip = lower_name.endswith('.kfx-zip')
                return filename, is_kfx_zip

        return None, False

    except Exception as e:
        print_error(f"Error searching directory {book_dir}: {e}")
        return None, False


# ---------------------------------------------------------------------------
# Standard EPUB conversion  (ebook-convert, 3-minute timeout)
# ---------------------------------------------------------------------------

def convert_book_to_epub(source_path, epub_path, timeout_seconds=180, raw_log_path=None, book_info=None):
    """
    Convert book to EPUB using ebook-convert.
    Returns: (success, error_message, error_type_dict)
    error_type keys: is_timeout, is_kfx_error, is_drm, is_pdf
    """
    from modules.utils import display_progress_timer, append_to_raw_log

    cmd = [
        'ebook-convert',
        source_path,
        epub_path,
        '--input-profile=default',
        '--output-profile=tablet',
        '--no-svg-cover',
        '--epub-version=3'
    ]

    try:
        process = subprocess.Popen(
            cmd,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            text=True,
            encoding='utf-8',
            errors='replace'
        )

        timer_stopped = threading.Event()
        timer_thread = threading.Thread(
            target=display_progress_timer,
            args=("Converting to EPUB", timeout_seconds, timer_stopped),
            daemon=True
        )
        timer_thread.start()

        try:
            stdout, stderr = process.communicate(timeout=timeout_seconds)
            result_returncode = process.returncode
            timer_stopped.set()
            timer_thread.join(timeout=1)
            print("\r" + " " * 80 + "\r", end='', flush=True)
        except subprocess.TimeoutExpired:
            timer_stopped.set()
            timer_thread.join(timeout=1)
            print("\r" + " " * 80 + "\r", end='', flush=True)
            process.kill()
            stdout, stderr = process.communicate()
            result_returncode = -1

        if raw_log_path:
            append_to_raw_log(raw_log_path, cmd, stdout, stderr, result_returncode, book_info=book_info)

        if result_returncode == 0 and os.path.exists(epub_path):
            return True, "", {'is_timeout': False, 'is_kfx_error': False, 'is_drm': False, 'is_pdf': False}

        if result_returncode == -1:
            return False, "Conversion timeout (3 minutes)", {'is_timeout': True, 'is_kfx_error': False, 'is_drm': False, 'is_pdf': False}

        combined = (stdout or "") + "\n" + (stderr or "")
        error_msg = (stderr or "Conversion failed").strip()

        # Detect specific error types from combined output
        is_pdf = any(x in combined for x in ["contains PDF content", "Convert to PDF", "PDF content"])
        is_kfx = any(x.lower() in combined.lower() for x in ["Amazon KFX book", "KFX book", "cannot be processed", "KFXError"])
        is_drm = any(x.lower() in combined.lower() for x in ["DRMError", "has DRM", "KFXDRMError"])

        if is_pdf:
            error_msg = "Book contains PDF content - cannot convert to EPUB"
        elif is_kfx:
            error_msg = "Amazon KFX book - requires KFX Input plugin"
        elif is_drm:
            error_msg = "Book has DRM and cannot be converted"

        return False, error_msg, {'is_timeout': False, 'is_kfx_error': is_kfx or is_drm, 'is_drm': is_drm, 'is_pdf': is_pdf}

    except Exception as e:
        return False, str(e), {'is_timeout': False, 'is_kfx_error': False, 'is_drm': False, 'is_pdf': False}


# ---------------------------------------------------------------------------
# Alternate EPUB conversion  (calibre-debug / KFX Input, 5-minute timeout)
# Includes real-time PDF-content detection and auto-switch to PDF extraction
# ---------------------------------------------------------------------------

def convert_book_alternate_method(source_path, epub_path, raw_log_path=None, book_info=None):
    """
    Alternate conversion using calibre-debug with KFX Input plugin.
    Streams output in real-time.  When "contains PDF content" is detected,
    the EPUB process is killed immediately and a PDF extraction is attempted.
    Returns: (success, error_message, error_type_dict)
    """
    from modules.utils import print_warn, print_colored, append_to_raw_log, display_progress_timer

    cmd = [
        'calibre-debug', '-r', 'KFX Input',
        '--',
        source_path,
        epub_path
    ]

    try:
        process = subprocess.Popen(
            cmd,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            text=True,
            encoding='utf-8',
            errors='replace',
            bufsize=1
        )

        stdout_lines = []
        stderr_lines = []
        pdf_detected = threading.Event()

        def _read_silently(pipe, lines_list):
            try:
                for line in iter(pipe.readline, ''):
                    if line:
                        lines_list.append(line)
                        if "contains PDF content" in line or "Use the --pdf option" in line:
                            pdf_detected.set()
                            try:
                                process.kill()
                            except Exception:
                                pass
                            return
            except Exception:
                pass

        t_out = threading.Thread(target=_read_silently, args=(process.stdout, stdout_lines), daemon=True)
        t_err = threading.Thread(target=_read_silently, args=(process.stderr, stderr_lines), daemon=True)
        t_out.start()
        t_err.start()

        timer_stopped = threading.Event()
        timeout_seconds = 300
        timer_thread = threading.Thread(
            target=display_progress_timer,
            args=("Converting (Alternate)", timeout_seconds, timer_stopped),
            daemon=True
        )
        timer_thread.start()

        # Poll until done / PDF detected / timed out
        start = time.time()
        timed_out = False
        while process.poll() is None:
            if pdf_detected.is_set():
                break
            if time.time() - start > timeout_seconds:
                timed_out = True
                process.kill()
                break
            time.sleep(0.1)

        timer_stopped.set()
        timer_thread.join(timeout=1)
        print("\r" + " " * 80 + "\r", end='', flush=True)

        t_out.join(timeout=2)
        t_err.join(timeout=2)

        if timed_out:
            if raw_log_path:
                append_to_raw_log(raw_log_path, cmd, ''.join(stdout_lines), ''.join(stderr_lines), -1, book_info=book_info)
            return False, "Alternate conversion timeout (5 minutes)", {
                'is_timeout': True, 'is_kfx_error': False, 'is_drm': False, 'is_pdf': False
            }

        # ---------------------------------------------------------------
        # PDF content detected – switch to PDF extraction
        # ---------------------------------------------------------------
        if pdf_detected.is_set():
            print()
            print_warn("  PDF CONTENT DETECTED - Cannot convert to EPUB")
            print_colored("  Switching to PDF extraction mode...", 'cyan')
            print()

            pdf_path = epub_path.replace('.epub', '.pdf')
            cmd_pdf = [
                'calibre-debug', '-r', 'KFX Input',
                '--', '-p',
                source_path,
                pdf_path
            ]

            try:
                proc_pdf = subprocess.Popen(
                    cmd_pdf,
                    stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                    text=True, encoding='utf-8', errors='replace', bufsize=1
                )

                pdf_out, pdf_err = [], []

                def _read_pdf(pipe, lst):
                    try:
                        for ln in iter(pipe.readline, ''):
                            if ln:
                                lst.append(ln)
                    except Exception:
                        pass

                tp1 = threading.Thread(target=_read_pdf, args=(proc_pdf.stdout, pdf_out), daemon=True)
                tp2 = threading.Thread(target=_read_pdf, args=(proc_pdf.stderr, pdf_err), daemon=True)
                tp1.start()
                tp2.start()

                pdf_timer = threading.Event()
                pdf_timer_thread = threading.Thread(
                    target=display_progress_timer,
                    args=("Extracting PDF", 300, pdf_timer),
                    daemon=True
                )
                pdf_timer_thread.start()

                try:
                    proc_pdf.wait(timeout=300)
                except subprocess.TimeoutExpired:
                    proc_pdf.kill()
                    pdf_timer.set()
                    pdf_timer_thread.join(timeout=1)
                    print("\r" + " " * 80 + "\r", end='', flush=True)
                    return False, "PDF extraction timeout (5 minutes)", {
                        'is_timeout': True, 'is_kfx_error': False, 'is_drm': False, 'is_pdf': True
                    }

                pdf_timer.set()
                pdf_timer_thread.join(timeout=1)
                print("\r" + " " * 80 + "\r", end='', flush=True)
                tp1.join(timeout=2)
                tp2.join(timeout=2)

                stdout_pdf_txt = ''.join(pdf_out)
                stderr_pdf_txt = ''.join(pdf_err)

                if raw_log_path:
                    append_to_raw_log(
                        raw_log_path, cmd_pdf,
                        stdout_pdf_txt, stderr_pdf_txt,
                        proc_pdf.returncode,
                        book_info=f"{book_info} (PDF extraction)"
                    )

                if proc_pdf.returncode == 0 and os.path.exists(pdf_path):
                    return True, "", {
                        'is_timeout': False, 'is_kfx_error': False, 'is_drm': False, 'is_pdf': True
                    }
                else:
                    return False, "Book contains PDF content - PDF extraction failed", {
                        'is_timeout': False, 'is_kfx_error': False, 'is_drm': False, 'is_pdf': True
                    }

            except Exception as exc:
                return False, f"PDF extraction failed: {exc}", {
                    'is_timeout': False, 'is_kfx_error': False, 'is_drm': False, 'is_pdf': True
                }

        # ---------------------------------------------------------------
        # Normal completion
        # ---------------------------------------------------------------
        stdout_txt = ''.join(stdout_lines)
        stderr_txt = ''.join(stderr_lines)
        combined = stdout_txt + "\n" + stderr_txt

        if raw_log_path:
            append_to_raw_log(raw_log_path, cmd, stdout_txt, stderr_txt, process.returncode, book_info=book_info)

        if process.returncode == 0 and os.path.exists(epub_path):
            return True, "", {'is_timeout': False, 'is_kfx_error': False, 'is_drm': False, 'is_pdf': False}

        error_msg = stderr_txt.strip() if stderr_txt.strip() else "Alternate conversion failed"
        is_kfx = any(x.lower() in combined.lower() for x in ["KFX", "cannot be processed", "KFXError"])
        is_drm = any(x.lower() in combined.lower() for x in ["DRMError", "has DRM", "KFXDRMError"])
        is_pdf = any(x in combined for x in ["contains PDF content", "PDF content"])
        return False, error_msg, {
            'is_timeout': False, 'is_kfx_error': is_kfx, 'is_drm': is_drm, 'is_pdf': is_pdf
        }

    except Exception as e:
        return False, str(e), {'is_timeout': False, 'is_kfx_error': False, 'is_drm': False, 'is_pdf': False}


# ---------------------------------------------------------------------------
# Comic / Manga bonus PDF generation
# ---------------------------------------------------------------------------

def convert_book_to_pdf(source_path, pdf_path, epub_fallback_path=None, raw_log_path=None, book_info=None):
    """
    Generate a PDF for a comic / manga / magazine book.

    Primary method  : calibre-debug -r "KFX Input" -- -p source.kfx pdf_path
                      Preferred for true comics/manga/magazines - preserves native page layout.
    Fallback method : ebook-convert epub_path pdf_path
                      Used automatically when the primary method fails.
                      Requires epub_fallback_path to point to the already-created EPUB.

    Both attempts are written to the RAW log (when enabled) with clear labels:
      (Comic PDF - KFX PRIMARY)
      (Comic PDF - EPUB FALLBACK)

    Returns: (success: bool, error_message: str)
    """
    from modules.utils import append_to_raw_log

    TIMEOUT = 300   # 5 minutes per attempt

    def _filter_syntax_warnings(text):
        """
        Strip Python SyntaxWarning lines emitted by third-party Calibre plugins
        (e.g. calibre_plugins.comicvine.utils) from stderr so they don't mask the
        real error when building failure reports.
        """
        filtered = [line for line in text.splitlines() if 'SyntaxWarning:' not in line]
        return '\n'.join(filtered).strip()

    # -----------------------------------------------------------------------
    # PRIMARY: calibre-debug -r "KFX Input" -- -p source.kfx pdf_path
    # -----------------------------------------------------------------------
    cmd = [
        'calibre-debug', '-r', 'KFX Input',
        '--', '-p',
        source_path,
        pdf_path
    ]

    primary_error = None   # Set on failure; triggers the fallback attempt.

    try:
        process = subprocess.Popen(
            cmd,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            text=True,
            encoding='utf-8',
            errors='replace',
            bufsize=1
        )

        stdout_lines = []
        stderr_lines = []

        def _drain(pipe, buf):
            try:
                for line in iter(pipe.readline, ''):
                    if line:
                        buf.append(line)
            except Exception:
                pass

        t_out = threading.Thread(target=_drain, args=(process.stdout, stdout_lines), daemon=True)
        t_err = threading.Thread(target=_drain, args=(process.stderr, stderr_lines), daemon=True)
        t_out.start()
        t_err.start()

        # Polling loop - print elapsed time on a NEW line every 5 seconds.
        # Using newlines (not \r) means calibre-debug's own console writes
        # cannot "reset" our cursor and make the timer invisible.
        start = time.time()
        last_print = start
        timed_out = False

        print(f"  [PDF] Generating... 00:00 / {TIMEOUT // 60:02d}:00", flush=True)

        while process.poll() is None:
            elapsed = time.time() - start
            if elapsed >= TIMEOUT:
                process.kill()
                timed_out = True
                break

            if time.time() - last_print >= 5:
                mins, secs = divmod(int(elapsed), 60)
                max_mins, max_secs = divmod(TIMEOUT, 60)
                print(f"  [PDF] Still generating... {mins:02d}:{secs:02d} / {max_mins:02d}:{max_secs:02d}", flush=True)
                last_print = time.time()

            time.sleep(0.25)

        t_out.join(timeout=2)
        t_err.join(timeout=2)

        stdout_txt = ''.join(stdout_lines)
        stderr_txt = ''.join(stderr_lines)

        if timed_out:
            if raw_log_path:
                append_to_raw_log(raw_log_path, cmd, stdout_txt, stderr_txt, -1,
                                  book_info=f"{book_info} (Comic PDF - KFX PRIMARY - TIMEOUT)")
            print(f"  [PDF] KFX->PDF timeout after {TIMEOUT // 60} minutes", flush=True)
            primary_error = f"KFX->PDF timeout ({TIMEOUT // 60} min)"
        elif process.returncode == 0 and os.path.exists(pdf_path):
            # Primary succeeded - no fallback needed.
            if raw_log_path:
                append_to_raw_log(raw_log_path, cmd, stdout_txt, stderr_txt,
                                  process.returncode, book_info=f"{book_info} (Comic PDF - KFX PRIMARY)")
            elapsed_total = int(time.time() - start)
            mins, secs = divmod(elapsed_total, 60)
            print(f"  [PDF] Done in {mins:02d}:{secs:02d}", flush=True)
            return True, ""
        else:
            # Primary failed - log it and fall through to EPUB fallback.
            if raw_log_path:
                append_to_raw_log(raw_log_path, cmd, stdout_txt, stderr_txt,
                                  process.returncode,
                                  book_info=f"{book_info} (Comic PDF - KFX PRIMARY - FAILED)")
            real_stderr = _filter_syntax_warnings(stderr_txt)
            primary_error = real_stderr if real_stderr else f"KFX->PDF failed (exit code {process.returncode})"

    except Exception as exc:
        if raw_log_path:
            append_to_raw_log(raw_log_path, cmd, '', str(exc), -1,
                              book_info=f"{book_info} (Comic PDF - KFX PRIMARY - EXCEPTION)")
        primary_error = f"KFX->PDF exception: {exc}"

    # -----------------------------------------------------------------------
    # FALLBACK: ebook-convert epub_path pdf_path
    # Same pipeline as the Calibre GUI "Convert to PDF" action, which the user
    # confirmed produces a valid PDF when run manually.
    # -----------------------------------------------------------------------
    print(f"  [PDF] KFX->PDF failed - attempting EPUB->PDF fallback...", flush=True)

    if not epub_fallback_path or not os.path.exists(epub_fallback_path):
        detail = f"KFX->PDF: {primary_error} | EPUB fallback: not available"
        if raw_log_path:
            append_to_raw_log(raw_log_path, [], '', detail, -1,
                              book_info=f"{book_info} (Comic PDF - EPUB FALLBACK - SKIPPED: no EPUB)")
        print("  [PDF] PDF generation failed.", flush=True)
        print("  [!] To get a PDF: in Calibre, right-click the book -> Convert books -> PDF.", flush=True)
        return False, ""

    cmd_fb = ['ebook-convert', epub_fallback_path, pdf_path]

    try:
        proc_fb = subprocess.Popen(
            cmd_fb,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            text=True,
            encoding='utf-8',
            errors='replace',
            bufsize=1
        )

        fb_out, fb_err = [], []

        def _drain_fb(pipe, buf):
            try:
                for line in iter(pipe.readline, ''):
                    if line:
                        buf.append(line)
            except Exception:
                pass

        t_fb_out = threading.Thread(target=_drain_fb, args=(proc_fb.stdout, fb_out), daemon=True)
        t_fb_err = threading.Thread(target=_drain_fb, args=(proc_fb.stderr, fb_err), daemon=True)
        t_fb_out.start()
        t_fb_err.start()

        start_fb = time.time()
        last_fb_print = start_fb
        timed_out_fb = False

        print(f"  [PDF] Fallback generating... 00:00 / {TIMEOUT // 60:02d}:00", flush=True)

        while proc_fb.poll() is None:
            elapsed_fb = time.time() - start_fb
            if elapsed_fb >= TIMEOUT:
                proc_fb.kill()
                timed_out_fb = True
                break

            if time.time() - last_fb_print >= 5:
                mins, secs = divmod(int(elapsed_fb), 60)
                print(f"  [PDF] Fallback still generating... {mins:02d}:{secs:02d} / {TIMEOUT // 60:02d}:00", flush=True)
                last_fb_print = time.time()

            time.sleep(0.25)

        t_fb_out.join(timeout=2)
        t_fb_err.join(timeout=2)

        fb_out_txt = ''.join(fb_out)
        fb_err_txt = ''.join(fb_err)

        if timed_out_fb:
            detail = f"KFX->PDF: {primary_error} | EPUB fallback: timeout after {TIMEOUT // 60} min"
            if raw_log_path:
                append_to_raw_log(raw_log_path, cmd_fb, fb_out_txt, fb_err_txt, -1,
                                  book_info=f"{book_info} (Comic PDF - EPUB FALLBACK - TIMEOUT)")
            print("  [PDF] PDF generation failed.", flush=True)
            print("  [!] To get a PDF: in Calibre, right-click the book -> Convert books -> PDF.", flush=True)
            return False, ""

        if raw_log_path:
            append_to_raw_log(raw_log_path, cmd_fb, fb_out_txt, fb_err_txt,
                              proc_fb.returncode,
                              book_info=f"{book_info} (Comic PDF - EPUB FALLBACK)")

        if proc_fb.returncode == 0 and os.path.exists(pdf_path):
            elapsed_fb_total = int(time.time() - start_fb)
            mins, secs = divmod(elapsed_fb_total, 60)
            print(f"  [PDF] Done via EPUB fallback in {mins:02d}:{secs:02d}", flush=True)
            return True, ""

        print("  [PDF] PDF generation failed.", flush=True)
        print("  [!] To get a PDF: in Calibre, right-click the book -> Convert books -> PDF.", flush=True)
        return False, ""

    except Exception as exc_fb:
        if raw_log_path:
            append_to_raw_log(raw_log_path, cmd_fb, '', str(exc_fb), -1,
                              book_info=f"{book_info} (Comic PDF - EPUB FALLBACK - EXCEPTION)")
        print("  [PDF] PDF generation failed.", flush=True)
        print("  [!] To get a PDF: in Calibre, right-click the book -> Convert books -> PDF.", flush=True)
        return False, ""


# ---------------------------------------------------------------------------
# Calibre format helpers
# ---------------------------------------------------------------------------

def add_epub_format_to_calibre(book_id, file_path, library_path):
    """Add EPUB (or PDF) format to existing Calibre book record"""
    cmd = [
        'calibredb', 'add_format',
        str(book_id),
        file_path,
        f'--library-path={library_path}'
    ]
    try:
        result = subprocess.run(cmd, capture_output=True, text=True, timeout=60)
        if result.returncode == 0:
            return True, ""
        return False, result.stderr or "Failed to add format"
    except subprocess.TimeoutExpired:
        return False, "Add format timeout"
    except Exception as e:
        return False, str(e)


def remove_format_from_calibre(book_id, format_name, library_path):
    """Remove a specific format from Calibre book record"""
    cmd = [
        'calibredb', 'remove_format',
        str(book_id),
        format_name.upper(),
        f'--library-path={library_path}'
    ]
    try:
        result = subprocess.run(cmd, capture_output=True, text=True, timeout=60)
        if result.returncode == 0:
            return True, ""
        return False, result.stderr or "Failed to remove format"
    except subprocess.TimeoutExpired:
        return False, "Remove format timeout"
    except Exception as e:
        return False, str(e)


# ---------------------------------------------------------------------------
# Phase 4B – retry helper
# ---------------------------------------------------------------------------

def process_phase_4b_retries(calibre_path, retry_books, book_info_map, working_dir=None):
    """
    Phase 4B: Retry timed-out / failed conversions using the alternate method.

    Args:
        calibre_path   : Calibre library path
        retry_books    : list of (book_id, title, author) that need a retry
        book_info_map  : dict {book_id: {'title', 'author', 'path'}} from Phase 4
        working_dir    : working directory for raw logs

    Returns: dict with keys total / converted / merged / failed / failed_conversions
    """
    from modules.utils import (
        print_step, print_ok, print_error, print_warn,
        get_raw_log_path, cleanup_old_raw_logs, write_raw_log_header
    )
    from modules.config import load_config

    config = load_config()
    enable_raw_logs = config.get('enable_raw_logs', False) if config else False
    raw_log_path = None
    if enable_raw_logs and working_dir:
        raw_log_path = get_raw_log_path(working_dir, 'conversion')
        write_raw_log_header(raw_log_path)   # reads Kindle version + mode from config
        logs_dir = os.path.join(working_dir, "Logs", "03_Conversion")
        cleanup_old_raw_logs(logs_dir, keep_count=10)

    stats = {
        'total': len(retry_books),
        'converted': 0,
        'merged': 0,
        'failed': 0,
        'failed_conversions': []
    }

    if not retry_books:
        return stats

    print_step(f"Retrying {len(retry_books)} failed book(s) with alternate method...")
    print()
    print_warn("Using calibre-debug with KFX Input plugin - 5-minute timeout per book")
    print()

    for idx, (book_id, title, author) in enumerate(retry_books, 1):
        print_step(f"Retry {idx}/{len(retry_books)}: '{title}' by {author}")

        if book_id not in book_info_map:
            print_error(f"  Book info not found for ID {book_id}")
            stats['failed'] += 1
            stats['failed_conversions'].append((book_id, title, author, "Book info not found"))
            print()
            continue

        info = book_info_map[book_id]
        book_dir = os.path.join(calibre_path, info['path'])

        source_filename, _ = find_source_file_in_directory(book_dir)
        if not source_filename:
            error_msg = "No source file found"
            print_error(f"  {error_msg}")
            stats['failed'] += 1
            stats['failed_conversions'].append((book_id, title, author, error_msg))
            print()
            continue

        source_path = os.path.join(book_dir, source_filename)
        epub_filename = os.path.splitext(source_filename)[0] + '.epub'
        epub_path = os.path.join(book_dir, epub_filename)
        book_info_str = f"{title} by {author} (ID: {book_id})"

        print(f"  Source: {source_filename}")
        print(f"  Target: {epub_filename}")

        success, error, error_type = convert_book_alternate_method(
            source_path, epub_path,
            raw_log_path=raw_log_path,
            book_info=book_info_str
        )

        if not success:
            print_error(f"  FAILED: {error}")
            stats['failed'] += 1
            stats['failed_conversions'].append((book_id, title, author, error))
            print()
            continue

        print_ok("  Conversion successful")
        stats['converted'] += 1

        # If the alternate method extracted a PDF instead of EPUB, merge the PDF
        if error_type.get('is_pdf', False):
            format_path = epub_path.replace('.epub', '.pdf')
            print("  Merging PDF format to Calibre...")
        else:
            format_path = epub_path
            print("  Merging EPUB format to Calibre...")

        merge_ok, merge_err = add_epub_format_to_calibre(book_id, format_path, calibre_path)
        if not merge_ok:
            print_error(f"  Merge failed: {merge_err}")
        else:
            print_ok("  Format merged successfully")
            stats['merged'] += 1

        print()

    # Summary
    print("--------------------------------------------------")
    print_step("Phase 4B Retry Summary:")
    print_ok(f"Retried:    {stats['total']} book(s)")
    print_ok(f"Converted:  {stats['converted']}")
    print_ok(f"Merged:     {stats['merged']}")
    if stats['failed'] > 0:
        print_error(f"Still failed: {stats['failed']}")
    print()

    return stats


# ---------------------------------------------------------------------------
# Fixed-layout detection
# ---------------------------------------------------------------------------

def is_fixed_layout_epub(epub_path):
    """
    Return True if the EPUB is a fixed-layout (pre-paginated) document.
    Fixed-layout EPUBs are comics, manga, and magazines.

    Detection uses two independent signals — either one triggers:

    Signal 1: OPF rendition:layout = pre-paginated  (EPUB 3 standard)
      Set by the KFX Input plugin before Calibre post-processes the EPUB.
      Reliable when the property survives Calibre's ebook-convert pipeline.

    Signal 2: Split-page spread naming convention  (-left.xhtml / -right.xhtml)
      When KFX Input converts a comic/manga KFX to EPUB it splits each page
      spread into a left and a right XHTML file.  Calibre's ebook-convert
      pipeline may strip the rendition:layout metadata but it always preserves
      the file names — so this pattern survives even when Signal 1 is lost.
      A minimum of 4 split-page items is required to avoid accidental matches.
      Reflowable text books never produce this naming pattern.

    Silently returns False on any error (missing file, bad zip, no OPF, etc.)
    """
    import zipfile
    import re
    try:
        with zipfile.ZipFile(epub_path, 'r') as z:
            # Locate the OPF via META-INF/container.xml
            with z.open('META-INF/container.xml') as f:
                container = f.read().decode('utf-8', errors='replace')
            match = re.search(r'full-path="([^"]+)"', container)
            if not match:
                return False
            opf_path = match.group(1)
            with z.open(opf_path) as f:
                opf = f.read().decode('utf-8', errors='replace')

        # Signal 1: EPUB 3 rendition:layout property (may be stripped by Calibre pipeline)
        if 'pre-paginated' in opf:
            return True

        # Signal 2: KFX Input split-page spread naming (-left.xhtml / -right.xhtml)
        # Require >= 4 items (i.e. at least 2 spreads) to avoid accidental matches.
        split_count = len(re.findall(r'href="[^"]*-(?:left|right)\.xhtml"', opf))
        if split_count >= 4:
            return True

        return False
    except Exception:
        return False


# ---------------------------------------------------------------------------
# Main Phase 4 orchestrator
# ---------------------------------------------------------------------------

def run_phase_4(calibre_path, book_ids, calibre_config=None, working_dir=None,
                generate_pdf_for_comics=False):
    """
    Run Phase 4: KFX to EPUB Conversion.

    Args:
        calibre_path          : Calibre library path
        book_ids              : list of Calibre book IDs from Phase 3
        calibre_config        : Calibre configuration dict (optional)
        working_dir           : working directory for logs (optional)
        generate_pdf_for_comics: when True, also generate PDF for fixed-layout
                                (comic/manga/magazine) books, detected via the
                                EPUB OPF rendition:layout = pre-paginated property

    Returns:
        dict with keys:
            success, converted, merged, failed, stats
            stats also contains:
                timeout_books  : list of (book_id, title, author) that timed out
                book_info_map  : {book_id: info} for use by Phase 4B
    """
    from modules.utils import (
        print_step, print_ok, print_error, print_warn,
        append_to_failed_books, get_raw_log_path, cleanup_old_raw_logs, write_raw_log_header
    )
    from modules.config import load_config

    source_file_management = 'keep_both'
    if calibre_config and isinstance(calibre_config, dict):
        source_file_management = calibre_config.get('source_file_management', 'keep_both')

    stats = {
        'total': len(book_ids),
        'converted': 0,
        'merged': 0,
        'failed': 0,
        'pdf_generated': 0,
        'errors': [],
        'failed_conversions': [],
        'source_files_deleted': 0,
        'timeout_books': [],       # (book_id, title, author) – for Phase 4B
        'book_info_map': {}        # populated below, shared with Phase 4B
    }

    if not book_ids:
        print_warn("No books to convert")
        return {'success': False, 'converted': 0, 'merged': 0, 'failed': 0, 'stats': stats}

    print_step(f"Converting {len(book_ids)} book(s) to EPUB format...")
    print()
    print_warn("NOTE: Each book has a 3-minute timeout for conversion")
    print()

    # Query database for book info
    print_step("Querying Calibre database for book information...")
    book_info = query_book_info_from_db(calibre_path, book_ids)

    if not book_info:
        print_error("Failed to retrieve book information from database")
        return {'success': False, 'converted': 0, 'merged': 0, 'failed': 0, 'stats': stats}

    stats['book_info_map'] = book_info
    print_ok(f"Retrieved information for {len(book_info)} book(s)")
    print()

    # Initialize RAW log
    config = load_config()
    enable_raw_logs = config.get('enable_raw_logs', False) if config else False
    raw_log_path = None
    if enable_raw_logs and working_dir:
        raw_log_path = get_raw_log_path(working_dir, 'conversion')
        write_raw_log_header(raw_log_path)   # reads Kindle version + mode from config
        logs_dir = os.path.join(working_dir, "Logs", "03_Conversion")
        cleanup_old_raw_logs(logs_dir, keep_count=10)

    # Process each book
    for idx, book_id in enumerate(book_ids, 1):
        if book_id not in book_info:
            continue

        info = book_info[book_id]
        title  = info['title']
        author = info['author']
        book_path = info['path']

        print_step(f"Processing book {idx}/{len(book_info)}: '{title}' by {author}")

        book_dir = os.path.join(calibre_path, book_path)
        source_filename, is_kfx_zip = find_source_file_in_directory(book_dir)

        if not source_filename:
            error_msg = f"No source file found in {book_path}"
            print_error(error_msg)
            stats['failed'] += 1
            stats['errors'].append(f"Book {book_id}: {error_msg}")
            stats['failed_conversions'].append((book_id, title, author, error_msg))
            if working_dir:
                append_to_failed_books(working_dir, title, str(book_id))
            print()
            continue

        source_path  = os.path.join(book_dir, source_filename)
        epub_filename = os.path.splitext(source_filename)[0] + '.epub'
        epub_path     = os.path.join(book_dir, epub_filename)

        print(f"  Source: {source_filename}")
        print(f"  Target: {epub_filename}")
        print("  Converting to EPUB...", end='', flush=True)

        book_info_str = f"{title} by {author} (ID: {book_id})"

        success, error, error_type = convert_book_to_epub(
            source_path, epub_path,
            timeout_seconds=180,
            raw_log_path=raw_log_path,
            book_info=book_info_str
        )

        if not success:
            if error_type.get('is_timeout'):
                print_error("  TIMEOUT")
                stats['timeout_books'].append((book_id, title, author))
            else:
                print_error("  FAILED")
            print_error(f"  {error}")
            stats['failed'] += 1
            stats['errors'].append(f"Book {book_id}: {error}")
            stats['failed_conversions'].append((book_id, title, author, error))
            if working_dir:
                append_to_failed_books(working_dir, title, str(book_id))
            print()
            continue

        print_ok("  Conversion successful")
        stats['converted'] += 1

        # Merge EPUB into Calibre
        print("  Merging EPUB format to Calibre...", end='', flush=True)
        merge_ok, merge_err = add_epub_format_to_calibre(book_id, epub_path, calibre_path)

        if not merge_ok:
            print_error(f"  {merge_err}")
        else:
            print_ok("  EPUB format merged successfully")
            stats['merged'] += 1

            # ----------------------------------------------------------
            # Optional: generate PDF for fixed-layout books.
            # Detection: read the EPUB OPF for rendition:layout=pre-paginated.
            # The KFX Input plugin sets this for all comics, manga, and
            # magazines — no database or ASIN lookup required.
            # ----------------------------------------------------------
            if generate_pdf_for_comics and is_fixed_layout_epub(epub_path):
                pdf_filename = os.path.splitext(source_filename)[0] + '.pdf'
                pdf_path = os.path.join(book_dir, pdf_filename)
                print("  Fixed-layout EPUB detected (comic/manga/magazine) - generating PDF...")

                pdf_ok, _ = convert_book_to_pdf(
                    source_path, pdf_path,
                    epub_fallback_path=epub_path,
                    raw_log_path=raw_log_path,
                    book_info=book_info_str
                )

                if pdf_ok:
                    pdf_merge_ok, pdf_merge_err = add_epub_format_to_calibre(
                        book_id, pdf_path, calibre_path
                    )
                    if pdf_merge_ok:
                        print_ok("  PDF format merged successfully")
                        stats['pdf_generated'] += 1
                    else:
                        print_warn(f"  PDF merge failed: {pdf_merge_err}")
                # Failure message and Calibre advice already printed by convert_book_to_pdf()

        # Source file management
        if source_file_management == 'delete_originals':
            source_ext = os.path.splitext(source_filename)[1].upper().replace('.', '')
            if source_ext in ['KFX', 'KFX-ZIP']:
                print(f"  Removing {source_ext} format from Calibre...", end='', flush=True)
                del_ok, del_err = remove_format_from_calibre(book_id, source_ext, calibre_path)
                if del_ok:
                    print_ok(f"  {source_ext} format removed")
                    stats['source_files_deleted'] += 1
                else:
                    print_warn(f"  Failed to remove {source_ext}: {del_err}")
            else:
                print(f"  Keeping {source_ext} format (only KFX/KFX-ZIP are deleted)")

        elif source_file_management == 'delete_kfx_zip_only' and is_kfx_zip:
            print("  Removing KFX-ZIP format from Calibre...", end='', flush=True)
            del_ok, del_err = remove_format_from_calibre(book_id, 'KFX-ZIP', calibre_path)
            if del_ok:
                print_ok("  KFX-ZIP format removed")
                stats['source_files_deleted'] += 1
            else:
                print_warn(f"  Failed to remove KFX-ZIP: {del_err}")

        print()

    # Summary
    print("--------------------------------------------------")
    print_step("Conversion Summary:")
    print()
    print_ok(f"Total books processed:          {stats['total']}")
    print_ok(f"Successfully converted:         {stats['converted']}")
    print_ok(f"Successfully merged to Calibre: {stats['merged']}")

    if stats.get('pdf_generated', 0) > 0:
        print_ok(f"Comic PDFs generated:           {stats['pdf_generated']}")

    if stats['failed'] > 0:
        print_error(f"Failed conversions:             {stats['failed']}")

    timeout_count = len(stats.get('timeout_books', []))
    if timeout_count > 0:
        print_warn(f"  Of which timed out:           {timeout_count} (will retry in Phase 4B)")

    print()

    return {
        'success': stats['converted'] > 0,
        'converted': stats['converted'],
        'merged': stats['merged'],
        'failed': stats['failed'],
        'stats': stats
    }
