"""
File Data Migration Script

Migrates VariableValue data for files (not projects) between SQL Server databases.
Maps DocumentIDs using the Documents table and Filename as the unique identifier.

Key differences from migrate.py:
- All files have ProjectID=2 in both databases
- Maps DocumentID instead of ProjectID
- Uses Filename from Documents table for mapping
- Filters for file-based records (DocumentID != 1)
"""

import json
import logging
import csv
import os
import glob
from datetime import datetime
from db_utils import DatabaseConnection


class FileDataMigration:
    """Handles migration of file-based VariableValue data between databases."""

    def __init__(self, config_file='config.json'):
        """Initialize migration with configuration file."""
        # Load configuration
        with open(config_file, 'r') as f:
            self.config = json.load(f)

        # Store database identifiers for progress tracking
        self.source_db_name = self.config['source_db']['database']
        self.target_db_name = self.config['target_db']['database']

        # Setup logging with timestamp
        self.timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
        log_filename = f'filedata_migration_{self.timestamp}.log'

        logging.basicConfig(
            level=logging.INFO,
            format='%(asctime)s - %(levelname)s - %(message)s',
            handlers=[
                logging.FileHandler(log_filename),
                logging.StreamHandler()
            ]
        )
        self.logger = logging.getLogger(__name__)

        # Initialize mappings
        self.variable_map = {}
        self.document_map = {}
        self.configuration_map = {}
        self.state_map = {}

        # Database connections
        self.source_conn = DatabaseConnection(self.config['source_db'])
        self.target_conn = DatabaseConnection(self.config['target_db'])

        self.logger.info("Database connections established.")

    def build_variable_mapping(self):
        """Build mapping of source VariableID to target VariableID based on variable names."""
        # Pulls all non-deleted, non-system variables from both the source and target databases,
        # then matches them by name. This produces a dict that translates source VariableIDs to
        # their corresponding target VariableIDs so migrated records point to the correct variable
        # in the new vault. Any source variables without a name match in the target are logged as unmapped.
        self.logger.info("Building variable ID mapping...")

        # Fetch variables from source (exclude deleted and system variables)
        source_variables = self.source_conn.execute_query(
            "SELECT VariableID, VariableName FROM Variable WHERE IsDeleted = 0 OR IsDeleted IS NULL"
        )

        # Fetch variables from target (exclude deleted and system variables)
        target_variables = self.target_conn.execute_query(
            "SELECT VariableID, VariableName FROM Variable WHERE IsDeleted = 0 OR IsDeleted IS NULL"
        )

        # Filter out system variables (names in curly brackets)
        source_user_vars = [v for v in source_variables if not (v['VariableName'].startswith('{') and v['VariableName'].endswith('}'))]
        target_user_vars = [v for v in target_variables if not (v['VariableName'].startswith('{') and v['VariableName'].endswith('}'))]

        self.logger.info(f"Found {len(source_user_vars)} user variables in source database")
        self.logger.info(f"Found {len(target_user_vars)} user variables in target database")

        # Create name-to-ID mapping for target
        target_var_map = {v['VariableName']: v['VariableID'] for v in target_user_vars}

        # Build source-to-target mapping
        mapped_count = 0
        unmapped_count = 0

        for source_var in source_user_vars:
            source_id = source_var['VariableID']
            var_name = source_var['VariableName']

            if var_name in target_var_map:
                target_id = target_var_map[var_name]
                self.variable_map[source_id] = target_id
                mapped_count += 1
            else:
                self.logger.warning(f"Variable '{var_name}' (ID: {source_id}) not found in target database")
                unmapped_count += 1

        self.logger.info(f"Variable mapping complete. Mapped {mapped_count} variables.")
        if unmapped_count > 0:
            self.logger.warning(f"Unmapped variables: {unmapped_count}")

    def transform_source_path(self, source_path):
        """Transform source path to expected target path by prepending root folder."""
        if not source_path:
            return None

        # Get target root folder from config
        target_root = self.config.get('path_mapping', {}).get('target_root_folder', 'Citadel')

        # Strip leading backslash if exists
        path_without_leading = source_path.lstrip('\\')

        # Prepend target root folder
        transformed = f'\\{target_root}\\{path_without_leading}'

        # Ensure trailing backslash if original had it
        if source_path.endswith('\\') and not transformed.endswith('\\'):
            transformed += '\\'

        return transformed

    def build_document_mapping(self):
        """Build mapping of source DocumentID to target DocumentID based on full vault path."""
        self.logger.info("Building document ID mapping using full vault path (folder path + filename)...")

        # Fetch documents with full paths from source
        # Join Documents -> DocumentsInProjects -> Projects to get folder path
        source_documents = self.source_conn.execute_query("""
            SELECT
                d.DocumentID,
                d.Filename,
                p.Path AS FolderPath,
                p.Path + d.Filename AS FullVaultPath
            FROM Documents d
            INNER JOIN DocumentsInProjects dp ON d.DocumentID = dp.DocumentID
            INNER JOIN Projects p ON dp.ProjectID = p.ProjectID
        """)

        # Fetch documents with full paths from target
        # Filter out deleted documents (Deleted=0) to avoid mapping to stale records
        # when a document has been added and deleted multiple times
        target_documents = self.target_conn.execute_query("""
            SELECT
                d.DocumentID,
                d.Filename,
                p.Path AS FolderPath,
                p.Path + d.Filename AS FullVaultPath
            FROM Documents d
            INNER JOIN DocumentsInProjects dp ON d.DocumentID = dp.DocumentID
            INNER JOIN Projects p ON dp.ProjectID = p.ProjectID
            WHERE d.Deleted = 0
        """)

        self.logger.info(f"Found {len(source_documents)} documents in source database")
        self.logger.info(f"Found {len(target_documents)} documents in target database")

        # Create full-path-to-ID mapping for target
        # Use case-insensitive comparison
        case_sensitive = self.config.get('path_mapping', {}).get('case_sensitive', False)

        target_doc_map = {}
        duplicate_count = 0
        for doc in target_documents:
            full_path = doc['FullVaultPath']
            if full_path:
                # Normalize path for case-insensitive matching if needed
                key = full_path if case_sensitive else full_path.lower()
                if key in target_doc_map:
                    duplicate_count += 1
                    # Keep the last occurrence
                target_doc_map[key] = doc

        self.logger.info(f"Built target document index with {len(target_doc_map)} unique paths")
        if duplicate_count > 0:
            self.logger.warning(f"Found {duplicate_count} duplicate paths in target (kept last occurrence)")

        # Build source-to-target mapping
        mapped_count = 0
        unmapped_count = 0
        null_path_count = 0

        for source_doc in source_documents:
            source_id = source_doc['DocumentID']
            filename = source_doc['Filename']
            folder_path = source_doc['FolderPath']
            source_full_path = source_doc['FullVaultPath']

            # Skip documents with no path
            if not source_full_path or not folder_path:
                self.logger.debug(f"Document '{filename}' (ID: {source_id}) has no path - skipping")
                null_path_count += 1
                continue

            # Transform source folder path to expected target folder path
            target_folder_path = self.transform_source_path(folder_path)

            if not target_folder_path:
                self.logger.debug(f"Document '{filename}' (ID: {source_id}) - path transformation failed")
                unmapped_count += 1
                continue

            # Construct expected target full path
            # Folder path already ends with '\', so just concatenate filename
            target_full_path = target_folder_path + filename

            # Look up in target using case-insensitive comparison
            key = target_full_path if case_sensitive else target_full_path.lower()

            if key in target_doc_map:
                target_doc = target_doc_map[key]
                target_id = target_doc['DocumentID']
                self.document_map[source_id] = target_id
                mapped_count += 1
            else:
                self.logger.debug(f"Document '{filename}' (ID: {source_id}) - Target path not found")
                self.logger.debug(f"  Source path: [{source_full_path}]")
                self.logger.debug(f"  Transformed to: [{target_full_path}]")
                unmapped_count += 1

        self.logger.info(f"Document mapping complete:")
        self.logger.info(f"  - Successfully mapped: {mapped_count} documents")
        self.logger.info(f"  - Unmapped (path not found): {unmapped_count} documents")
        self.logger.info(f"  - Skipped (null path): {null_path_count} documents")
        self.logger.info(f"  - Total in document_map: {len(self.document_map)} documents")

    def build_configuration_mapping(self):
        """Build mapping of source ConfigurationID to target ConfigurationID based on ConfigurationName."""
        """ NOTE: find the manual overrides by searching for all the instances where ConfigurationName has dupe in the DB"""
        self.logger.info("Building configuration ID mapping...")

        # Fetch configurations from source
        source_configs = self.source_conn.execute_query(
            "SELECT ConfigurationID, ConfigurationName FROM DocumentConfiguration"
        )

        # Fetch configurations from target
        target_configs = self.target_conn.execute_query(
            "SELECT ConfigurationID, ConfigurationName FROM DocumentConfiguration"
        )

        self.logger.info(f"Found {len(source_configs)} configurations in source database")
        self.logger.info(f"Found {len(target_configs)} configurations in target database")

        # Create ConfigurationName-to-ID mapping for target
        target_config_map = {}
        target_duplicates = {}

        for config in target_configs:
            config_name = config['ConfigurationName']
            config_id = config['ConfigurationID']

            if config_name in target_config_map:
                # Track duplicates
                if config_name not in target_duplicates:
                    target_duplicates[config_name] = [target_config_map[config_name]]
                target_duplicates[config_name].append(config_id)
            else:
                target_config_map[config_name] = config_id

        if target_duplicates:
            self.logger.warning(f"Found {len(target_duplicates)} duplicate ConfigurationNames in target:")
            for name, ids in target_duplicates.items():
                self.logger.warning(f"  '{name}': IDs {ids}")

        # Load manual overrides from config file
        manual_overrides = self.config.get('configuration_mapping_overrides', {})
        if manual_overrides:
            self.logger.warning("=" * 70)
            self.logger.warning("MANUAL CONFIGURATION MAPPING OVERRIDES DETECTED!")
            self.logger.warning(f"Found {len(manual_overrides)} manual configuration mapping overrides in config.json")
            self.logger.warning("=" * 70)
            self.logger.warning("Please verify these mappings are correct before proceeding:")

            # Convert string keys to int (JSON keys are always strings)
            manual_overrides = {int(k): int(v) for k, v in manual_overrides.items()}

            # Display the manual overrides
            for source_id, target_id in sorted(manual_overrides.items()):
                self.logger.warning(f"  Source ConfigurationID {source_id} -> Target ConfigurationID {target_id}")

            self.logger.warning("=" * 70)

            # Prompt user for confirmation
            try:
                response = input("\nHave you verified these configuration mappings are correct? (yes/no): ").strip().lower()
                if response != 'yes':
                    self.logger.error("Migration cancelled by user - please verify configuration mappings in config.json")
                    raise ValueError("User cancelled migration - configuration mappings not verified")
            except EOFError:
                # Non-interactive mode - log warning but continue
                self.logger.warning("Running in non-interactive mode - cannot prompt for confirmation")
                self.logger.warning("PROCEEDING WITH MANUAL CONFIGURATION OVERRIDES - ENSURE THESE ARE CORRECT!")

            self.logger.info(f"Proceeding with {len(manual_overrides)} manual configuration mapping overrides")

        # Build source-to-target ID mapping
        mapped_count = 0
        unmapped_count = 0
        override_count = 0

        for source_config in source_configs:
            source_id = source_config['ConfigurationID']
            config_name = source_config['ConfigurationName']

            # Check if there's a manual override first
            if source_id in manual_overrides:
                target_id = manual_overrides[source_id]
                self.configuration_map[source_id] = target_id
                self.logger.debug(f"Manual override: ConfigurationID {source_id} -> {target_id} ('{config_name}')")
                override_count += 1
                mapped_count += 1
            elif config_name in target_config_map:
                target_id = target_config_map[config_name]
                self.configuration_map[source_id] = target_id
                self.logger.debug(f"Mapped Configuration '{config_name}': {source_id} -> {target_id}")
                mapped_count += 1
            else:
                self.logger.warning(f"Configuration '{config_name}' (ID: {source_id}) not found in target database")
                unmapped_count += 1

        self.logger.info(f"Configuration mapping complete:")
        self.logger.info(f"  - Successfully mapped: {mapped_count} configurations")
        self.logger.info(f"  - Manual overrides applied: {override_count} configurations")
        self.logger.info(f"  - Unmapped: {unmapped_count} configurations")
        self.logger.info(f"  - Total in configuration_map: {len(self.configuration_map)} configurations")

    def build_state_mapping(self):
        """Build mapping of source CurrentStatusID to target CurrentStatusID based on explicit config.json mappings."""
        self.logger.info("Building state (CurrentStatusID) mapping...")

        # Load manual state mappings from config file
        # Unlike configuration mapping, state mapping ONLY uses explicit mappings from config
        # If no mapping exists, the target DB value is left unchanged
        manual_mappings = self.config.get('state_mapping_overrides', {})

        if not manual_mappings:
            self.logger.info("No state mapping overrides found in config.json")
            self.logger.info("CurrentStatusID values will remain unchanged in target database")
            return

        self.logger.warning("=" * 70)
        self.logger.warning("MANUAL STATE (CurrentStatusID) MAPPING OVERRIDES DETECTED!")
        self.logger.warning(f"Found {len(manual_mappings)} manual state mapping overrides in config.json")
        self.logger.warning("=" * 70)
        self.logger.warning("Please verify these mappings are correct before proceeding:")

        # Convert string keys to int (JSON keys are always strings)
        manual_mappings = {int(k): int(v) for k, v in manual_mappings.items()}

        # Display the manual mappings
        for source_id, target_id in sorted(manual_mappings.items()):
            self.logger.warning(f"  Source CurrentStatusID {source_id} -> Target CurrentStatusID {target_id}")

        self.logger.warning("=" * 70)

        # Prompt user for confirmation
        try:
            response = input("\nHave you verified these state mappings are correct? (yes/no): ").strip().lower()
            if response != 'yes':
                self.logger.error("Migration cancelled by user - please verify state mappings in config.json")
                raise ValueError("User cancelled migration - state mappings not verified")
        except EOFError:
            # Non-interactive mode - log warning but continue
            self.logger.warning("Running in non-interactive mode - cannot prompt for confirmation")
            self.logger.warning("PROCEEDING WITH MANUAL STATE OVERRIDES - ENSURE THESE ARE CORRECT!")

        # Store the mappings
        self.state_map = manual_mappings

        self.logger.info(f"State mapping complete:")
        self.logger.info(f"  - Total explicit mappings loaded: {len(self.state_map)} state mappings")
        self.logger.info("  - Any CurrentStatusID not in this mapping will remain unchanged in target")

    def export_mappings_to_csv(self):
        """Export variable and document mappings to CSV files for verification."""
        self.logger.info("Exporting mappings to CSV files...")

        # Export variable mappings
        var_csv_filename = f'mapping_variables_filedata_{self.timestamp}.csv'
        with open(var_csv_filename, 'w', newline='', encoding='utf-8') as csvfile:
            writer = csv.writer(csvfile)
            writer.writerow(['VariableName', 'SourceID', 'TargetID'])

            # Get variable names for the mapping
            source_vars = self.source_conn.execute_query(
                "SELECT VariableID, VariableName FROM Variable"
            )
            source_var_names = {v['VariableID']: v['VariableName'] for v in source_vars}

            for source_id, target_id in self.variable_map.items():
                var_name = source_var_names.get(source_id, 'Unknown')
                writer.writerow([var_name, source_id, target_id])

        self.logger.info(f"Variable mappings exported to: {var_csv_filename} ({len(self.variable_map)} mappings)")

        # Export document mappings
        doc_csv_filename = f'mapping_documents_filedata_{self.timestamp}.csv'
        with open(doc_csv_filename, 'w', newline='', encoding='utf-8') as csvfile:
            writer = csv.writer(csvfile)
            writer.writerow(['Filename', 'SourceDocID', 'TargetDocID', 'SourcePath', 'TargetPath'])

            # Get filenames and full paths for the mapping
            source_docs = self.source_conn.execute_query("""
                SELECT
                    d.DocumentID,
                    d.Filename,
                    p.Path + d.Filename AS FullVaultPath
                FROM Documents d
                INNER JOIN DocumentsInProjects dp ON d.DocumentID = dp.DocumentID
                INNER JOIN Projects p ON dp.ProjectID = p.ProjectID
            """)
            source_doc_info = {d['DocumentID']: {'Filename': d['Filename'], 'FullVaultPath': d['FullVaultPath']} for d in source_docs}

            target_docs = self.target_conn.execute_query("""
                SELECT
                    d.DocumentID,
                    d.Filename,
                    p.Path + d.Filename AS FullVaultPath
                FROM Documents d
                INNER JOIN DocumentsInProjects dp ON d.DocumentID = dp.DocumentID
                INNER JOIN Projects p ON dp.ProjectID = p.ProjectID
            """)
            target_doc_info = {d['DocumentID']: {'Filename': d['Filename'], 'FullVaultPath': d['FullVaultPath']} for d in target_docs}

            for source_id, target_id in self.document_map.items():
                source_info = source_doc_info.get(source_id, {})
                target_info = target_doc_info.get(target_id, {})
                filename = source_info.get('Filename', 'Unknown')
                source_path = source_info.get('FullVaultPath', 'Unknown')
                target_path = target_info.get('FullVaultPath', 'Unknown')
                writer.writerow([filename, source_id, target_id, source_path, target_path])

        self.logger.info(f"Document mappings exported to: {doc_csv_filename} ({len(self.document_map)} mappings)")

    def save_migration_progress(self, batch_num, total_batches, stats):
        """Save migration progress to resume on failure."""
        progress_file = f"filedata_migration_progress_{self.source_db_name}_to_{self.target_db_name}_{self.timestamp}.json"

        progress_data = {
            'timestamp': datetime.now().isoformat(),
            'migration_timestamp': self.timestamp,
            'source_database': self.source_db_name,
            'target_database': self.target_db_name,
            'last_completed_batch': batch_num,
            'total_batches': total_batches,
            'records_inserted': stats['inserted'],
            'records_updated': stats['updated'],
            'records_errors': stats['errors']
        }

        with open(progress_file, 'w') as f:
            json.dump(progress_data, f, indent=2)

        self.logger.info(f"Progress saved: batch {batch_num}/{total_batches}")

    def load_migration_progress(self):
        """Load previous migration progress if exists for current database pair."""
        progress_pattern = f"filedata_migration_progress_{self.source_db_name}_to_{self.target_db_name}_*.json"
        progress_files = glob.glob(progress_pattern)

        if not progress_files:
            self.logger.info("No previous file data migration progress found for this database pair")
            return None

        latest_progress = max(progress_files, key=os.path.getmtime)

        with open(latest_progress, 'r') as f:
            progress = json.load(f)

        if (progress.get('source_database') != self.source_db_name or
            progress.get('target_database') != self.target_db_name):
            self.logger.warning(f"Found progress file but database names don't match. Ignoring.")
            return None

        self.logger.info(f"Found previous file data migration progress: {latest_progress}")
        self.logger.info(f"  Source DB: {progress['source_database']} -> Target DB: {progress['target_database']}")
        self.logger.info(f"  Last completed batch: {progress['last_completed_batch']}/{progress['total_batches']}")
        self.logger.info(f"  Records inserted: {progress['records_inserted']}")
        self.logger.info(f"  Records updated: {progress['records_updated']}")

        return progress

    def cleanup_progress_file(self):
        """Remove progress file after successful completion for this database pair."""
        progress_pattern = f"filedata_migration_progress_{self.source_db_name}_to_{self.target_db_name}_*.json"
        progress_files = glob.glob(progress_pattern)

        for pf in progress_files:
            os.remove(pf)
            self.logger.info(f"Removed progress file: {pf}")

    def preview_migration(self, num_records=20):
        """
        Preview what the first batch of VariableValue migrations would look like.
        Shows the actual SQL parameters that will be inserted/updated.
        """
        self.logger.info("="*70)
        self.logger.info(f"MIGRATION PREVIEW - First {num_records} Records")
        self.logger.info("="*70)
        self.logger.info("Showing actual SQL MERGE parameters that will be executed")
        self.logger.info("="*70)

        # Fetch source VariableValue records (same filter as actual migration)
        source_values = self.source_conn.execute_query(
            f"""SELECT TOP {num_records} * FROM VariableValue
               WHERE ProjectID = 2 AND DocumentID != 1
               ORDER BY DocumentID, VariableID, RevisionNo"""
        )

        if not source_values:
            self.logger.info("No source records found to preview.")
            return

        self.logger.info(f"\nFound {len(source_values)} source records to preview\n")

        # Preview each record
        mapped_count = 0
        skipped_count = 0

        for i, record in enumerate(source_values, 1):
            # Extract source IDs
            source_variable_id = record.get('VariableID')
            source_document_id = record.get('DocumentID')

            # Map to target IDs
            target_variable_id = self.variable_map.get(source_variable_id)
            target_document_id = self.document_map.get(source_document_id)

            self.logger.info(f"Record #{i}:")
            self.logger.info(f"  SOURCE VariableValue Row:")
            self.logger.info(f"    VariableID    : {record.get('VariableID')}")
            self.logger.info(f"    DocumentID    : {record.get('DocumentID')}")
            self.logger.info(f"    ProjectID     : {record.get('ProjectID')}")
            self.logger.info(f"    RevisionNo    : {record.get('RevisionNo')}")
            self.logger.info(f"    ConfigurationID: {record.get('ConfigurationID')}")
            self.logger.info(f"    ValueText     : {record.get('ValueText')}")
            self.logger.info(f"    ValueInt      : {record.get('ValueInt')}")
            self.logger.info(f"    ValueFloat    : {record.get('ValueFloat')}")
            self.logger.info(f"    ValueDate     : {record.get('ValueDate')}")
            self.logger.info(f"    ValueCache    : {record.get('ValueCache')}")
            self.logger.info(f"    IsLongText    : {record.get('IsLongText')}")

            # Check if mapping exists
            if target_variable_id is None or target_document_id is None:
                self.logger.warning(f"  TARGET: WILL BE SKIPPED (unmapped)")
                if target_variable_id is None:
                    self.logger.warning(f"    - Source VariableID {source_variable_id} not mapped to target")
                if target_document_id is None:
                    self.logger.warning(f"    - Source DocumentID {source_document_id} not mapped to target")
                skipped_count += 1
            else:
                # Prepare params exactly as migration would (same logic as migrate_file_variable_values)
                params = (
                    target_variable_id,              # VariableID
                    target_document_id,              # DocumentID
                    2,                               # ProjectID (always 2 for files)
                    record.get('RevisionNo'),        # RevisionNo
                    record.get('ConfigurationID'),   # ConfigurationID
                    record.get('ValueText'),         # ValueText
                    record.get('ValueInt'),          # ValueInt
                    record.get('ValueFloat'),        # ValueFloat
                    record.get('ValueDate'),         # ValueDate
                    record.get('ValueCache'),        # ValueCache
                    record.get('IsLongText')         # IsLongText
                )

                self.logger.info(f"  TARGET MERGE Parameters (will be UPSERTED):")
                self.logger.info(f"    VariableID    : {params[0]} (mapped from source {source_variable_id})")
                self.logger.info(f"    DocumentID    : {params[1]} (mapped from source {source_document_id})")
                self.logger.info(f"    ProjectID     : {params[2]} (always 2 for files)")
                self.logger.info(f"    RevisionNo    : {params[3]}")
                self.logger.info(f"    ConfigurationID: {params[4]}")
                self.logger.info(f"    ValueText     : {params[5]}")
                self.logger.info(f"    ValueInt      : {params[6]}")
                self.logger.info(f"    ValueFloat    : {params[7]}")
                self.logger.info(f"    ValueDate     : {params[8]}")
                self.logger.info(f"    ValueCache    : {params[9]}")
                self.logger.info(f"    IsLongText    : {params[10]}")
                mapped_count += 1

            self.logger.info("")

        self.logger.info("="*70)
        self.logger.info(f"PREVIEW SUMMARY")
        self.logger.info("="*70)
        self.logger.info(f"Records that WILL be migrated: {mapped_count}")
        self.logger.info(f"Records that will be SKIPPED: {skipped_count}")
        self.logger.info("")

    def migrate_file_variable_values(self):
        """
        Migrate VariableValue records for files (ProjectID=2, DocumentID != 1).
        Uses UPSERT mode with periodic commits and resume capability.
        """
        self.logger.info("Starting file-based VariableValue migration (UPSERT mode)...")

        # Get batch settings
        batch_size = self.config.get('migration', {}).get('batch_size', 500)
        commit_interval = self.config.get('migration', {}).get('commit_interval', 10)

        self.logger.info(f"Migration settings: batch_size={batch_size}, mode=UPSERT (insert new, update existing)")

        # Fetch ONLY the latest revision for each VariableID+DocumentID+ConfigurationID combination
        # Filter: ProjectID=2 (files) and DocumentID != 1 (not project-level variables)
        # Uses window function to get only the highest RevisionNo for each variable per document per configuration
        source_values = self.source_conn.execute_query(
            """SELECT * FROM (
                   SELECT *,
                          ROW_NUMBER() OVER (PARTITION BY DocumentID, VariableID, ConfigurationID
                                            ORDER BY RevisionNo DESC) as rn
                   FROM VariableValue
                   WHERE ProjectID = 2 AND DocumentID != 1
               ) ranked
               WHERE rn = 1
               ORDER BY DocumentID, VariableID, ConfigurationID"""
        )

        self.logger.info(f"Found {len(source_values)} latest-revision VariableValue records in source (ProjectID=2, DocumentID!=1).")

        # Prepare MERGE query for UPSERT operation
        # Match on all primary key columns: VariableID + DocumentID + ProjectID + RevisionNo + ConfigurationID
        merge_query = """
            MERGE INTO VariableValue AS target
            USING (SELECT ? AS VariableID, ? AS DocumentID, ? AS ProjectID, ? AS RevisionNo,
                          ? AS ConfigurationID, ? AS ValueText, ? AS ValueInt, ? AS ValueFloat,
                          ? AS ValueDate, ? AS ValueCache, ? AS IsLongText) AS source
            ON (target.VariableID = source.VariableID
                AND target.DocumentID = source.DocumentID
                AND target.ProjectID = source.ProjectID
                AND target.RevisionNo = source.RevisionNo
                AND target.ConfigurationID = source.ConfigurationID)
            WHEN MATCHED THEN
                UPDATE SET
                    ValueText = source.ValueText,
                    ValueInt = source.ValueInt,
                    ValueFloat = source.ValueFloat,
                    ValueDate = source.ValueDate,
                    ValueCache = source.ValueCache,
                    IsLongText = source.IsLongText
            WHEN NOT MATCHED THEN
                INSERT (VariableID, DocumentID, ProjectID, RevisionNo, ConfigurationID, ValueText, ValueInt, ValueFloat, ValueDate, ValueCache, IsLongText)
                VALUES (source.VariableID, source.DocumentID, source.ProjectID, source.RevisionNo, source.ConfigurationID, source.ValueText, source.ValueInt, source.ValueFloat, source.ValueDate, source.ValueCache, source.IsLongText);
        """

        # Collect records to upsert
        params_list = []
        skipped_unmapped = 0

        for record in source_values:
            # Extract source IDs
            source_variable_id = record.get('VariableID')
            source_document_id = record.get('DocumentID')

            # Map to target IDs
            target_variable_id = self.variable_map.get(source_variable_id)
            target_document_id = self.document_map.get(source_document_id)

            # Skip if mapping not found
            if target_variable_id is None:
                self.logger.debug(f"Skipping record: VariableID {source_variable_id} not mapped")
                skipped_unmapped += 1
                continue

            if target_document_id is None:
                self.logger.debug(f"Skipping record: DocumentID {source_document_id} not mapped")
                skipped_unmapped += 1
                continue

            # Extract other fields
            source_rev = record.get('RevisionNo')  # Read but not used - we always insert as Rev 1
            source_configID = record.get('ConfigurationID')
            source_valueText = record.get('ValueText')
            source_valueInt = record.get('ValueInt')
            source_valueFloat = record.get('ValueFloat')
            source_valueDate = record.get('ValueDate')
            source_valueCache = record.get('ValueCache')
            source_islongtext = record.get('IsLongText')

            # Map ConfigurationID to target
            target_config_id = self.configuration_map.get(source_configID)

            # Skip if configuration mapping not found
            if target_config_id is None:
                self.logger.debug(f"Skipping record: ConfigurationID {source_configID} not mapped")
                skipped_unmapped += 1
                continue

            # Prepare params for MERGE statement
            # ProjectID is always 2 for files in target database
            # RevisionNo is always 1 (we only migrate latest revision as new Rev 1)
            params = (
                target_variable_id,   # VariableID
                target_document_id,   # DocumentID
                2,                    # ProjectID (always 2 for files)
                1,                    # RevisionNo (always 1 for migrated data)
                target_config_id,     # ConfigurationID (mapped)
                source_valueText,     # ValueText
                source_valueInt,      # ValueInt
                source_valueFloat,    # ValueFloat
                source_valueDate,     # ValueDate
                source_valueCache,    # ValueCache
                source_islongtext     # IsLongText
            )

            params_list.append(params)

        self.logger.info(f"Prepared {len(params_list)} records for UPSERT")
        self.logger.info(f"Skipped {skipped_unmapped} records (unmapped IDs)")

        # Check for previous progress
        previous_progress = self.load_migration_progress()

        # Calculate batches
        total_batches = (len(params_list) + batch_size - 1) // batch_size

        # Initialize statistics
        total_stats = {'inserted': 0, 'updated': 0, 'errors': 0}
        start_batch = 0

        # Handle resume
        if previous_progress:
            self.logger.info(f"*** Previous file data migration found ***")
            self.logger.info(f"Last completed batch: {previous_progress['last_completed_batch']}/{previous_progress['total_batches']}")
            self.logger.info(f"Records inserted: {previous_progress['records_inserted']}")
            self.logger.info(f"Records updated: {previous_progress['records_updated']}")

            try:
                print(f"\n*** Previous file data migration found ***")
                print(f"Last completed batch: {previous_progress['last_completed_batch']}/{previous_progress['total_batches']}")
                print(f"Records inserted: {previous_progress['records_inserted']}")
                print(f"Records updated: {previous_progress['records_updated']}")
                response = input(f"Resume from batch {previous_progress['last_completed_batch'] + 1}? (y/n): ").strip().lower()
            except EOFError:
                response = 'y'
                self.logger.info("Running in non-interactive mode - automatically resuming")

            if response == 'y':
                start_batch = previous_progress['last_completed_batch']
                total_stats['inserted'] = previous_progress['records_inserted']
                total_stats['updated'] = previous_progress['records_updated']
                total_stats['errors'] = previous_progress['records_errors']
                self.logger.info(f"Resuming from batch {start_batch + 1}")
            else:
                self.logger.info("Starting fresh migration (previous progress will be overwritten)")
                self.cleanup_progress_file()

        self.logger.info(f"Commit interval: every {commit_interval} batches")

        # Process in batches
        for batch_num in range(start_batch, total_batches):
            batch_start = batch_num * batch_size
            batch_end = min(batch_start + batch_size, len(params_list))
            batch_params = params_list[batch_start:batch_end]

            self.logger.info(f"Processing batch {batch_num + 1}/{total_batches} ({len(batch_params)} records)...")

            # Execute MERGE for this batch
            for params in batch_params:
                try:
                    cursor = self.target_conn.connection.cursor()
                    cursor.execute(merge_query, params)

                    # Check if record was inserted or updated
                    # (This is approximate - SQL Server doesn't easily report MERGE action)
                    if cursor.rowcount > 0:
                        total_stats['inserted'] += 1
                    else:
                        total_stats['updated'] += 1

                    cursor.close()

                except Exception as e:
                    total_stats['errors'] += 1
                    self.logger.error(f"Error upserting record: {e}")

            self.logger.info(f"Batch {batch_num + 1} complete: inserted={total_stats['inserted']}, updated={total_stats['updated']}, errors={total_stats['errors']}")

            # Commit every N batches to keep connection alive and save progress
            if (batch_num + 1) % commit_interval == 0:
                self.target_conn.commit()
                self.save_migration_progress(batch_num + 1, total_batches, total_stats)
                self.logger.info(f"[COMMIT] Transaction committed at batch {batch_num + 1} (every {commit_interval} batches)")

        # Final commit
        self.target_conn.commit()
        self.logger.info(f"[SUCCESS] Final transaction committed")

        # Clean up progress file on success
        self.cleanup_progress_file()

        self.logger.info("File-based VariableValue migration completed successfully!")
        self.logger.info(f"Total records inserted: {total_stats['inserted']}")
        self.logger.info(f"Total records updated: {total_stats['updated']}")
        self.logger.info(f"Total errors: {total_stats['errors']}")

    def validate_migration(self):
        # After the migration runs we will run this script to validate that everything was inserted correctly into the Target DB
        self.logger.info("Running Validation on the Target to ensure we completed migration successfully")

        # the primary key columns are VariableID, DocumentID, ProjectID, RevisionNo, ConfigurationID
        # IMPORTANT: We only migrated the LATEST revision from source, so only validate those
        source_values = self.source_conn.execute_query(
            """SELECT * FROM (
                   SELECT VariableID, DocumentID, ProjectID, RevisionNo, ConfigurationID,
                          ROW_NUMBER() OVER (PARTITION BY DocumentID, VariableID, ConfigurationID
                                            ORDER BY RevisionNo DESC) as rn
                   FROM VariableValue
                   WHERE ProjectID = 2 AND DocumentID != 1
               ) ranked
               WHERE rn = 1
               ORDER BY DocumentID, VariableID, ConfigurationID"""
        )

        # Target should have all records with RevisionNo = 1
        target_values = self.target_conn.execute_query(
            """SELECT VariableID, DocumentID, ProjectID, RevisionNo, ConfigurationID FROM VariableValue
               WHERE ProjectID = 2 AND DocumentID != 1 AND RevisionNo = 1
               ORDER BY DocumentID, VariableID, ConfigurationID"""
        )

        # --------------------------------------
        # Convert target_values to a set of tuples for fast lookup
        # ---------------------------------------
        self.logger.info("Building target record set for comparison...")
        target_set = set()  # this is every row of the VariableValue table in the Target DB
        for record in target_values:
            key = (
                record['VariableID'],
                record['DocumentID'],
                record['ProjectID'],
                record['RevisionNo'],
                record['ConfigurationID']
            )
            target_set.add(key)

        error_list = []  # this is the container we are going to use to hold the errors we find
        success_count = 0  # we will just tally the records we find
        ignore_count = 0  # counter for rows that we didnt map because we couldnt find a documentID or variableID in the Target DB

        # now we search the target for each row using the mapped values to make sure that it is in there, we log it if we can't find it
        # --------------------------------------
        # Create a CSV to log all the rows we think we are missing and begin the scan
        # ---------------------------------------
        doc_csv_filename = f'validation_missing_filedata_{self.timestamp}.csv'

        with open(doc_csv_filename, 'w', newline='', encoding='utf-8') as csvfile:
            writer = csv.writer(csvfile)
            writer.writerow(['Target_VariableID', 'Target_DocumentID', 'ProjectID', 'RevisionNo', 'ConfigurationID'])  # write header
            self.logger.info(f"Prepared {len(source_values)} records. Beginning validation with Target DB")

            for record in source_values:
                # Extract source IDs
                source_variable_id = record.get('VariableID')
                source_document_id = record.get('DocumentID')

                # Map to target IDs
                target_variable_id = self.variable_map.get(source_variable_id)
                target_document_id = self.document_map.get(source_document_id)

                # Extract other fields
                source_rev = record.get('RevisionNo')  # Read but not used - we always insert as Rev 1
                source_configID = record.get('ConfigurationID')
                source_projectID = record.get("ProjectID")  # even though all of these should be 2 we are still going to pull the actual value

                # Map ConfigurationID to target
                target_config_id = self.configuration_map.get(source_configID)

                # Prepare params for validation
                # ProjectID is always 2 for files in target database
                # RevisionNo is always 1 (we migrated latest revision as Rev 1)
                params = (
                    target_variable_id,   # mapped VariableID
                    target_document_id,   # mapped DocumentID
                    source_projectID,     # ProjectID (should be 2 for files)
                    1,                    # RevisionNo (always 1 in target)
                    target_config_id,     # ConfigurationID (mapped)
                )

                # Some documents, variables, or configurations didnt map into the Target DB
                # These wont have a target ID so we are going to ignore them
                if not target_document_id or not target_variable_id or not target_config_id:
                    ignore_count += 1
                else:
                    if params in target_set:
                        success_count += 1
                    else:
                        self.logger.warning(f"Failed to find match for {params} in Target")
                        error_list.append(params)  # record which record is missing
                        writer.writerow([params[0], params[1], params[2], params[3], params[4]])


        self.logger.info("=" * 50)
        self.logger.info("$  Migration Validation Completed!")
        self.logger.info("=" * 50)
        self.logger.info(f"Gross Success rate: {(success_count / len(source_values)) * 100:.2f}%")
        self.logger.info(f"Success rate w/o Ignored Files: {(success_count / (len(source_values) - ignore_count)) * 100:.2f}%")
        self.logger.info(f"{success_count} of {len(source_values)} Rows were found")
        self.logger.info(f"-" * 50)
        self.logger.info(f"MISSING ROW COUNT:{len(error_list)} - See CSV output for details")
        self.logger.info(f"We ignored a total of {ignore_count} rows. We couldn't map these to the TargetDB. Either bad Var, Doc, or Config ID")

    def migrate_documents_status(self):
        """
        Migrate CurrentStatusID from source Documents to target Documents.
        Only updates CurrentStatusID if a state mapping exists in config.json.
        If no mapping exists, the target DB value remains unchanged.
        """
        self.logger.info("Starting Documents CurrentStatusID migration...")

        # Check if there are any state mappings
        if not self.state_map:
            self.logger.info("No state mappings configured - skipping CurrentStatusID migration")
            return

        # Get batch settings - use dedicated document status batch size if configured
        batch_size = self.config.get('migration', {}).get('document_status_batch_size', 5000)
        commit_interval = self.config.get('migration', {}).get('commit_interval', 10)

        self.logger.info(f"Migration settings: batch_size={batch_size}, commit_interval={commit_interval}")
        self.logger.info(f"Using executemany() for fast batch updates")

        # Fetch CurrentStatusID from source Documents
        source_documents = self.source_conn.execute_query("""
            SELECT DocumentID, CurrentStatusID
            FROM Documents
            WHERE CurrentStatusID IS NOT NULL
        """)

        self.logger.info(f"Found {len(source_documents)} documents with CurrentStatusID in source database")

        # Prepare update parameters
        update_params = []
        skipped_no_mapping = 0
        skipped_unmapped_doc = 0

        for doc in source_documents:
            source_doc_id = doc['DocumentID']
            source_status_id = doc['CurrentStatusID']

            # Map source DocumentID to target DocumentID
            target_doc_id = self.document_map.get(source_doc_id)

            if target_doc_id is None:
                # Document not mapped to target - skip
                skipped_unmapped_doc += 1
                continue

            # Map source CurrentStatusID to target CurrentStatusID
            target_status_id = self.state_map.get(source_status_id)

            if target_status_id is None:
                # No state mapping exists - leave target DB value unchanged
                self.logger.debug(f"Document {source_doc_id}: No state mapping for CurrentStatusID={source_status_id} - skipping")
                skipped_no_mapping += 1
                continue

            # Add to update list
            update_params.append((target_status_id, target_doc_id))

        self.logger.info(f"Prepared {len(update_params)} documents for CurrentStatusID update")
        self.logger.info(f"Skipped {skipped_unmapped_doc} documents (not mapped to target)")
        self.logger.info(f"Skipped {skipped_no_mapping} documents (no state mapping configured)")

        if not update_params:
            self.logger.info("No documents to update - migration complete")
            return

        # Update query
        update_query = """
            UPDATE Documents
            SET CurrentStatusID = ?
            WHERE DocumentID = ?
        """

        # Calculate batches
        total_batches = (len(update_params) + batch_size - 1) // batch_size

        # Initialize statistics
        total_updated = 0
        total_errors = 0

        # Process in batches
        for batch_num in range(total_batches):
            batch_start = batch_num * batch_size
            batch_end = min(batch_start + batch_size, len(update_params))
            batch_params = update_params[batch_start:batch_end]

            self.logger.info(f"Processing batch {batch_num + 1}/{total_batches} ({len(batch_params)} documents)...")

            # Execute updates for this batch using executemany (much faster than individual executes)
            try:
                cursor = self.target_conn.connection.cursor()
                cursor.executemany(update_query, batch_params)

                # executemany returns total affected rows
                batch_updated = cursor.rowcount
                total_updated += batch_updated

                cursor.close()

                self.logger.info(f"Batch {batch_num + 1} complete: {batch_updated} documents updated in this batch")

            except Exception as e:
                total_errors += len(batch_params)
                self.logger.error(f"Error updating batch {batch_num + 1}: {e}")
                self.logger.error(f"Failed to update {len(batch_params)} documents in this batch")

            # Commit every N batches
            if (batch_num + 1) % commit_interval == 0:
                self.target_conn.commit()
                self.logger.info(f"[COMMIT] Transaction committed at batch {batch_num + 1} (every {commit_interval} batches)")

        # Final commit
        self.target_conn.commit()
        self.logger.info(f"[SUCCESS] Final transaction committed")

        self.logger.info("Documents CurrentStatusID migration completed successfully!")
        self.logger.info(f"Total documents updated: {total_updated}")
        self.logger.info(f"Total errors: {total_errors}")

    def run(self):
        """Execute the complete file data migration process."""
        try:
            self.logger.info("=" * 50)
            self.logger.info("Starting File Data Migration")
            self.logger.info("=" * 50)

            # Build mappings
            self.build_variable_mapping()
            self.build_document_mapping()
            self.build_configuration_mapping()
            self.build_state_mapping()

            # Export mappings for verification
            self.export_mappings_to_csv()

            # Preview first batch of migrations
            # self.preview_migration(num_records=20)

            # Perform migration
            self.migrate_file_variable_values()

            # Migrate Documents CurrentStatusID if state mappings are configured
            # TODO DONT TOUCH THIS UNTIL A NEW VAULT MIGRATION
            # TODO State Mapping via SQL does not work
            # self.migrate_documents_status()

            # Perform validation
            # self.validate_migration()

            self.logger.info("=" * 50)
            self.logger.info("File Data Migration Completed Successfully!")
            self.logger.info("=" * 50)

        except Exception as e:
            self.logger.error(f"Migration failed: {e}", exc_info=True)
            raise

        finally:
            # Close connections
            self.source_conn.close()
            self.target_conn.close()


def main():
    """Main entry point."""
    migrator = FileDataMigration()
    migrator.run()


if __name__ == '__main__':
    main()
