Text Encoding & Character Sets

import chardet
import os

def analyze_corpus(directory):
    encodings = {}
    for filename in os.listdir(directory):
        if filename.endswith('.txt'):
            filepath = os.path.join(directory, filename)
            with open(filepath, 'rb') as f:
                result = chardet.detect(f.read())
            encodings[filename] = result
    return encodings

import ftfy

def repair_text_file(input_path, output_path):
    # Try multiple strategies
    strategies = [
        lambda: open(input_path, 'r', encoding='utf-8').read(),
        lambda: open(input_path, 'r', encoding='latin-1').read(),
        lambda: open(input_path, 'r', encoding='cp1252').read(),
    ]
    
    for i, strategy in enumerate(strategies):
        try:
            text = strategy()
            # Apply ftfy to fix common issues
            fixed_text = ftfy.fix_text(text)
            
            # Save repaired version
            with open(output_path, 'w', encoding='utf-8') as f:
                f.write(fixed_text)
            
            print(f"Repaired {input_path} using strategy {i+1}")
            return True
        except UnicodeDecodeError:
            continue
    
    print(f"Could not repair {input_path}")
    return False

def validate_repair(original_path, repaired_path):
    with open(repaired_path, 'r', encoding='utf-8') as f:
        text = f.read()
    
    # Check for common encoding artifacts
    problems = []
    if '�' in text:
        problems.append("Replacement characters found")
    if 'Ã' in text and len([c for c in text if ord(c) > 127]) > len(text) * 0.1:
        problems.append("Possible UTF-8/Latin-1 mix")
    
    return problems