rebuilt backtesting pipeline - 10x faster parameter optimization

# Old slow approach
for lookback in range(10, 60, 5):
    for vol_threshold in np.arange(0.5, 3.0, 0.1):
        for profit_target in np.arange(0.3, 0.8, 0.05):
            results = run_backtest(lookback, vol_threshold, profit_target)
            store_results(results)

from multiprocessing import Pool
from functools import partial
import numpy as np

class BacktestOptimizer:
    def __init__(self, strategy_class, data, n_workers=8):
        self.strategy_class = strategy_class
        self.data = data
        self.n_workers = n_workers

    def optimize_parameters(self, param_grid):
        """Run parallel parameter optimization"""
        # Generate all parameter combinations
        param_combinations = self._generate_combinations(param_grid)

        # Create partial function with data pre-loaded
        backtest_func = partial(
            self._run_single_backtest,
            strategy_class=self.strategy_class,
            data=self.data
        )

        # Run parallel backtests
        with Pool(processes=self.n_workers) as pool:
            results = pool.map(backtest_func, param_combinations)

        return self._rank_results(results)

    @staticmethod
    def _run_single_backtest(params, strategy_class, data):
        """Single backtest execution"""
        strategy = strategy_class(**params)
        engine = BacktestEngine(strategy, data)

        # Run backtest
        trades = engine.run()

        # Calculate metrics
        sharpe = calculate_sharpe(trades)
        max_dd = calculate_max_drawdown(trades)
        win_rate = calculate_win_rate(trades)
        profit_factor = calculate_profit_factor(trades)

        return {
            'params': params,
            'sharpe': sharpe,
            'max_dd': max_dd,
            'win_rate': win_rate,
            'profit_factor': profit_factor,
            'total_trades': len(trades),
            'score': sharpe * (1 - max_dd/100)  # Custom fitness
        }

class DataCache:
    def __init__(self):
        self.cache = {}

    def load_symbol_data(self, symbol, start_date, end_date):
        """Load and cache market data"""
        cache_key = f"{symbol}_{start_date}_{end_date}"

        if cache_key not in self.cache:
            df = self._fetch_from_db(symbol, start_date, end_date)
            df = self._calculate_indicators(df)
            self.cache[cache_key] = df

        return self.cache[cache_key]

    def _calculate_indicators(self, df):
        """Pre-calculate common indicators"""
        df['sma_20'] = df['close'].rolling(20).mean()
        df['sma_50'] = df['close'].rolling(50).mean()
        df['volatility'] = df['close'].pct_change().rolling(20).std()
        df['atr'] = self._calculate_atr(df)
        return df

class VectorizedBacktest:
    """Ultra-fast vectorized backtesting"""

    def __init__(self, data: pd.DataFrame):
        self.data = data

    def run_strategy(self, lookback: int, vol_threshold: float):
        """Vectorized strategy execution"""
        df = self.data.copy()

        # Calculate signals vectorized
        df['signal'] = 0
        df['vol'] = df['close'].pct_change().rolling(lookback).std()

        # Entry conditions (all at once)
        long_entry = (
            (df['sma_20'] > df['sma_50']) &
            (df['vol'] < vol_threshold) &
            (df['vol'].shift(1) >= vol_threshold)
        )

        df.loc[long_entry, 'signal'] = 1

        # Calculate returns vectorized
        df['strategy_returns'] = df['signal'].shift(1) * df['close'].pct_change()
        df['equity_curve'] = (1 + df['strategy_returns']).cumprod()

        return self._calculate_metrics(df)

    def _calculate_metrics(self, df):
        """Fast metric calculation"""
        returns = df['strategy_returns'].dropna()

        sharpe = np.sqrt(252) * returns.mean() / returns.std()

        # Vectorized drawdown calculation
        cumulative = (1 + returns).cumprod()
        running_max = cumulative.expanding().max()
        drawdown = (cumulative - running_max) / running_max
        max_dd = drawdown.min()

        return {
            'sharpe': sharpe,
            'max_dd': max_dd * 100,
            'total_return': (cumulative.iloc[-1] - 1) * 100
        }

class AdaptiveOptimizer:
    """Prunes unpromising parameter regions early"""

    def __init__(self, param_grid, n_rounds=3):
        self.param_grid = param_grid
        self.n_rounds = n_rounds

    def optimize(self, backtest_func):
        """Multi-round optimization with pruning"""
        current_grid = self.param_grid

        for round_num in range(self.n_rounds):
            print(f"Round {round_num + 1}: testing {len(current_grid)} combinations")

            # Test current grid
            results = self._parallel_backtest(current_grid, backtest_func)

            # Keep top 30% performers
            top_performers = self._get_top_n_percent(results, 30)

            if round_num < self.n_rounds - 1:
                # Narrow grid around top performers
                current_grid = self._create_refined_grid(top_performers)

        return top_performers[0]  # Best overall

    def _create_refined_grid(self, top_results):
        """Create refined grid around best parameters"""
        refined = []

        for result in top_results[:5]:  # Top 5 only
            params = result['params']

            # Create tighter grid around these params
            for key, value in params.items():
                delta = self._get_delta(key)
                refined.extend([
                    {**params, key: value - delta},
                    {**params, key: value},
                    {**params, key: value + delta}
                ])

        return refined