"""
Optimus Research Center - Mission ORC-27 Comprehensive Scientific Audit & Acceptance Test Suite
Validates Sections 4.A through 4.H and Section 5 (Independent Scientific Acceptance Cases):
  - 4.A: ExternalCloudAIProvider honesty (never returns COMPLETED without real/contract execution).
  - 4.B: Algorithm Lab optimization (rastrigin, sphere, rosenbrock, ackley), predictive, search/planning, anomaly, and walk_forward_backtest circuit.
  - 4.C: AI Lab training, 3-way split without leakage, seed reproducibility, baseline comparison, and AI_ROBOTICS_SIM circuit.
  - 4.D: Inter-Lab Collaboration numeric acceptance (ACCEPTED) and rejection (REJECTED) with persisted evidence.
  - 4.E: Negative safety boundary tests for RoboticsSimulationEngine and QuantitativeResearchEngine.
  - 4.F: ProcessSandbox environment scrubbing, memory limit enforcement, and Category B untrusted code block.
  - 4.G: ScientificMemory concurrency-safe SQLite online backup and artifact SHA-256 integrity.
  - 4.H: Crash recovery reconciliation (RECONCILED_AS_FAILED, resumed=False) vs verified checkpoint requeue.
"""

import os
import json
import sqlite3
import hashlib
import pytest
import numpy as np
from pathlib import Path
from datetime import datetime

from app.config import settings
from app.core.database import SessionLocal, init_db
from app.core import models
from app.core.ai_provider import ExternalCloudAIProvider, LocalHeuristicAIProvider
from app.core.lab_registry import LabRegistry
from app.core.lab_adapters import AlgorithmLabAdapter, AILabAdapter
from app.core.coordinator import ScientificCoordinator
from app.core.collaboration import InterLabCoordinator
from app.core.sandbox import ProcessSandbox, ExecutionPolicy
from app.core.scientific_memory import ScientificMemory
from app.lab_algorithms.engine import AlgorithmBenchmarkingEngine
from app.lab_algorithms.service import AlgorithmLabService
from app.lab_algorithms.financial import QuantitativeResearchEngine, RealTradingProhibitedError
from app.lab_algorithms.algorithms.search_planning import AStarSearch, DijkstraSearch, CriticalPathPlanner
from app.lab_algorithms.algorithms.anomaly import IsolationForestAnomalyDetector, RollingZScoreAnomalyDetector
from app.lab_ai.service import AILabService
from app.lab_ai.trainer import AIModelTrainer, split_train_val_test
from app.lab_ai.robotics_sim import RoboticsSimulationEngine, HardwareActuationLockedError


@pytest.fixture(scope="module")
def db_session():
    init_db()
    db = SessionLocal()
    yield db
    db.close()


# =============================================================================
# 4.A — AI PROVIDER HONESTY (ExternalCloudAIProvider)
# =============================================================================

def test_4a_external_provider_never_fakes_synthesis_when_unconfigured(monkeypatch):
    """Verifies ExternalCloudAIProvider.synthesize_findings returns NEEDS_MODEL when no real model exists."""
    provider = ExternalCloudAIProvider("gemini")
    resp = provider.synthesize_findings({"status": "COMPLETED", "metrics": {"accuracy": 0.91}})
    assert resp.success is False
    assert resp.status == "NEEDS_MODEL"
    assert resp.structured_result is not None
    assert resp.structured_result.origin_type == "NONE"

    # Even if API key and ALLOW_EXTERNAL_API_CALLS are forced True without a live SDK client,
    # synthesize_findings must NOT fabricate a COMPLETED response.
    monkeypatch.setattr(settings, "GEMINI_API_KEY", "AIzaSyFakeKeyForAuditTest123456")
    monkeypatch.setattr(settings, "ALLOW_EXTERNAL_API_CALLS", True)
    provider_forced = ExternalCloudAIProvider("gemini", client_mock=None)
    resp_forced = provider_forced.synthesize_findings({"status": "COMPLETED", "metrics": {"accuracy": 0.91}})
    assert resp_forced.success is False
    assert resp_forced.status == "NEEDS_MODEL"


# =============================================================================
# 4.B — ALGORITHM LAB CAPABILITIES & CIRCUIT ROUTING
# =============================================================================

def test_4b_optimization_all_declared_benchmarks_and_reproducibility(db_session):
    """Verifies sphere, rastrigin, rosenbrock, and ackley benchmarks and deterministic seed reproducibility."""
    for prob in ["sphere", "rastrigin", "rosenbrock", "ackley"]:
        r1 = AlgorithmBenchmarkingEngine.benchmark_optimization(
            algorithms=["SimulatedAnnealing", "DifferentialEvolution", "GeneticAlgorithm", "BayesianSurrogate"],
            problem_type=prob,
            dim=4,
            random_seed=42
        )
        r2 = AlgorithmBenchmarkingEngine.benchmark_optimization(
            algorithms=["SimulatedAnnealing", "DifferentialEvolution", "GeneticAlgorithm", "BayesianSurrogate"],
            problem_type=prob,
            dim=4,
            random_seed=42
        )
        assert r1["top_fitness"] == r2["top_fitness"]
        assert r1["top_candidate"] == r2["top_candidate"]
        assert len(r1["runs"]) == 4

    # Also verify AlgorithmLabService.investigate_new_algorithm_flow supports rosenbrock & ackley
    svc = AlgorithmLabService(db_session)
    inv_rosen = svc.investigate_new_algorithm_flow(
        problem_description="Minimize 4D Rosenbrock valley",
        candidate_algorithms=["SimulatedAnnealing", "DifferentialEvolution"],
        benchmark_problem="rosenbrock",
        dimension=4
    )
    assert inv_rosen["status"] == "COMPLETED"
    assert inv_rosen["experiment_id"] is not None


def test_4b_predictive_and_quantitative_circuits_via_adapter_and_coordinator(db_session):
    """Verifies QUANTITATIVE_RESEARCH and ALGORITHM_PREDICTIVE work end-to-end via Adapter and Coordinator."""
    adapter = AlgorithmLabAdapter(db_session)

    # 1. Predictive experiment via adapter
    pred_res = adapter.execute_experiment(
        "ALGORITHM_PREDICTIVE",
        {"project_id": 1, "algorithm_names": ["ridge_regression", "gradient_boosting", "knn_regressor"], "n_samples": 120}
    )
    assert pred_res["status"] == "COMPLETED"
    assert pred_res["experiment_id"] is not None
    assert len(pred_res["report_sha256"]) == 64

    # 2. Quantitative research via adapter (previously failed with AttributeError: walk_forward_backtest)
    quant_res = adapter.execute_experiment(
        "QUANTITATIVE_RESEARCH",
        {"project_id": 1, "strategy": "MOMENTUM_RSI_MACD", "train_ratio": 0.7}
    )
    assert quant_res["status"] == "COMPLETED"
    assert quant_res["leakage_prevented"] is True
    assert quant_res["live_trading_blocked"] is True
    assert "walk_forward" in quant_res and "backtest" in quant_res
    assert quant_res["experiment_id"] is not None

    # 3. Quantitative research via ScientificCoordinator task queue
    coord = ScientificCoordinator(db_session)
    task = coord.submit_task(
        task_type="QUANTITATIVE_RESEARCH",
        payload={"project_id": 1, "strategy": "MOMENTUM_RSI_MACD", "train_ratio": 0.7},
        priority=2
    )
    exec_out = coord.process_next_task(task_id=task.id)
    assert exec_out["status"] == "COMPLETED"
    assert exec_out["evidence_hash"] is not None


def test_4b_search_planning_and_anomaly_algorithms():
    """Verifies AStarSearch, DijkstraSearch, CriticalPathPlanner, IsolationForest, and RollingZScore."""
    grid = [
        [0, 0, 0, 0],
        [1, 1, 0, 1],
        [0, 0, 0, 0],
        [0, 1, 1, 0]
    ]
    astar = AStarSearch.search(grid, start=(0, 0), goal=(3, 3))
    assert astar["path_found"] is True
    assert astar["total_cost"] == 6

    graph = {"A": [("B", 2.0), ("C", 5.0)], "B": [("C", 1.0), ("D", 4.0)], "C": [("D", 1.0)], "D": []}
    dijk = DijkstraSearch.search(graph, "A", "D")
    assert dijk["path_found"] is True
    assert dijk["shortest_distance"] == 4.0
    assert dijk["path"] == ["A", "B", "C", "D"]

    cpm = CriticalPathPlanner.plan({
        "A": {"duration": 3, "predecessors": []},
        "B": {"duration": 4, "predecessors": ["A"]},
        "C": {"duration": 2, "predecessors": ["A"]},
        "D": {"duration": 5, "predecessors": ["B", "C"]}
    })
    assert cpm["total_project_duration"] == 12
    assert cpm["critical_path"] == ["A", "B", "D"]

    rng = np.random.RandomState(42)
    X_norm = rng.normal(0, 1, size=(100, 3))
    X_Out = np.vstack([X_norm, np.array([[15.0, 15.0, 15.0], [-15.0, -15.0, -15.0]])])
    iso = IsolationForestAnomalyDetector(contamination=0.05, random_seed=42).detect(X_Out)
    assert iso["anomalies_detected"] >= 2
    assert 100 in iso["anomaly_indices"] or 101 in iso["anomaly_indices"]

    series = np.zeros(50)
    series[25] = 20.0
    z_res = RollingZScoreAnomalyDetector(window_size=10, z_threshold=2.5).detect_series(series)
    assert 25 in z_res["anomaly_indices"]


# =============================================================================
# 4.C — AI LAB TRAINING, LEAKAGE PREVENTION & ROBOTICS CIRCUIT
# =============================================================================

def test_4c_ai_3way_split_reproducibility_and_robotics_circuit(db_session):
    """Verifies 3-way split, seed reproducibility, baseline comparison, and AI_ROBOTICS_SIM routing."""
    from sklearn.datasets import make_classification
    X, y = make_classification(n_samples=200, n_features=10, n_informative=6, random_state=42)
    X_tr, X_val, X_te, y_tr, y_val, y_te = split_train_val_test(X, y, test_size=0.2, val_size=0.2, random_seed=42)
    assert len(X_tr) == 120 and len(X_val) == 40 and len(X_te) == 40

    _, m1, _ = AIModelTrainer.train_pytorch_mlp_3way(X_tr, y_tr, X_val, y_val, X_te, y_te, hidden_dim=16, epochs=10, random_seed=42)
    _, m2, _ = AIModelTrainer.train_pytorch_mlp_3way(X_tr, y_tr, X_val, y_val, X_te, y_te, hidden_dim=16, epochs=10, random_seed=42)
    assert m1["val_accuracy"] == m2["val_accuracy"]
    assert m1["test_accuracy"] == m2["test_accuracy"]
    assert m1["scaler_fitted_on"] == "TRAIN_ONLY"

    # Verify AI_ROBOTICS_SIM via AILabAdapter and ScientificCoordinator (previously failed with TypeError)
    ai_adapter = AILabAdapter(db_session)
    sim_res = ai_adapter.execute_experiment(
        "AI_ROBOTICS_SIM",
        {"project_id": 1, "hypothesis_id": None, "simulation_type": "kalman_filter", "steps": 60}
    )
    assert sim_res["status"] == "COMPLETED"
    assert sim_res["simulation_result"]["physical_actuators_active"] is False
    assert sim_res["simulation_result"]["samples_processed"] == 60
    assert len(sim_res["report_sha256"]) == 64

    coord = ScientificCoordinator(db_session)
    t_sim = coord.submit_task(
        task_type="AI_ROBOTICS_SIM",
        payload={"project_id": 1, "simulation_type": "grid_navigation", "steps": 50},
        priority=2
    )
    sim_task_out = coord.process_next_task(task_id=t_sim.id)
    assert sim_task_out["status"] == "COMPLETED"
    assert sim_task_out["evidence_hash"] is not None


# =============================================================================
# 4.D — INTER-LAB COLLABORATION (EMPIRICAL ACCEPTANCE & REJECTION)
# =============================================================================

def test_4d_interlab_collaboration_acceptance_and_rejection(db_session):
    """Verifies InterLabCoordinator accepts genuine improvements and rejects inferior candidates."""
    collab = InterLabCoordinator(db_session)
    algo_svc = AlgorithmLabService(db_session)

    # Create origin experiment
    origin_res = algo_svc.run_optimization_experiment(
        project_id=1,
        hypothesis_id=None,
        algorithm_names=["SimulatedAnnealing", "DifferentialEvolution"],
        problem_type="rastrigin",
        dim=4,
        random_seed=42
    )
    origin_exp_id = origin_res["experiment_id"]

    # Case 1: Propose technique where candidate (GeneticAlgorithm) outperforms baseline (SimulatedAnnealing) -> ACCEPTED
    prop_accept = collab.propose_technique(
        title="GA Evolutionary Crossover Transfer",
        source_lab="AI_LAB",
        target_lab="ALGORITHM_LAB",
        technique_name="GA_EVOLUTIONARY_CROSSOVER",
        technique_version="1.0.0",
        payload={
            "candidate_algorithms": ["SimulatedAnnealing", "GeneticAlgorithm"],
            "benchmark_problem": "rastrigin",
            "dimension": 5
        },
        origin_experiment_id=origin_exp_id
    )
    res_accept = collab.execute_and_submit_evaluation(collaboration_id=prop_accept.id)
    assert res_accept.status == "ACCEPTED"
    metrics_accept = json.loads(res_accept.evaluation_metrics_json)
    assert metrics_accept["is_improvement"] is True
    assert metrics_accept["delta"] > 0.0

    # Also verify Algorithm Lab -> AI Lab collaboration acceptance
    prop_ai_accept = collab.propose_technique(
        title="Bayesian Tuned MLP Capacity",
        source_lab="ALGORITHM_LAB",
        target_lab="AI_LAB",
        technique_name="BAYESIAN_MLP_CAPACITY_64",
        technique_version="1.0.0",
        payload={"tuned_hidden_dim": 64, "tuned_epochs": 30},
        origin_experiment_id=origin_exp_id
    )
    res_ai_accept = collab.execute_and_submit_evaluation(collaboration_id=prop_ai_accept.id)
    assert res_ai_accept.status == "ACCEPTED"

    # Case 2: Propose technique where candidate does NOT outperform baseline -> REJECTED
    prop_reject = collab.propose_technique(
        title="No-Gain Static Schedule Transfer",
        source_lab="AI_LAB",
        target_lab="ALGORITHM_LAB",
        technique_name="STATIC_NO_GAIN_SCHEDULE",
        technique_version="1.0.0",
        payload={
            "candidate_algorithms": ["SimulatedAnnealing"],
            "benchmark_problem": "sphere",
            "dimension": 4
        },
        origin_experiment_id=origin_exp_id
    )
    res_reject = collab.execute_and_submit_evaluation(collaboration_id=prop_reject.id)
    assert res_reject.status == "REJECTED"
    metrics_reject = json.loads(res_reject.evaluation_metrics_json)
    assert metrics_reject["is_improvement"] is False
    assert metrics_reject["delta"] <= 0.0


# =============================================================================
# 4.E — NEGATIVE SAFETY TESTS FOR ROBOTICS & FINANCE
# =============================================================================

def test_4e_robotics_and_financial_negative_safety_locks(monkeypatch):
    """Verifies active safety assertions raise exceptions when physical hardware or live trading is attempted."""
    # Default safe state passes
    r_state = RoboticsSimulationEngine.assert_simulation_only()
    assert r_state["simulation_only"] is True
    f_state = QuantitativeResearchEngine.assert_no_live_trading()
    assert f_state["live_trading_enabled"] is False

    # Negative tests for RoboticsSimulationEngine
    with pytest.raises(HardwareActuationLockedError):
        RoboticsSimulationEngine.assert_simulation_only(allow_physical_actuation=True)

    with pytest.raises(HardwareActuationLockedError):
        RoboticsSimulationEngine.assert_simulation_only(hardware_interface="ROS_SERIAL_COM3")

    monkeypatch.setenv("ALLOW_PHYSICAL_ACTUATION", "true")
    with pytest.raises(HardwareActuationLockedError):
        RoboticsSimulationEngine.assert_simulation_only()
    monkeypatch.delenv("ALLOW_PHYSICAL_ACTUATION", raising=False)

    # Negative tests for QuantitativeResearchEngine
    with pytest.raises(RealTradingProhibitedError):
        QuantitativeResearchEngine.assert_no_live_trading(live_order=True)

    with pytest.raises(RealTradingProhibitedError):
        QuantitativeResearchEngine.assert_no_live_trading(broker_endpoint="https://api.alpaca.markets/v2/orders")

    monkeypatch.setenv("ALLOW_LIVE_TRADING", "true")
    with pytest.raises(RealTradingProhibitedError):
        QuantitativeResearchEngine.assert_no_live_trading()
    monkeypatch.delenv("ALLOW_LIVE_TRADING", raising=False)


# =============================================================================
# 4.F — PROCESS SANDBOX HARDENING & CATEGORY B BLOCK
# =============================================================================

def test_4f_process_sandbox_env_scrubbing_and_memory_enforcement(monkeypatch):
    """Verifies ProcessSandbox scrubs secret env vars, enforces memory limit, and labels isolation level."""
    monkeypatch.setenv("OPENAI_API_KEY", "sk-secret-do-not-leak-999")
    monkeypatch.setenv("BROKER_PRIVATE_KEY", "priv-secret-key-888")

    env_script = settings.sandbox_tmp_path / "test_env_scrub.py"
    env_script.write_text(
        "import os, json\n"
        "print(json.dumps({\n"
        "  'has_openai': 'OPENAI_API_KEY' in os.environ,\n"
        "  'has_broker': 'BROKER_PRIVATE_KEY' in os.environ,\n"
        "  'sandbox_flag': os.environ.get('OPTIMUS_SANDBOX_SUBPROCESS')\n"
        "}))\n",
        encoding="utf-8"
    )
    try:
        sb = ProcessSandbox(timeout_seconds=5, max_memory_mb=256)
        res = sb.execute_in_sandbox(env_script)
        assert res["success"] is True
        assert res["env_scrubbed"] is True
        assert res["isolation_level"] == "SUBPROCESS_HARNESS_NOT_OS_CONTAINER"
        payload = json.loads(res["stdout"].strip())
        assert payload["has_openai"] is False
        assert payload["has_broker"] is False
        assert payload["sandbox_flag"] == "1"
    finally:
        env_script.unlink(missing_ok=True)

    # Memory limit enforcement test (set low limit of 5 MB so Python interpreter RSS exceeds it immediately)
    mem_script = settings.sandbox_tmp_path / "test_mem_limit.py"
    mem_script.write_text(
        "import time\n"
        "data = bytearray(40 * 1024 * 1024)\n"
        "time.sleep(0.5)\n",
        encoding="utf-8"
    )
    try:
        sb_low_mem = ProcessSandbox(timeout_seconds=5, max_memory_mb=5)
        res_mem = sb_low_mem.execute_in_sandbox(mem_script)
        assert res_mem["success"] is False
        assert "memory limit" in res_mem.get("error", "").lower()
    finally:
        mem_script.unlink(missing_ok=True)

    # Category B untrusted code remains strictly SANDBOX_UNAVAILABLE
    cat_b = ExecutionPolicy.execute_untrusted_code("def custom_kernel(x): return x * 2")
    assert cat_b["status"] == "SANDBOX_UNAVAILABLE"


# =============================================================================
# 4.G — SCIENTIFIC MEMORY ONLINE SQLITE BACKUP & INTEGRITY
# =============================================================================

def test_4g_scientific_memory_online_backup_integrity(db_session):
    """Verifies ScientificMemory.create_memory_backup creates a WAL-consistent SQLite backup."""
    mem = ScientificMemory(db_session)
    backup_path = mem.create_memory_backup()
    assert backup_path.exists()
    assert backup_path.stat().st_size > 0

    conn = sqlite3.connect(str(backup_path))
    try:
        integrity = conn.execute("PRAGMA integrity_check").fetchone()[0]
        assert integrity == "ok"
        tables = conn.execute("SELECT name FROM sqlite_master WHERE type='table'").fetchall()
        assert len(tables) >= 12
    finally:
        conn.close()
        backup_path.unlink(missing_ok=True)


# =============================================================================
# 4.H — CRASH RECOVERY RECONCILIATION VS CHECKPOINT RESUMPTION
# =============================================================================

def test_4h_recovery_distinguishes_reconciliation_from_checkpoint_resumption(db_session):
    """Verifies recover_interrupted_tasks marks uncheckpointed tasks FAILED (resumed=False) and requeues valid checkpoints."""
    coord = ScientificCoordinator(db_session)

    # 1. Uncheckpointed interrupted task -> RECONCILED_AS_FAILED, resumed=False
    t_unck = coord.submit_task(task_type="ALGORITHM_BENCHMARK", payload={"problem_type": "sphere"}, priority=3)
    coord.update_task_status(t_unck.id, "RUNNING")

    # 2. Task with verified SHA-256 checkpoint -> CHECKPOINT_REQUEUED_FOR_RESUMPTION, resumed=True
    ck_state = {"round_completed": 1, "best_candidate": "DifferentialEvolution", "best_fitness": -0.0012}
    ck_hash = hashlib.sha256(json.dumps(ck_state, sort_keys=True).encode("utf-8")).hexdigest()
    t_ck = coord.submit_task(
        task_type="ALGORITHM_BENCHMARK",
        payload={
            "problem_type": "sphere",
            "checkpoint": {
                "resumable": True,
                "state": ck_state,
                "checkpoint_sha256": ck_hash
            }
        },
        priority=2
    )
    coord.update_task_status(t_ck.id, "RUNNING")

    recovered = coord.recover_interrupted_tasks()
    rec_map = {r["task_id"]: r for r in recovered}

    assert t_unck.id in rec_map
    assert rec_map[t_unck.id]["new_status"] == "FAILED"
    assert rec_map[t_unck.id]["recovery_action"] == "RECONCILED_AS_FAILED"
    assert rec_map[t_unck.id]["resumed"] is False

    assert t_ck.id in rec_map
    assert rec_map[t_ck.id]["new_status"] == "PENDING"
    assert rec_map[t_ck.id]["recovery_action"] == "CHECKPOINT_REQUEUED_FOR_RESUMPTION"
    assert rec_map[t_ck.id]["resumed"] is True
    assert rec_map[t_ck.id]["checkpoint_verified"] is True

    # Clean up the requeued task
    coord.cancel_task(t_ck.id, reason="Test cleanup")