Restart Cluster

{
  "update_configuration": {
    "cpu_cores": 64,              // Update CPU allocation
    "memory_gb": 512,             // Update memory allocation  
    "storage_gb": 2000,           // Expand storage (cannot shrink)
    "gpu_count": 8,               // Update GPU count (if available)
    "region": "us-west-2"         // Change region (requires data migration)
  }
}

def apply_system_updates(cluster_ids, maintenance_window):
    """Apply system updates during maintenance window"""
    
    results = []
    
    for cluster_id in cluster_ids:
        # Get current cluster configuration
        cluster_info = requests.get(
            f"https://api.tensorone.ai/v1/clusters/{cluster_id}",
            headers={"Authorization": f"Bearer {API_KEY}"}
        ).json()
        
        if not cluster_info["success"]:
            results.append({"cluster_id": cluster_id, "success": False, "error": "Failed to get cluster info"})
            continue
        
        cluster = cluster_info["data"]
        
        # Determine restart strategy based on cluster type
        restart_config = {
            "wait_for_ready": True,
            "timeout_minutes": 20,
            "apply_system_updates": True,
            "restart_reason": f"Maintenance window: {maintenance_window['id']}"
        }
        
        # Production clusters get more graceful treatment
        if "prod" in cluster["name"].lower():
            restart_config["grace_period_minutes"] = 15
            restart_config["preserve_state"] = True
        else:
            restart_config["grace_period_minutes"] = 5
        
        # Apply any pending Docker image updates
        if maintenance_window.get("docker_updates", {}).get(cluster_id):
            restart_config["update_docker_image"] = maintenance_window["docker_updates"][cluster_id]
        
        try:
            result = restart_cluster_basic(cluster_id, restart_config["restart_reason"])
            
            results.append({
                "cluster_id": cluster_id,
                "name": cluster["name"],
                "success": result["success"],
                "downtime_seconds": result.get("data", {}).get("downtime_seconds"),
                "updates_applied": result.get("data", {}).get("configuration_changes", [])
            })
            
        except Exception as e:
            results.append({
                "cluster_id": cluster_id,
                "success": False,
                "error": str(e)
            })
    
    return {
        "maintenance_window": maintenance_window["id"],
        "total_clusters": len(cluster_ids),
        "successful_updates": len([r for r in results if r["success"]]),
        "results": results
    }

async function deployModelVersion(clusterId, modelDeployment) {
  const deploymentConfig = {
    wait_for_ready: true,
    timeout_minutes: 30,
    update_docker_image: modelDeployment.dockerImage,
    environment_updates: {
      MODEL_VERSION: modelDeployment.version,
      MODEL_PATH: modelDeployment.modelPath,
      CONFIG_PATH: modelDeployment.configPath,
      DEPLOYMENT_ID: modelDeployment.deploymentId
    },
    restart_reason: `Model deployment: ${modelDeployment.name} v${modelDeployment.version}`
  };
  
  // Add hardware scaling if specified
  if (modelDeployment.scaling) {
    deploymentConfig.update_configuration = {
      gpu_count: modelDeployment.scaling.gpuCount,
      memory_gb: modelDeployment.scaling.memoryGb,
      cpu_cores: modelDeployment.scaling.cpuCores
    };
  }
  
  // Add port mappings for new services
  if (modelDeployment.services) {
    deploymentConfig.port_mapping_updates = modelDeployment.services.map(service => ({
      internal_port: service.port,
      external_port: 0, // Auto-assign
      protocol: 'tcp',
      description: service.description
    }));
  }
  
  try {
    console.log(`Deploying ${modelDeployment.name} v${modelDeployment.version} to cluster ${clusterId}`);
    
    const result = await restartWithUpdates(clusterId, deploymentConfig);
    
    // Validate deployment
    const healthCheck = await validateModelDeployment(clusterId, modelDeployment);
    
    if (!healthCheck.success) {
      throw new Error(`Model deployment validation failed: ${healthCheck.error}`);
    }
    
    return {
      success: true,
      clusterId: clusterId,
      deploymentId: modelDeployment.deploymentId,
      modelVersion: modelDeployment.version,
      proxyUrl: result.proxyUrl,
      services: result.portMappings || [],
      deploymentTime: result.restartDuration,
      healthCheck: healthCheck
    };
    
  } catch (error) {
    console.error(`Model deployment failed for cluster ${clusterId}:`, error);
    
    // Attempt rollback if previous version info is available
    if (modelDeployment.previousVersion) {
      console.log('Attempting rollback to previous version...');
      try {
        await deployModelVersion(clusterId, {
          ...modelDeployment,
          version: modelDeployment.previousVersion.version,
          dockerImage: modelDeployment.previousVersion.dockerImage,
          deploymentId: `rollback_${modelDeployment.deploymentId}`
        });
      } catch (rollbackError) {
        console.error('Rollback failed:', rollbackError);
      }
    }
    
    throw error;
  }
}

async function validateModelDeployment(clusterId, modelDeployment) {
  // Wait for services to be ready
  await new Promise(resolve => setTimeout(resolve, 30000));
  
  try {
    // Get cluster info to check services
    const response = await fetch(`https://api.tensorone.ai/v1/clusters/${clusterId}`, {
      headers: { 'Authorization': 'Bearer YOUR_API_KEY' }
    });
    
    const cluster = await response.json();
    
    if (!cluster.success) {
      return { success: false, error: 'Failed to get cluster info' };
    }
    
    // Check if all expected services are running
    const runningServices = cluster.data.network.port_mappings || [];
    const expectedServices = modelDeployment.services || [];
    
    for (const expectedService of expectedServices) {
      const runningService = runningServices.find(s => s.internal_port === expectedService.port);
      
      if (!runningService || runningService.status !== 'active') {
        return { 
          success: false, 
          error: `Service ${expectedService.description} is not active` 
        };
      }
      
      // Test service endpoint if health check URL provided
      if (expectedService.healthCheckPath) {
        try {
          const serviceUrl = `${runningService.url}${expectedService.healthCheckPath}`;
          const healthResponse = await fetch(serviceUrl, { timeout: 10000 });
          
          if (!healthResponse.ok) {
            return {
              success: false,
              error: `Service ${expectedService.description} health check failed`
            };
          }
        } catch (error) {
          return {
            success: false,
            error: `Service ${expectedService.description} is not responding`
          };
        }
      }
    }
    
    return {
      success: true,
      servicesValidated: expectedServices.length,
      allServicesHealthy: true
    };
    
  } catch (error) {
    return {
      success: false,
      error: `Validation failed: ${error.message}`
    };
  }
}

def recover_cluster_from_error(cluster_id, recovery_strategy="restart"):
    """Recover cluster from error state using specified strategy"""
    
    # Get current cluster state
    cluster_info = requests.get(
        f"https://api.tensorone.ai/v1/clusters/{cluster_id}",
        headers={"Authorization": f"Bearer {API_KEY}"}
    ).json()
    
    if not cluster_info["success"]:
        return {"success": False, "error": "Failed to get cluster info"}
    
    cluster = cluster_info["data"]
    
    if cluster["status"] != "error":
        return {"success": False, "error": f"Cluster is not in error state (current: {cluster['status']})"}
    
    recovery_actions = []
    
    if recovery_strategy == "restart":
        # Simple restart with force
        restart_config = {
            "force": True,
            "wait_for_ready": True,
            "apply_system_updates": True,
            "restart_reason": "Error recovery - forced restart"
        }
        
        result = requests.post(
            f"https://api.tensorone.ai/v1/clusters/{cluster_id}/restart",
            headers={"Authorization": f"Bearer {API_KEY}"},
            json=restart_config
        ).json()
        
        recovery_actions.append("Forced restart")
        
    elif recovery_strategy == "restore_snapshot":
        # Find latest snapshot and restore
        snapshots = cluster.get("storage", {}).get("snapshots", [])
        
        if not snapshots:
            return {"success": False, "error": "No snapshots available for restore"}
        
        latest_snapshot = max(snapshots, key=lambda s: s["created_at"])
        
        restart_config = {
            "force": True,
            "wait_for_ready": True,
            "restore_from_snapshot": latest_snapshot["id"],
            "restart_reason": f"Error recovery - restore from snapshot {latest_snapshot['id']}"
        }
        
        result = requests.post(
            f"https://api.tensorone.ai/v1/clusters/{cluster_id}/restart",
            headers={"Authorization": f"Bearer {API_KEY}"},
            json=restart_config
        ).json()
        
        recovery_actions.append(f"Restored from snapshot: {latest_snapshot['name']}")
        
    elif recovery_strategy == "rebuild":
        # Restart with fresh image and reset configuration
        original_template = cluster.get("template_info", {})
        
        restart_config = {
            "force": True,
            "wait_for_ready": True,
            "update_docker_image": original_template.get("docker_image"),
            "environment_updates": {
                "RECOVERY_MODE": "true",
                "RECOVERY_TIMESTAMP": datetime.now().isoformat()
            },
            "restart_reason": "Error recovery - rebuild from template"
        }
        
        result = requests.post(
            f"https://api.tensorone.ai/v1/clusters/{cluster_id}/restart",
            headers={"Authorization": f"Bearer {API_KEY}"},
            json=restart_config
        ).json()
        
        recovery_actions.append("Rebuilt from original template")
    
    if result["success"]:
        return {
            "success": True,
            "cluster_id": cluster_id,
            "recovery_strategy": recovery_strategy,
            "recovery_actions": recovery_actions,
            "restart_duration": result.get("data", {}).get("restart_duration_seconds"),
            "new_status": result.get("data", {}).get("status"),
            "recovery_completed_at": result.get("data", {}).get("restart_completed_at")
        }
    
    return result