Delete Cluster

def cleanup_completed_project(project_id, project_name):
    """Clean up all resources for a completed project"""
    
    print(f"Starting cleanup for completed project: {project_name}")
    
    # Get all clusters in the project
    clusters_response = requests.get(
        "https://api.tensorone.ai/v1/clusters",
        headers={"Authorization": f"Bearer {API_KEY}"},
        params={"project_id": project_id}
    )
    
    clusters = clusters_response.json()["data"]["clusters"]
    
    cleanup_summary = {
        "project_id": project_id,
        "project_name": project_name,
        "total_clusters": len(clusters),
        "clusters_deleted": 0,
        "total_cost_recovered": 0,
        "snapshots_preserved": 0,
        "cleanup_details": []
    }
    
    for cluster in clusters:
        cluster_type = "unknown"
        
        # Determine cluster type and appropriate cleanup strategy
        if "prod" in cluster["name"].lower():
            cluster_type = "production"
            create_backup = True
            preserve_snapshots = True
        elif "train" in cluster["name"].lower():
            cluster_type = "training"
            create_backup = True
            preserve_snapshots = True
        elif "dev" in cluster["name"].lower():
            cluster_type = "development"
            create_backup = False
            preserve_snapshots = False
        else:
            cluster_type = "other"
            create_backup = True
            preserve_snapshots = True
        
        try:
            result = delete_cluster_safe(
                cluster["id"],
                reason=f"Project completion cleanup - {project_name}",
                create_backup=create_backup
            )
            
            if result["success"]:
                cleanup_summary["clusters_deleted"] += 1
                cleanup_summary["total_cost_recovered"] += result.get("data", {}).get("billing_summary", {}).get("total_cost", 0)
                cleanup_summary["snapshots_preserved"] += len(result.get("data", {}).get("snapshots_preserved", []))
                
                cleanup_summary["cleanup_details"].append({
                    "cluster_id": cluster["id"],
                    "cluster_name": cluster["name"],
                    "cluster_type": cluster_type,
                    "success": True,
                    "final_cost": result.get("data", {}).get("billing_summary", {}).get("total_cost"),
                    "final_snapshot_id": result.get("data", {}).get("final_snapshot_id")
                })
            
        except Exception as e:
            cleanup_summary["cleanup_details"].append({
                "cluster_id": cluster["id"],
                "cluster_name": cluster["name"],
                "cluster_type": cluster_type,
                "success": False,
                "error": str(e)
            })
    
    print(f"Project cleanup completed:")
    print(f"  Clusters deleted: {cleanup_summary['clusters_deleted']}/{cleanup_summary['total_clusters']}")
    print(f"  Total cost recovered: ${cleanup_summary['total_cost_recovered']:.2f}")
    print(f"  Snapshots preserved: {cleanup_summary['snapshots_preserved']}")
    
    return cleanup_summary

async function cleanupExpensiveIdleClusters(costThreshold = 10.0, idleHours = 24) {
  console.log(`Cleaning up idle clusters costing more than $${costThreshold}/hour idle for ${idleHours}+ hours`);
  
  try {
    // Get all running clusters
    const response = await fetch('https://api.tensorone.ai/v1/clusters?status=running&include_metrics=true', {
      headers: { 'Authorization': 'Bearer YOUR_API_KEY' }
    });
    
    const clustersData = await response.json();
    const clusters = clustersData.data.clusters;
    
    const expensiveIdleClusters = clusters.filter(cluster => {
      const hourlyRate = cluster.cost.hourly_rate;
      const gpuUtilization = cluster.metrics.current.gpu_utilization;
      const lastAccessed = new Date(cluster.last_accessed);
      const hoursIdle = (Date.now() - lastAccessed.getTime()) / (1000 * 60 * 60);
      
      return hourlyRate >= costThreshold && 
             gpuUtilization < 5 && // Very low utilization
             hoursIdle >= idleHours;
    });
    
    console.log(`Found ${expensiveIdleClusters.length} expensive idle clusters`);
    
    const deletionResults = [];
    let totalSavings = 0;
    
    for (const cluster of expensiveIdleClusters) {
      try {
        console.log(`Deleting idle cluster: ${cluster.name} ($${cluster.cost.hourly_rate}/hr)`);
        
        const result = await deleteClusterSafely(cluster.id, {
          createBackup: true, // Always backup expensive clusters
          preserveSnapshots: true,
          reason: `Cost optimization - idle cluster cleanup (${cluster.cost.hourly_rate}/hr for ${Math.round((Date.now() - new Date(cluster.last_accessed).getTime()) / (1000 * 60 * 60))} hours)`,
          waitForCompletion: false
        });
        
        totalSavings += cluster.cost.hourly_rate;
        
        deletionResults.push({
          clusterId: cluster.id,
          name: cluster.name,
          hourlyRate: cluster.cost.hourly_rate,
          hoursIdle: Math.round((Date.now() - new Date(cluster.last_accessed).getTime()) / (1000 * 60 * 60)),
          success: true,
          finalSnapshotId: result.finalSnapshotId
        });
        
      } catch (error) {
        deletionResults.push({
          clusterId: cluster.id,
          name: cluster.name,
          hourlyRate: cluster.cost.hourly_rate,
          success: false,
          error: error.message
        });
      }
      
      // Brief delay between deletions
      await new Promise(resolve => setTimeout(resolve, 2000));
    }
    
    const successfulDeletions = deletionResults.filter(r => r.success);
    
    return {
      totalCandidates: expensiveIdleClusters.length,
      successfulDeletions: successfulDeletions.length,
      hourlySavings: totalSavings,
      estimatedMonthlySavings: totalSavings * 24 * 30,
      results: deletionResults
    };
    
  } catch (error) {
    console.error('Cost optimization cleanup failed:', error);
    throw error;
  }
}

def emergency_security_cleanup(incident_id, affected_cluster_ids, security_reason):
    """Emergency deletion of clusters due to security incident"""
    
    print(f"EMERGENCY SECURITY CLEANUP - Incident: {incident_id}")
    print(f"Reason: {security_reason}")
    print(f"Affected clusters: {len(affected_cluster_ids)}")
    
    cleanup_results = {
        "incident_id": incident_id,
        "security_reason": security_reason,
        "total_clusters": len(affected_cluster_ids),
        "clusters_deleted": 0,
        "clusters_failed": 0,
        "emergency_actions": [],
        "cleanup_timestamp": datetime.now().isoformat()
    }
    
    for cluster_id in affected_cluster_ids:
        try:
            print(f"Emergency deleting cluster: {cluster_id}")
            
            # Force delete with minimal delay
            emergency_payload = {
                "confirm_deletion": "DELETE",
                "force": True,
                "create_final_snapshot": False,  # Skip backup for speed
                "delete_snapshots": True,        # Remove all data for security
                "cleanup_network_resources": True,
                "wait_for_completion": True,
                "timeout_minutes": 5,            # Short timeout
                "deletion_reason": f"SECURITY INCIDENT {incident_id}: {security_reason}"
            }
            
            response = requests.delete(
                f"https://api.tensorone.ai/v1/clusters/{cluster_id}",
                headers={"Authorization": f"Bearer {API_KEY}"},
                json=emergency_payload
            )
            
            result = response.json()
            
            if result["success"]:
                cleanup_results["clusters_deleted"] += 1
                cleanup_results["emergency_actions"].append({
                    "cluster_id": cluster_id,
                    "action": "force_deleted",
                    "deleted_at": result["data"]["deleted_at"],
                    "resources_cleaned": result["data"]["resources_cleaned_up"]
                })
                print(f"✅ Emergency deleted: {cluster_id}")
            else:
                cleanup_results["clusters_failed"] += 1
                cleanup_results["emergency_actions"].append({
                    "cluster_id": cluster_id,
                    "action": "delete_failed",
                    "error": result["error"]["message"]
                })
                print(f"❌ Failed to delete: {cluster_id} - {result['error']['message']}")
                
        except Exception as e:
            cleanup_results["clusters_failed"] += 1
            cleanup_results["emergency_actions"].append({
                "cluster_id": cluster_id,
                "action": "delete_error",
                "error": str(e)
            })
            print(f"❌ Error deleting {cluster_id}: {str(e)}")
    
    print(f"\nEmergency cleanup completed:")
    print(f"  Successfully deleted: {cleanup_results['clusters_deleted']}")
    print(f"  Failed deletions: {cleanup_results['clusters_failed']}")
    
    # Log security incident for audit
    security_log = {
        "incident_id": incident_id,
        "timestamp": cleanup_results["cleanup_timestamp"],
        "action": "emergency_cluster_cleanup",
        "affected_resources": len(affected_cluster_ids),
        "successful_cleanup": cleanup_results["clusters_deleted"],
        "reason": security_reason
    }
    
    return cleanup_results, security_log