Start Cluster

{
  "update_configuration": {
    "cpu_cores": 64,              // Update CPU allocation
    "memory_gb": 512,             // Update memory allocation
    "storage_gb": 2000,           // Expand storage (cannot shrink)
    "docker_image": "new_image",  // Update Docker image
    "gpu_count": 8                // Update GPU count (if available)
  }
}

def track_startup_progress(cluster_id):
    """Track cluster startup progress with real-time updates"""
    
    import time
    
    while True:
        response = requests.get(
            f"https://api.tensorone.ai/v1/clusters/{cluster_id}",
            headers={"Authorization": f"Bearer {API_KEY}"}
        )
        
        cluster = response.json()["data"]
        status = cluster["status"]
        
        if status == "running":
            print(f"✅ Cluster {cluster_id} is ready!")
            print(f"Access URL: {cluster['network']['proxy_url']}")
            break
        elif status == "starting":
            # Show progress if available
            if "startup_progress" in cluster:
                progress = cluster["startup_progress"]
                print(f"🔄 {progress['current_step']} ({progress['percentage']}%)")
            else:
                print(f"🔄 Starting cluster... (Status: {status})")
        elif status == "error":
            print(f"❌ Cluster startup failed: {cluster.get('error_message', 'Unknown error')}")
            break
        
        time.sleep(10)  # Check every 10 seconds

def start_dev_environment(user_id, project_id):
    """Start a development environment for a user"""
    
    # Find user's development cluster
    clusters_response = requests.get(
        "https://api.tensorone.ai/v1/clusters",
        headers={"Authorization": f"Bearer {API_KEY}"},
        params={
            "project_id": project_id,
            "search": f"dev-{user_id}",
            "status": "stopped"
        }
    )
    
    clusters = clusters_response.json()["data"]["clusters"]
    
    if not clusters:
        return {"error": "No development cluster found for user"}
    
    cluster_id = clusters[0]["id"]
    
    # Start with updated environment
    start_config = {
        "wait_for_ready": True,
        "timeout_minutes": 10,
        "environment_updates": {
            "USER_ID": user_id,
            "PROJECT_ID": project_id,
            "WORKSPACE": f"/workspace/{user_id}"
        },
        "auto_terminate_updates": {
            "enabled": True,
            "idle_minutes": 120  # 2 hours for dev work
        }
    }
    
    response = requests.post(
        f"https://api.tensorone.ai/v1/clusters/{cluster_id}/start",
        headers={"Authorization": f"Bearer {API_KEY}"},
        json=start_config
    )
    
    return response.json()

async function resumeTrainingJob(jobId, checkpointPath, newConfig = {}) {
  // Find training cluster for this job
  const clustersResponse = await fetch(`https://api.tensorone.ai/v1/clusters?search=training-${jobId}&status=stopped`, {
    headers: { 'Authorization': 'Bearer ' + API_KEY }
  });
  
  const clusters = await clustersResponse.json();
  
  if (!clusters.success || clusters.data.clusters.length === 0) {
    throw new Error(`No training cluster found for job ${jobId}`);
  }
  
  const clusterId = clusters.data.clusters[0].id;
  
  const startConfig = {
    wait_for_ready: true,
    timeout_minutes: 15,
    environment_updates: {
      RESUME_FROM_CHECKPOINT: checkpointPath,
      JOB_ID: jobId,
      ...newConfig.environment
    },
    update_configuration: {
      ...newConfig.hardware
    },
    auto_terminate_updates: {
      enabled: true,
      cost_limit_usd: newConfig.costLimit || 500.0,
      idle_minutes: 30
    }
  };
  
  const response = await fetch(`https://api.tensorone.ai/v1/clusters/${clusterId}/start`, {
    method: 'POST',
    headers: {
      'Authorization': 'Bearer ' + API_KEY,
      'Content-Type': 'application/json'
    },
    body: JSON.stringify(startConfig)
  });
  
  const result = await response.json();
  
  if (result.success) {
    return {
      clusterId: result.data.id,
      jobId: jobId,
      accessUrl: result.data.network.proxy_url,
      sshCommand: result.data.network.ssh_connection.connection_string,
      estimatedCost: result.data.cost.hourly_rate
    };
  }
  
  throw new Error(`Failed to resume training job: ${result.error.message}`);
}

def scheduled_cluster_start(cluster_configs, schedule_time=None):
    """Start multiple clusters for scheduled batch jobs"""
    
    import schedule
    import threading
    
    def start_batch_clusters():
        results = []
        
        for config in cluster_configs:
            cluster_id = config["cluster_id"]
            updates = config.get("updates", {})
            
            start_payload = {
                "wait_for_ready": False,
                "update_configuration": updates.get("hardware", {}),
                "environment_updates": {
                    "BATCH_JOB_ID": config.get("job_id"),
                    "START_TIME": datetime.now().isoformat(),
                    **updates.get("environment", {})
                },
                "auto_terminate_updates": {
                    "enabled": True,
                    "max_runtime_hours": config.get("max_runtime_hours", 8),
                    "cost_limit_usd": config.get("cost_limit", 100.0)
                }
            }
            
            response = requests.post(
                f"https://api.tensorone.ai/v1/clusters/{cluster_id}/start",
                headers={"Authorization": f"Bearer {API_KEY}"},
                json=start_payload
            )
            
            results.append({
                "cluster_id": cluster_id,
                "job_id": config.get("job_id"),
                "success": response.json().get("success", False),
                "response": response.json()
            })
        
        return results
    
    if schedule_time:
        schedule.every().day.at(schedule_time).do(start_batch_clusters)
        
        def run_scheduler():
            while True:
                schedule.run_pending()
                time.sleep(60)
        
        scheduler_thread = threading.Thread(target=run_scheduler)
        scheduler_thread.daemon = True
        scheduler_thread.start()
    else:
        return start_batch_clusters()