fix: orchestrator logs (#266)

* do not return full training log

* revert 0.01 epochs

* changelog

* update cluster resources

CHANGELOG.md

@@ -18,7 +18,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### **Changed**
 - changed `ray-image` to pull from AWS Public ECR to avoid docker pull rate limits
-- changed `ray-orchestrator` sample script epochs to 0.01 to support demo scenarios
+- changed `ray-orchestrator` to not retrieve full training job logs and avoid `States.DataLimitExceeded`
+- update `ray-on-eks` manifest cluster resources
 
 ## v1.7.0
 

manifests/ray-on-eks/core-modules.yaml

@@ -44,9 +44,9 @@ parameters:
           eks_node_labels:
             usage: core
         - eks_ng_name: ng-gpu
-          eks_node_quantity: 1
+          eks_node_quantity: 5
           eks_node_max_quantity: 10
-          eks_node_min_quantity: 1
+          eks_node_min_quantity: 5
           eks_node_disk_size: 400
           eks_node_instance_type: "g4dn.4xlarge"
           eks_node_labels:

manifests/ray-on-eks/ray-cluster-modules.yaml

@@ -31,11 +31,11 @@ parameters:
   - name: HeadResources
     value:
       requests:
-        cpu: "1"
-        memory: "8G"
+        cpu: "8"
+        memory: "24G"
       limits:
-        cpu: "4"
-        memory: "16G"
+        cpu: "8"
+        memory: "24G"
   - name: WorkerReplicas
     value: 1
   - name: WorkerMinReplicas
@@ -45,8 +45,8 @@ parameters:
   - name: WorkerResources
     value:
       requests:
-        cpu: "4"
-        memory: "8G"
+        cpu: "14"
+        memory: "60G"
       limits:
         cpu: "14"
         memory: "60G"

modules/eks/ray-orchestrator/ray_orchestrator_stack.py

@@ -200,7 +200,7 @@ def __init__(
                     "Namespace": namespace_name,
                     "CertificateAuthority": eks_cert_auth_data,
                     "Endpoint": eks_cluster_endpoint,
-                    "LogOptions": {"RetrieveLogs": True},
+                    "LogOptions": {"RetrieveLogs": False},
                     "Job": training_body,
                 },
             },

modules/eks/ray-orchestrator/scripts/training-6B.py

@@ -205,7 +205,7 @@ def compute_metrics(eval_pred):
 trainer = TorchTrainer(
     train_loop_per_worker=train_func,
     train_loop_config={
-        "epochs": 0.01,
+        "epochs": 1,
         "batch_size": batch_size,  # per device
         "steps_per_epoch": steps_per_epoch,
     },