Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 8 additions & 8 deletions Makefile.rhai
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ SED ?= sed
KUBECTL ?= $(shell command -v oc 2>/dev/null || command -v kubectl 2>/dev/null)
RHOAI_MANIFESTS_DIR ?= $(PROJECT_DIR)/manifests/rhoai
RHAI_TEST_MANIFESTS_DIR ?= $(PROJECT_DIR)/odh_utils/tests/manifests
APPLICATIONS_NAMESPACE ?= opendatahub
APPLICATIONS_NAMESPACE ?= redhat-ods-applications
NFS_PROVISIONER_OPERATOR_MANIFEST ?= $(RHAI_TEST_MANIFESTS_DIR)/nfs-provisioner-operator.yaml
NFS_PROVISIONER_MANIFEST ?= $(RHAI_TEST_MANIFESTS_DIR)/nfs-provisioner.yaml
TRAINER_IMAGE ?=
Expand Down Expand Up @@ -260,19 +260,19 @@ deploy-rhoai: install-jobset-operator ## Install Trainer using RHOAI manifests w
echo "Using Trainer image: $(TRAINER_IMAGE)"
$(SED) -i "s|odh-kubeflow-trainer-controller-image=.*|odh-kubeflow-trainer-controller-image=$(TRAINER_IMAGE)|" "$(RHOAI_MANIFESTS_DIR)/params.env"
fi
@echo "Deploying RHOAI Training Operator to namespace: opendatahub"
$(KUBECTL) create namespace opendatahub --dry-run=client -o yaml | $(KUBECTL) apply -f - >/dev/null 2>&1
$(KUBECTL) label namespace opendatahub openshift.io/cluster-monitoring=true --overwrite
@echo "Deploying RHOAI Training Operator to namespace: $(APPLICATIONS_NAMESPACE)"
$(KUBECTL) create namespace $(APPLICATIONS_NAMESPACE) --dry-run=client -o yaml | $(KUBECTL) apply -f - >/dev/null 2>&1
$(KUBECTL) label namespace $(APPLICATIONS_NAMESPACE) openshift.io/cluster-monitoring=true --overwrite
# The PR environment deploys Trainer without the full RHOAI operator bundle,
# so provide the namespace-scoped discovery permissions normally installed by RHOAI.
$(KUBECTL) create role prometheus-k8s-monitoring \
--verb=get --verb=list --verb=watch \
--resource=pods,services,endpoints \
-n opendatahub --dry-run=client -o yaml | $(KUBECTL) apply -f - >/dev/null
-n $(APPLICATIONS_NAMESPACE) --dry-run=client -o yaml | $(KUBECTL) apply -f - >/dev/null
$(KUBECTL) create rolebinding prometheus-k8s-monitoring \
--role=prometheus-k8s-monitoring \
--serviceaccount=openshift-monitoring:prometheus-k8s \
-n opendatahub --dry-run=client -o yaml | $(KUBECTL) apply -f - >/dev/null
-n $(APPLICATIONS_NAMESPACE) --dry-run=client -o yaml | $(KUBECTL) apply -f - >/dev/null
echo "Applying CRDs first..."
$(KUBECTL) apply --server-side=true -k $(RHOAI_MANIFESTS_DIR)/../base/crds
echo "Waiting for CRDs to be established..."
Expand All @@ -284,7 +284,7 @@ deploy-rhoai: install-jobset-operator ## Install Trainer using RHOAI manifests w
$(KUBECTL) apply -k $(RHOAI_MANIFESTS_DIR) --server-side=true --force-conflicts
echo "Waiting for deployment to be ready..."
$(KUBECTL) wait --for=condition=available --timeout=300s \
deployment/kubeflow-trainer-controller-manager -n opendatahub 2>/dev/null
deployment/kubeflow-trainer-controller-manager -n $(APPLICATIONS_NAMESPACE) 2>/dev/null
echo "RHOAI Training Operator deployed successfully!"

.PHONY: install-test-runtimes
Expand Down Expand Up @@ -583,7 +583,7 @@ test-integration-rhai: ## Run the complete RHOAI Trainer integration test suite.

.PHONY: undeploy-rhoai
undeploy-rhoai: check-kubectl ## Undeploy Trainer using RHOAI manifests.
@echo "Undeploying RHOAI Training Operator from namespace: opendatahub"
@echo "Undeploying RHOAI Training Operator from namespace: $(APPLICATIONS_NAMESPACE)"
echo "Deleting all TrainJob and TrainingRuntime CRs..."
$(KUBECTL) delete trainjobs.trainer.kubeflow.org --all --all-namespaces --ignore-not-found=true --timeout=60s
$(KUBECTL) delete trainingruntimes.trainer.kubeflow.org --all --all-namespaces --ignore-not-found=true --timeout=60s
Expand Down
2 changes: 1 addition & 1 deletion manifests/rhoai/kustomization.yaml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# RHOAI configuration for Kubeflow Training Operator (KFTO)

# Adds namespace to all resources.
namespace: opendatahub
namespace: redhat-ods-applications

sortOptions:
order: legacy
Expand Down
8 changes: 7 additions & 1 deletion test/e2e/e2e_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,9 @@ var _ = ginkgo.Describe("TrainJob e2e", func() {
// Create a TrainJob.
trainJob := testingutil.MakeTrainJobWrapper(ns.Name, "e2e-test-torch").
RuntimeRef(trainer.SchemeGroupVersion.WithKind(trainer.ClusterTrainingRuntimeKind), torchRuntime).
Trainer(&trainer.Trainer{
Command: []string{"python", "-c", "print(\"Trainer E2E\")"},
}).
Obj()

ginkgo.By("Create a TrainJob with torch-distributed runtime reference", func() {
Expand Down Expand Up @@ -107,7 +110,10 @@ var _ = ginkgo.Describe("TrainJob e2e", func() {
ginkgo.When("Creating TrainJob to perform OpenMPI workload", func() {
// Verify the `deepspeed-distributed` ClusterTrainingRuntime.
ginkgo.It("should create TrainJob with DeepSpeed runtime reference", func() {
// Create a TrainJob.
// TODO: Remove this skip once an OpenShift-compatible MPI runtime image is available.
ginkgo.Skip("MPI runtime image is not available")

// Create a multi-node MPI TrainJob.
trainJob := testingutil.MakeTrainJobWrapper(ns.Name, "e2e-test-deepspeed").
RuntimeRef(trainer.SchemeGroupVersion.WithKind(trainer.ClusterTrainingRuntimeKind), deepSpeedRuntime).
Obj()
Expand Down
Loading