diff --git a/.github/workflows/armada.yml b/.github/workflows/armada.yml index f45d3d50559c..ba7dba5ebbe1 100644 --- a/.github/workflows/armada.yml +++ b/.github/workflows/armada.yml @@ -11,11 +11,37 @@ jobs: - uses: actions/checkout@v4 with: path: spark-armada + - uses: actions/checkout@v4 + with: + repository: richscott/armada + ref: rich/scala-build-container2 + path: armada - uses: actions/checkout@v4 with: repository: armadaproject/armada-operator path: armada-operator - - run: | + - name: Install Scala and SBT + env: + SBT_VERSION: 1.10.7 + SCALA_VERSION: 2.13.15 + run: | + curl -s -O https://downloads.lightbend.com/scala/${SCALA_VERSION}/scala-${SCALA_VERSION}.deb + sudo apt install -y ./scala-${SCALA_VERSION}.deb + curl -s -L -O https://github.com/sbt/sbt/releases/download/v${SBT_VERSION}/sbt-${SBT_VERSION}.tgz + tar -xzvf ./sbt-${SBT_VERSION}.tgz + echo "$PWD/sbt/bin" >> "${GITHUB_PATH}" + - name: Build Armada Scala Client + run: | + cd armada + docker build -t armada-scala-client-builder -f build/scala-client/Dockerfile . + echo docker run --rm -u $(id -u):$(id -u) -v "$PWD:/build" -w /build armada-scala-client-builder + docker run --rm -u $(id -u):$(id -u) -v "$PWD:/build" -w /build armada-scala-client-builder + cd .. + + cd armada/client/scala/scala-armada-client + sbt -Dsbt.io.implicit.relative.glob.conversion=allow publishM2 + - name: Build Spark Docker image + run: | cd spark-armada ./build/sbt package -Parmada -Pkubernetes ./bin/docker-image-tool.sh -t testing build @@ -26,6 +52,8 @@ jobs: make kind-all ./bin/tooling/kind load image-archive ../spark_testing.tar --name armada + - name: Submit Spark app + run: | # sleep a bit, or we see: create queue request failed: rpc error: code = DeadlineExceeded sleep 60 @@ -34,13 +62,13 @@ jobs: # sleep a bit, or we see: rpc error: code = PermissionDenied desc = could not find queue "test" sleep 60 - ./bin/app/armadactl submit ../spark-armada/examples/spark-driver-job.yaml - ./bin/app/armadactl submit ../spark-armada/examples/spark-executor-job.yaml + ./bin/app/armadactl submit ../spark-armada/examples/spark-pi-driver.yaml # wait for the jobs to start sleep 60 - # inspect jobs + - name: Inspect k8s pods + run: | kubectl get pods for pod in $(kubectl get pods | grep armada | cut -d " " -f 1) do