diff --git a/.drone.yml b/.drone.yml deleted file mode 100644 index 9d86283f295d..000000000000 --- a/.drone.yml +++ /dev/null @@ -1,571 +0,0 @@ ---- -kind: pipeline -name: amd64 - -platform: - os: linux - arch: amd64 - -trigger: - event: - exclude: - - cron - - pull_request - -clone: - retries: 3 - -steps: -- name: build - image: rancher/dapper:v0.6.0 - secrets: [ AWS_SECRET_ACCESS_KEY-k3s-ci-uploader, AWS_ACCESS_KEY_ID-k3s-ci-uploader, unprivileged_github_token ] - environment: - GITHUB_TOKEN: - from_secret: unprivileged_github_token - AWS_SECRET_ACCESS_KEY: - from_secret: AWS_SECRET_ACCESS_KEY-k3s-ci-uploader - AWS_ACCESS_KEY_ID: - from_secret: AWS_ACCESS_KEY_ID-k3s-ci-uploader - commands: - - dapper ci - - echo "${DRONE_TAG}-amd64" | sed -e 's/+/-/g' >.tags - volumes: - - name: docker - path: /var/run/docker.sock - -- name: fossa - image: rancher/drone-fossa:latest - failure: ignore - settings: - api_key: - from_secret: FOSSA_API_KEY - when: - instance: - - drone-publish.k3s.io - ref: - include: - - "refs/heads/master" - - "refs/heads/release-*" - event: - - push - - tag -- name: docker-publish - image: plugins/docker - settings: - dockerfile: package/Dockerfile - password: - from_secret: docker_password - repo: "rancher/k3s" - username: - from_secret: docker_username - build_args_from_env: - - DRONE_TAG - when: - instance: - - drone-publish.k3s.io - ref: - - refs/head/master - - refs/tags/* - event: - - tag - -- name: test - image: rancher/dapper:v0.6.0 - secrets: [ AWS_SECRET_ACCESS_KEY-k3s-ci-uploader, AWS_ACCESS_KEY_ID-k3s-ci-uploader ] - environment: - ENABLE_REGISTRY: 'true' - AWS_SECRET_ACCESS_KEY: - from_secret: AWS_SECRET_ACCESS_KEY-k3s-ci-uploader - AWS_ACCESS_KEY_ID: - from_secret: AWS_ACCESS_KEY_ID-k3s-ci-uploader - commands: - - docker build --target test-k3s -t k3s:test-${DRONE_STAGE_ARCH}-${DRONE_COMMIT} -f Dockerfile.test . - - > - docker run -i -e REPO -e TAG -e DRONE_TAG -e DRONE_BUILD_EVENT -e IMAGE_NAME -e AWS_SECRET_ACCESS_KEY -e AWS_ACCESS_KEY_ID -e SONOBUOY_VERSION -e ENABLE_REGISTRY - -v /var/run/docker.sock:/var/run/docker.sock --privileged --network host -v /tmp:/tmp k3s:test-${DRONE_STAGE_ARCH}-${DRONE_COMMIT} - volumes: - - name: docker - path: /var/run/docker.sock - -volumes: -- name: docker - host: - path: /var/run/docker.sock - ---- -kind: pipeline -name: conformance - -platform: - os: linux - arch: amd64 - -trigger: - event: - - cron - cron: - - nightly - -steps: -- name: build - image: rancher/dapper:v0.6.0 - commands: - - dapper ci - - echo "${DRONE_TAG}-amd64" | sed -e 's/+/-/g' >.tags - volumes: - - name: docker - path: /var/run/docker.sock - -- name: test - image: rancher/dapper:v0.6.0 - environment: - ENABLE_REGISTRY: 'true' - commands: - - docker build --target test-k3s -t k3s:test-${DRONE_STAGE_ARCH}-${DRONE_COMMIT} -f Dockerfile.test . - - > - docker run -i -e REPO -e TAG -e DRONE_TAG -e DRONE_BUILD_EVENT -e IMAGE_NAME -e SONOBUOY_VERSION -e ENABLE_REGISTRY - -v /var/run/docker.sock:/var/run/docker.sock --privileged --network host -v /tmp:/tmp k3s:test-${DRONE_STAGE_ARCH}-${DRONE_COMMIT} - volumes: - - name: docker - path: /var/run/docker.sock - -volumes: -- name: docker - host: - path: /var/run/docker.sock - ---- -kind: pipeline -name: arm64 - -platform: - os: linux - arch: arm64 - -trigger: - event: - exclude: - - cron - - pull_request - -clone: - retries: 3 - -steps: -- name: build - image: rancher/dapper:v0.6.0 - secrets: [ AWS_SECRET_ACCESS_KEY-k3s-ci-uploader, AWS_ACCESS_KEY_ID-k3s-ci-uploader ] - environment: - AWS_SECRET_ACCESS_KEY: - from_secret: AWS_SECRET_ACCESS_KEY-k3s-ci-uploader - AWS_ACCESS_KEY_ID: - from_secret: AWS_ACCESS_KEY_ID-k3s-ci-uploader - commands: - - dapper ci - - echo "${DRONE_TAG}-arm64" | sed -e 's/+/-/g' >.tags - volumes: - - name: docker - path: /var/run/docker.sock - -- name: docker-publish - image: plugins/docker - settings: - dockerfile: package/Dockerfile - password: - from_secret: docker_password - repo: "rancher/k3s" - username: - from_secret: docker_username - build_args_from_env: - - DRONE_TAG - when: - instance: - - drone-publish.k3s.io - ref: - - refs/head/master - - refs/tags/* - event: - - tag - -- name: test - image: rancher/dapper:v0.6.0 - secrets: [ AWS_SECRET_ACCESS_KEY-k3s-ci-uploader, AWS_ACCESS_KEY_ID-k3s-ci-uploader ] - environment: - ENABLE_REGISTRY: 'true' - AWS_SECRET_ACCESS_KEY: - from_secret: AWS_SECRET_ACCESS_KEY-k3s-ci-uploader - AWS_ACCESS_KEY_ID: - from_secret: AWS_ACCESS_KEY_ID-k3s-ci-uploader - commands: - - docker build --target test-k3s -t k3s:test-${DRONE_STAGE_ARCH}-${DRONE_COMMIT} -f Dockerfile.test . - - > - docker run -i -e REPO -e TAG -e DRONE_TAG -e DRONE_BUILD_EVENT -e IMAGE_NAME -e AWS_SECRET_ACCESS_KEY -e AWS_ACCESS_KEY_ID -e SONOBUOY_VERSION -e ENABLE_REGISTRY - -v /var/run/docker.sock:/var/run/docker.sock --privileged --network host -v /tmp:/tmp k3s:test-${DRONE_STAGE_ARCH}-${DRONE_COMMIT} - volumes: - - name: docker - path: /var/run/docker.sock - -volumes: -- name: docker - host: - path: /var/run/docker.sock - ---- -kind: pipeline -name: arm - -platform: - os: linux - arch: arm - -trigger: - event: - exclude: - - cron - -clone: - retries: 3 - -steps: -- name: skipfiles - image: plugins/git - commands: - - export NAME=$(test $DRONE_BUILD_EVENT = pull_request && echo remotes/origin/${DRONE_COMMIT_BRANCH:-master} || echo ${DRONE_COMMIT_SHA}~) - - export DIFF=$(git --no-pager diff --name-only $NAME | grep -v -f .droneignore); - - if [ -z "$DIFF" ]; then - echo "All files in PR are on ignore list"; - exit 78; - else - echo "Some files in PR are not ignored, $DIFF"; - fi; - when: - event: - - pull_request - -- name: build - # Keeping Dapper at v0.5.0 for armv7, as newer versions fails with - # Bad system call on this architecture. xref: - # - # https://github.com/k3s-io/k3s/pull/8959#discussion_r1439736566 - # https://drone-pr.k3s.io/k3s-io/k3s/7922/3/3 - image: rancher/dapper:v0.5.0 - secrets: [ AWS_SECRET_ACCESS_KEY-k3s-ci-uploader, AWS_ACCESS_KEY_ID-k3s-ci-uploader ] - environment: - AWS_SECRET_ACCESS_KEY: - from_secret: AWS_SECRET_ACCESS_KEY-k3s-ci-uploader - AWS_ACCESS_KEY_ID: - from_secret: AWS_ACCESS_KEY_ID-k3s-ci-uploader - commands: - - dapper ci - - echo "${DRONE_TAG}-arm" | sed -e 's/+/-/g' >.tags - volumes: - - name: docker - path: /var/run/docker.sock -- name: docker-publish - image: plugins/docker:linux-arm - settings: - dockerfile: package/Dockerfile - password: - from_secret: docker_password - repo: "rancher/k3s" - username: - from_secret: docker_username - build_args_from_env: - - DRONE_TAG - when: - instance: - - drone-publish.k3s.io - ref: - - refs/head/master - - refs/tags/* - event: - - tag - -- name: test - # Refer to comment for arm/build. - image: rancher/dapper:v0.5.0 - secrets: [ AWS_SECRET_ACCESS_KEY-k3s-ci-uploader, AWS_ACCESS_KEY_ID-k3s-ci-uploader ] - environment: - ENABLE_REGISTRY: 'true' - AWS_SECRET_ACCESS_KEY: - from_secret: AWS_SECRET_ACCESS_KEY-k3s-ci-uploader - AWS_ACCESS_KEY_ID: - from_secret: AWS_ACCESS_KEY_ID-k3s-ci-uploader - commands: - - docker build --target test-k3s -t k3s:test-${DRONE_STAGE_ARCH}-${DRONE_COMMIT} -f Dockerfile.test . - - > - docker run -i -e REPO -e TAG -e DRONE_TAG -e DRONE_BUILD_EVENT -e IMAGE_NAME -e AWS_SECRET_ACCESS_KEY -e AWS_ACCESS_KEY_ID -e SONOBUOY_VERSION -e ENABLE_REGISTRY - -v /var/run/docker.sock:/var/run/docker.sock --privileged --network host -v /tmp:/tmp k3s:test-${DRONE_STAGE_ARCH}-${DRONE_COMMIT} - volumes: - - name: docker - path: /var/run/docker.sock - -volumes: -- name: docker - host: - path: /var/run/docker.sock - ---- -kind: pipeline -name: manifest - -platform: - os: linux - arch: amd64 - -steps: -- name: skipfiles - image: plugins/git - commands: - - export NAME=$(test $DRONE_BUILD_EVENT = pull_request && echo remotes/origin/${DRONE_COMMIT_BRANCH:-master} || echo ${DRONE_COMMIT_SHA}~) - - export DIFF=$(git --no-pager diff --name-only $NAME | grep -v -f .droneignore); - - if [ -z "$DIFF" ]; then - echo "All files in PR are on ignore list"; - exit 78; - else - echo "Some files in PR are not ignored, $DIFF"; - fi; - when: - event: - - push - - pull_request - -- name: manifest - image: plugins/docker - environment: - DOCKER_USERNAME: - from_secret: docker_username - DOCKER_PASSWORD: - from_secret: docker_password - settings: - dry_run: true - dockerfile: Dockerfile.manifest - repo: "rancher/k3s-manifest" - build_args_from_env: - - DOCKER_USERNAME - - DOCKER_PASSWORD - - DRONE_TAG -trigger: - instance: - - drone-publish.k3s.io - ref: - - refs/head/master - - refs/tags/* - event: - include: - - tag - exclude: - - cron - -depends_on: -- amd64 -- arm64 -- arm - ---- -kind: pipeline -name: dispatch - -platform: - os: linux - arch: amd64 - -clone: - retries: 3 - -steps: -- name: skipfiles - image: plugins/git - commands: - - export NAME=$(test $DRONE_BUILD_EVENT = pull_request && echo remotes/origin/${DRONE_COMMIT_BRANCH:-master} || echo ${DRONE_COMMIT_SHA}~) - - export DIFF=$(git --no-pager diff --name-only $NAME | grep -v -f .droneignore); - - if [ -z "$DIFF" ]; then - echo "All files in PR are on ignore list"; - exit 78; - else - echo "Some files in PR are not ignored, $DIFF"; - fi; - when: - event: - - push - - pull_request - -- name: dispatch - image: curlimages/curl:7.74.0 - secrets: [ pat_username, github_token, release_token_k3s ] - user: root - environment: - PAT_USERNAME: - from_secret: pat_username - PAT_TOKEN: - from_secret: github_token - K3S_RELEASE_TOKEN: - from_secret: release_token_k3s - commands: - - apk -U --no-cache add bash - - scripts/dispatch - -trigger: - instance: - - drone-publish.k3s.io - ref: - - refs/head/master - - refs/tags/* - event: - - tag - -depends_on: -- manifest - ---- -kind: pipeline -name: e2e -type: docker - -platform: - os: linux - arch: amd64 - -clone: - retries: 3 - -steps: -- name: skipfiles - image: plugins/git - commands: - - export NAME=$(test $DRONE_BUILD_EVENT = pull_request && echo remotes/origin/${DRONE_COMMIT_BRANCH:-master} || echo ${DRONE_COMMIT_SHA}~) - - export DIFF=$(git --no-pager diff --name-only $NAME | grep -v -f .droneignore); - - if [ -z "$DIFF" ]; then - echo "All files in PR are on ignore list"; - exit 78; - else - echo "Some files in PR are not ignored, $DIFF"; - fi; - when: - event: - - push - - pull_request - -- name: build-e2e-image - image: docker:25.0.5 - commands: - - DOCKER_BUILDKIT=1 docker build --target test-e2e -t test-e2e -f Dockerfile.test . - - apk add make git bash - - GOCOVER=1 make local-binary - - cp dist/artifacts/* /tmp/artifacts/ - volumes: - - name: cache - path: /tmp/artifacts - - name: docker - path: /var/run/docker.sock - -- name: test-e2e-validatecluster - depends_on: - - build-e2e-image - image: test-e2e - pull: never - resources: - cpu: 6000 - memory: 10Gi - environment: - E2E_REGISTRY: 'true' - E2E_GOCOVER: 'true' - commands: - - mkdir -p dist/artifacts - - cp /tmp/artifacts/* dist/artifacts/ - # Cleanup VMs that are older than 2h. Happens if a previous test panics or is canceled - - tests/e2e/scripts/cleanup_vms.sh - - tests/e2e/scripts/drone_registries.sh - # Stagger the launch of this test with the parallel splitserver test - # to prevent conflicts over libvirt network interfaces - - | - cd tests/e2e/validatecluster - ../scripts/cleanup_vms.sh 'validatecluster_([0-9]+)_(server|agent)' - sleep 15 - go test -v -timeout=45m ./validatecluster_test.go -ci -local - cp ./coverage.out /tmp/artifacts/validate-coverage.out - volumes: - - name: libvirt - path: /var/run/libvirt/ - - name: docker - path: /var/run/docker.sock - - name: cache - path: /tmp/artifacts - -- name: test-e2e-splitserver - depends_on: - - build-e2e-image - image: test-e2e - pull: never - resources: - cpu: 6000 - memory: 10Gi - environment: - E2E_REGISTRY: 'true' - E2E_GOCOVER: 'true' - commands: - - mkdir -p dist/artifacts - - cp /tmp/artifacts/* dist/artifacts/ - - tests/e2e/scripts/drone_registries.sh - - | - cd tests/e2e/splitserver - ../scripts/cleanup_vms.sh 'splitserver_([0-9]+)' - go test -v -timeout=30m ./splitserver_test.go -ci -local - cp ./coverage.out /tmp/artifacts/split-coverage.out - - | - if [ "$DRONE_BUILD_EVENT" = "pull_request" ]; then - cd ../upgradecluster - ../scripts/cleanup_vms.sh 'upgradecluster_([0-9]+)_(server|agent)' - # Convert release-1.XX branch to v1.XX channel - if [ "$DRONE_BRANCH" = "master" ]; then - UPGRADE_CHANNEL="latest" - else - UPGRADE_CHANNEL=$(echo $DRONE_BRANCH | sed 's/release-/v/') - # Check if the UPGRADE_CHANNEL exists, in the case of new minor releases it won't - if ! curl --head --silent --fail https://update.k3s.io/v1-release/channels/$UPGRADE_CHANNEL; then - UPGRADE_CHANNEL="latest" - fi - fi - E2E_RELEASE_CHANNEL=$UPGRADE_CHANNEL go test -v -timeout=45m ./upgradecluster_test.go -ci -local -ginkgo.v - cp ./coverage.out /tmp/artifacts/upgrade-coverage.out - fi - - volumes: - - name: libvirt - path: /var/run/libvirt/ - - name: docker - path: /var/run/docker.sock - - name: cache - path: /tmp/artifacts - -- name: upload to codecov - depends_on: - - test-e2e-validatecluster - - test-e2e-splitserver - image: robertstettner/drone-codecov - settings: - token: - from_secret: codecov_token - files: - - /tmp/artifacts/validate-coverage.out - - /tmp/artifacts/split-coverage.out - - /tmp/artifacts/upgrade-coverage.out - flags: - - e2etests - when: - event: - - push - - volumes: - - name: cache - path: /tmp/artifacts - -volumes: -- name: docker - host: - path: /var/run/docker.sock -- name: libvirt - host: - path: /var/run/libvirt/ -- name: cache - temp: {} diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 8b4155e8ef25..244ca38e66c6 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -51,7 +51,8 @@ jobs: strategy: fail-fast: false matrix: - etest: [btrfs, embeddedmirror, externalip, privateregistry, rootless, s3, startup, wasm] + # etest: [btrfs, embeddedmirror, externalip, privateregistry, rootless, s3, startup, wasm] + etest: [ multus ] max-parallel: 5 steps: - name: "Checkout" @@ -116,102 +117,4 @@ jobs: flags: e2etests # optional verbose: true # optional (default = false) - build-go-tests: - name: "Build Go Tests" - strategy: - matrix: - arch: [amd64, arm64] - runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} - outputs: - channel: ${{ steps.channel_step.outputs.channel }} - steps: - - name: Checkout - uses: actions/checkout@v5 - - name: Install Go - uses: ./.github/actions/setup-go - - name: Build Go Tests - run: | - mkdir -p ./dist/artifacts - go test -c -ldflags="-w -s" -o ./dist/artifacts ./tests/docker/... - - name: Upload Go Tests - uses: actions/upload-artifact@v4 - with: - name: docker-go-tests-${{ matrix.arch }} - path: ./dist/artifacts/*.test - compression-level: 9 - retention-days: 1 - # For upgrade and skew tests, we need to know the channel this run is based off. - # Since this is predetermined, we can run this step before the actual test job, saving time. - - name: Determine channel - id: channel_step - run: | - . ./scripts/version.sh - MINOR_VER=$(echo $VERSION_TAG | cut -d'.' -f1,2) - echo "CHANNEL=$MINOR_VER" >> $GITHUB_OUTPUT - # channel name should be v1.XX or latest - - name: Fail if channel name does not match pattern - run: | - if [[ ! ${{ steps.channel_step.outputs.channel }} =~ ^v1\.[0-9]+$|latest$ ]]; then - echo "Channel name ${{ steps.channel_step.outputs.channel }} does not match pattern" - exit 1 - fi - - docker-go: - needs: [build, build-arm64, build-go-tests] - name: Docker - timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - dtest: [autoimport, basics, bootstraptoken, cacerts, etcd, hardened, lazypull, skew, secretsencryption, snapshotrestore, svcpoliciesandfirewall, token, upgrade] - arch: [amd64, arm64] - exclude: - - dtest: autoimport - arch: arm64 - - dtest: secretsencryption - arch: arm64 - - dtest: snapshotrestore - arch: arm64 - - dtest: svcpoliciesandfirewall - arch: arm64 - runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} - env: - CHANNEL: ${{ needs.build-go-tests.outputs.channel }} - steps: - - name: Checkout - uses: actions/checkout@v5 - - name: "Download K3s image" - uses: actions/download-artifact@v5 - with: - name: k3s-${{ matrix.arch }} - path: ./dist/artifacts - - name: Load and set K3s image - run: | - if [ ${{ matrix.arch }} = "arm64" ]; then - mv ./dist/artifacts/k3s-arm64 ./dist/artifacts/k3s - fi - chmod +x ./dist/artifacts/k3s - docker image load -i ./dist/artifacts/k3s-image.tar - IMAGE_TAG=$(docker image ls --format '{{.Repository}}:{{.Tag}}' | grep 'rancher/k3s') - echo "K3S_IMAGE=$IMAGE_TAG" >> $GITHUB_ENV - - name: Download Go Tests - uses: actions/download-artifact@v5 - with: - name: docker-go-tests-${{ matrix.arch }} - path: ./dist/artifacts - - name: Run ${{ matrix.dtest }} Test - # Put the compiled test binary back in the same place as the test source - run: | - chmod +x ./dist/artifacts/${{ matrix.dtest }}.test - mv ./dist/artifacts/${{ matrix.dtest }}.test ./tests/docker/${{ matrix.dtest }}/ - cd ./tests/docker/${{ matrix.dtest }} - - # These tests use rancher/systemd-node and have different flags. - CI_TESTS="autoimport hardened secretsencryption snapshotrestore svcpoliciesandfirewall token" - if [ ${{ matrix.dtest }} = "upgrade" ] || [ ${{ matrix.dtest }} = "skew" ]; then - ./${{ matrix.dtest }}.test -test.timeout=0 -test.v -ginkgo.v -k3sImage=$K3S_IMAGE -channel=$CHANNEL - elif [[ $CI_TESTS =~ ${{ matrix.dtest }} ]]; then - ./${{ matrix.dtest }}.test -test.timeout=0 -test.v -ginkgo.v -ci - else - ./${{ matrix.dtest }}.test -test.timeout=0 -test.v -ginkgo.v -k3sImage=$K3S_IMAGE - fi + \ No newline at end of file diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml deleted file mode 100644 index 180c078ba1a4..000000000000 --- a/.github/workflows/integration.yaml +++ /dev/null @@ -1,128 +0,0 @@ -name: Integration Test Coverage -on: - push: - paths-ignore: - - "**.md" - - "channel.yaml" - - "install.sh" - - "tests/**" - - "!tests/integration**" - - ".github/**" - - "!.github/workflows/integration.yaml" - pull_request: - paths-ignore: - - "**.md" - - "channel.yaml" - - "install.sh" - - "tests/**" - - "!tests/integration**" - - "!tests/e2e**" - - ".github/**" - - "!.github/workflows/integration.yaml" - - "!.github/workflows/build-k3s.yaml" - workflow_dispatch: {} - -permissions: - contents: read - -env: - GOCOVERDIR: /tmp/k3scov - -jobs: - build: - uses: ./.github/workflows/build-k3s.yaml - with: - os: linux - build-windows: - uses: ./.github/workflows/build-k3s.yaml - with: - os: windows - itest: - needs: build - name: Integration Tests - runs-on: ubuntu-latest - timeout-minutes: 45 - strategy: - fail-fast: false - matrix: - itest: [certrotation, cacertrotation, etcdrestore, localstorage, startup, custometcdargs, etcdsnapshot, kubeflags, longhorn, secretsencryption, flannelnone] - max-parallel: 3 - steps: - - name: Checkout - uses: actions/checkout@v5 - with: - fetch-depth: 1 - - name: Install Go - uses: ./.github/actions/setup-go - - name: "Download k3s binary" - uses: actions/download-artifact@v5 - with: - name: k3s-amd64 - path: ./dist/artifacts - - name: Run Integration Tests - run: | - chmod +x ./dist/artifacts/k3s - mkdir -p $GOCOVERDIR - sudo -E env "PATH=$PATH" go test -timeout=45m ./tests/integration/${{ matrix.itest }}/... -run Integration -ginkgo.v -test.v - - name: On Failure, Launch Debug Session - uses: lhotari/action-upterm@v1 - if: ${{ failure() }} - with: - ## If no one connects after 5 minutes, shut down server. - wait-timeout-minutes: 5 - - name: Generate coverage report - run: go tool covdata textfmt -i $GOCOVERDIR -o ${{ matrix.itest }}.out - - name: Upload Results To Codecov - uses: codecov/codecov-action@v5 - with: - token: ${{ secrets.CODECOV_TOKEN }} - files: ./${{ matrix.itest }}.out - flags: inttests # optional - verbose: true # optional (default = false) - itest-windows: - name: Integration Tests (windows) - needs: build-windows - runs-on: windows-2022 - timeout-minutes: 10 - env: - GOCOVERDIR: "D:/tmp/k3scov" - steps: - - name: Checkout - uses: actions/checkout@v5 - with: {fetch-depth: 1} - - name: Install Go - uses: ./.github/actions/setup-go - - name: Download k3s binary - uses: actions/download-artifact@v5 - with: - name: k3s-windows - path: dist/artifacts/ - - name: Run K3s - timeout-minutes: 5 - env: - CONTAINERD_LOG_LEVEL: "debug" - run: | - $ErrorActionPreference = "Continue" - $PSNativeCommandUseErrorActionPreference = $true - New-Item -Type Directory -Force $Env:GOCOVERDIR | Out-Null - $Server = Start-Job -ScriptBlock { ./dist/artifacts/k3s.exe server --token=token --debug --disable=metrics-server } - Start-Sleep -Seconds 15 - D:/var/lib/rancher/k3s/data/current/bin/k3s.exe kubectl apply -f ./tests/integration/startup/testdata/agnhost.yaml - D:/var/lib/rancher/k3s/data/current/bin/k3s.exe kubectl wait --for=jsonpath='{.status.phase}'=Running --timeout=5m pod/agnhost - D:/var/lib/rancher/k3s/data/current/bin/k3s.exe crictl ps - D:/var/lib/rancher/k3s/data/current/bin/k3s.exe kubectl get pod -A -o wide - D:/var/lib/rancher/k3s/data/current/bin/k3s.exe kubectl get node -o wide - $RET = $LASTEXITCODE - Stop-Job -Job $Server - Receive-Job -Wait -Job $Server - Remove-Job -Job $Server - exit $RET - - name: Generate coverage report - run: go tool covdata textfmt -i $Env:GOCOVERDIR -o windows.out - - name: Upload Results To Codecov - uses: codecov/codecov-action@v5 - with: - token: ${{ secrets.CODECOV_TOKEN }} - files: ./windows.out - flags: inttests # optional - verbose: true # optional (default = false) diff --git a/.github/workflows/unitcoverage.yaml b/.github/workflows/unitcoverage.yaml deleted file mode 100644 index e5a16de0e376..000000000000 --- a/.github/workflows/unitcoverage.yaml +++ /dev/null @@ -1,89 +0,0 @@ -name: Unit Test Coverage -on: - push: - paths-ignore: - - "**.md" - - "channel.yaml" - - "install.sh" - - "tests/snapshotter/**" - - "tests/install/**" - - "tests/cgroup/**" - - ".github/**" - - "!.github/workflows/unitcoverage.yaml" - pull_request: - paths-ignore: - - "**.md" - - "channel.yaml" - - "install.sh" - - "tests/snapshotter/**" - - "tests/install/**" - - "tests/cgroup/**" - - ".github/**" - - "!.github/workflows/unitcoverage.yaml" - workflow_dispatch: {} - -permissions: - contents: read - -jobs: - test-unit-linux: - name: Unit Tests (linux) - runs-on: ubuntu-24.04 - timeout-minutes: 20 - steps: - - name: Checkout - uses: actions/checkout@v5 - with: - fetch-depth: 1 - - name: Install Go - uses: ./.github/actions/setup-go - - name: Run Unit Tests - run: | - go test -coverpkg ./pkg/... -coverprofile coverage.out ./pkg/... -run Unit - go tool cover -func coverage.out - - name: On Failure, Launch Debug Session - if: ${{ failure() }} - uses: lhotari/action-upterm@v1 - with: - wait-timeout-minutes: 5 - - name: Upload Results To Codecov - uses: codecov/codecov-action@v5 - with: - token: ${{ secrets.CODECOV_TOKEN }} - files: ./coverage.out - flags: unittests # optional - verbose: true # optional (default = false) - test-unit-windows: - name: Unit Tests (windows) - runs-on: windows-2022 - timeout-minutes: 20 - steps: - - name: Checkout - uses: actions/checkout@v5 - with: - fetch-depth: 1 - - name: Install Go - uses: actions/setup-go@v5 - with: - cache: false - - name: Run Unit Tests - run: | - go test -coverpkg ./pkg/... -coverprofile coverage.out ./pkg/... -run Unit - go tool cover -func coverage.out - - name: Upload Results To Codecov - uses: codecov/codecov-action@v5 - with: - token: ${{ secrets.CODECOV_TOKEN }} - files: ./coverage.out - flags: unittests # optional - verbose: true # optional (default = false) - test-mods: - name: Test K8s Modules - runs-on: ubuntu-24.04 - steps: - - name: Checkout - uses: actions/checkout@v5 - - name: Build test-mods - run: docker build --target test-mods -t k3s:mod -f Dockerfile.test . - - name: Run test-mods - run: docker run -i k3s:mod \ No newline at end of file diff --git a/tests/e2e/amd64_resource_files/multus_network_attach.yaml b/tests/e2e/amd64_resource_files/multus_network_attach.yaml new file mode 100644 index 000000000000..10c95d3d2961 --- /dev/null +++ b/tests/e2e/amd64_resource_files/multus_network_attach.yaml @@ -0,0 +1,18 @@ +apiVersion: k8s.cni.cncf.io/v1 +kind: NetworkAttachmentDefinition +metadata: + name: macvlan-whereabouts +spec: + config: |- + { + "cniVersion": "1.0.0", + "type": "macvlan", + "master": "eth1", + "mode": "bridge", + "ipam": { + "type": "whereabouts", + "range": "172.17.0.0/24", + "gateway": "172.17.0.1", + "configuration_path": "/var/lib/rancher/k3s/agent/etc/cni/net.d/whereabouts.d/whereabouts.conf" + } + } diff --git a/tests/e2e/amd64_resource_files/multus_pod_client.yaml b/tests/e2e/amd64_resource_files/multus_pod_client.yaml new file mode 100644 index 000000000000..bdca0cbf0a11 --- /dev/null +++ b/tests/e2e/amd64_resource_files/multus_pod_client.yaml @@ -0,0 +1,42 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: multus-demo + labels: + app: multus-demo +spec: + replicas: 2 + selector: + matchLabels: + app: multus-demo + template: + metadata: + annotations: + k8s.v1.cni.cncf.io/networks: macvlan-whereabouts + labels: + app: multus-demo + spec: + containers: + - image: wbitt/network-multitool:extra + imagePullPolicy: Always + name: network-multitool + env: + - name: HTTP_PORT + value: "1180" + - name: HTTPS_PORT + value: "11443" + ports: + - containerPort: 1180 + name: http-port + - containerPort: 11443 + name: https-port + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - multus-demo + topologyKey: kubernetes.io/hostname \ No newline at end of file diff --git a/tests/e2e/multus/Vagrantfile b/tests/e2e/multus/Vagrantfile new file mode 100644 index 000000000000..50b64e0c3577 --- /dev/null +++ b/tests/e2e/multus/Vagrantfile @@ -0,0 +1,88 @@ +ENV['VAGRANT_NO_PARALLEL'] = 'no' +NODE_ROLES = (ENV['E2E_NODE_ROLES'] || + ["server-0", "agent-0" ]) +NODE_BOXES = (ENV['E2E_NODE_BOXES'] || + ['bento/ubuntu-24.04', 'bento/ubuntu-24.04']) +GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master") +RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "") +GOCOVER = (ENV['E2E_GOCOVER'] || "") +NODE_CPUS = (ENV['E2E_NODE_CPUS'] || 2).to_i +NODE_MEMORY = (ENV['E2E_NODE_MEMORY'] || 2048).to_i +NETWORK4_PREFIX = "10.10.10" +MULTUS_PREFIX = "192.168.1" +install_type = "" + +def provision(vm, role, role_num, node_num) + vm.box = NODE_BOXES[node_num] + vm.hostname = role + node_ip4 = "#{NETWORK4_PREFIX}.#{100+node_num}" + vm.network "private_network", :ip => node_ip4, :netmask => "255.255.255.0" + #2nd interface to be used by multus + # multus_ip4 = "#{MULTUS_PREFIX}.#{100+node_num}" + # vm.network "private_network", ip:multus_ip4, netmask: "255.255.255.0" + + scripts_location = Dir.exist?("./scripts") ? "./scripts" : "../scripts" + vagrant_defaults = File.exist?("./vagrantdefaults.rb") ? "./vagrantdefaults.rb" : "../vagrantdefaults.rb" + load vagrant_defaults + + defaultOSConfigure(vm) + addCoverageDir(vm, role, GOCOVER) + install_type = getInstallType(vm, RELEASE_VERSION, GITHUB_BRANCH) + + if role.include?("server") && role_num == 0 + vm.provision :k3s, run: 'once' do |k3s| + k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321 + k3s.args = "server " + k3s.config = <<~YAML + node-ip: #{node_ip4} + token: vagrant + flannel-iface: eth1 + YAML + k3s.env = ["K3S_KUBECONFIG_MODE=0644", install_type] + end + #install multus manifest + vm.provision "file", source: "rke2-multus-config.yaml", destination: "rke2-multus-config.yaml" + vm.provision "shell", inline: "sudo mkdir -p /var/lib/rancher/k3s/server/manifests/ && sudo cp rke2-multus-config.yaml /var/lib/rancher/k3s/server/manifests/" + end + if role.include?("agent") + vm.provision :k3s, run: 'once' do |k3s| + k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321 + k3s.args = "agent " + k3s.config = <<~YAML + server: https://#{NETWORK4_PREFIX}.100:6443 + token: vagrant + node-ip: #{node_ip4} + flannel-iface: eth1 + YAML + k3s.env = ["K3S_KUBECONFIG_MODE=0644", install_type] + end + end + # Make sure cluster traffic cannot flow via eth1 + vm.provision "Disable forwarding", type: "shell", inline: "sysctl -w net.ipv4.conf.eth1.forwarding=0" + # # Make sure cluster traffic cannot flow via eth1 + # vm.provision "Disable forwarding", type: "shell", inline: "sysctl -w net.ipv4.conf.eth2.forwarding=0" +end + +Vagrant.configure("2") do |config| + config.vagrant.plugins = ["vagrant-k3s", "vagrant-reload", "vagrant-libvirt"] + config.vm.provider "libvirt" do |v| + v.cpus = NODE_CPUS + v.memory = NODE_MEMORY + # We replicate the default prefix, but add a timestamp to enable parallel runs and cleanup of old VMs + v.default_prefix = File.basename(Dir.getwd) + "_" + Time.now.to_i.to_s + "_" + end + + if NODE_ROLES.kind_of?(String) + NODE_ROLES = NODE_ROLES.split(" ", -1) + end + if NODE_BOXES.kind_of?(String) + NODE_BOXES = NODE_BOXES.split(" ", -1) + end + + NODE_ROLES.each_with_index do |role, i| + role_num = role.split("-", -1).pop.to_i + config.vm.define role do |node| + provision(node.vm, role, role_num, i) + end + end +end diff --git a/tests/e2e/multus/multus_test.go b/tests/e2e/multus/multus_test.go new file mode 100644 index 000000000000..3f1b7f947f92 --- /dev/null +++ b/tests/e2e/multus/multus_test.go @@ -0,0 +1,263 @@ +// This test verifies that two nodes, which can't connect using the local network, are +// able to still connect using the node-external-ip. In real life, node-external-ip +// would be a public IP. In the test, we create two networks, one sets the node +// internal-ip and the other sets the node-external-ip. Traffic is blocked on the former + +package multus + +import ( + "encoding/json" + "flag" + "fmt" + "os" + "strings" + "testing" + "time" + + "github.com/k3s-io/k3s/tests" + "github.com/k3s-io/k3s/tests/e2e" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// json structure stored in metadata.annotations.k8s\.v1\.cni\.cncf\.io\/network-status +type NetworkConfig struct { + Name string `json:"name"` + Interface string `json:"interface,omitempty"` + IPs []string `json:"ips"` + Default bool `json:"default,omitempty"` + MAC string `json:"mac,omitempty"` + DNS map[string]any `json:"dns,omitempty"` +} + +const successMessage = "5 packets transmitted, 5 received, 0% packet loss" + +// Valid nodeOS: bento/ubuntu-24.04, opensuse/Leap-15.6.x86_64 +var nodeOS = flag.String("nodeOS", "bento/ubuntu-24.04", "VM operating system") +var serverCount = flag.Int("serverCount", 1, "number of server nodes") +var agentCount = flag.Int("agentCount", 1, "number of agent nodes") +var ci = flag.Bool("ci", false, "running on CI") +var local = flag.Bool("local", false, "deploy a locally built K3s binary") + +// getMultusIp returns the IP address on the multus network of the multus-demo pod running on nodeName +func getMultusIp(kubeConfigFile, nodeName string) (string, error) { + cmd := `kubectl get pods -l app=multus-demo --field-selector spec.nodeName=` + nodeName + ` -o jsonpath='{.items[0]..metadata.annotations.k8s\.v1\.cni\.cncf\.io\/network-status}' --kubeconfig=` + kubeConfigFile + res, err := e2e.RunCommand(cmd) + if err != nil { + return "", err + } + + var networkStatus []NetworkConfig + + err = json.Unmarshal([]byte(res), &networkStatus) + if err != nil { + return "", err + } + + fmt.Printf("network status: %v\n", networkStatus) + + return networkStatus[1].IPs[0], nil +} + +func pingOverMultusNetwork(kubeConfigFile, sourceNodeName, destMultusIP string) (bool, error) { + //get the name of the multus-demo pod + cmd := `kubectl get pods -l app=multus-demo --field-selector spec.nodeName=` + sourceNodeName + ` -o jsonpath='{.items[0].metadata.name}' --kubeconfig=` + kubeConfigFile + podName, err := e2e.RunCommand(cmd) + if err != nil { + return false, err + } + fmt.Println(podName) + + //run ping command in that pod + cmd = `kubectl exec --kubeconfig=` + kubeConfigFile + ` ` + podName + ` -- ping -c 5 ` + destMultusIP + res, err := e2e.RunCommand(cmd) + fmt.Println(res) + if err != nil { + return false, err + } + + //run curl command + podCmd := `curl -m 5 -s -f http://` + destMultusIP + `:1180` + + cmd = `kubectl exec --kubeconfig=` + kubeConfigFile + ` ` + podName + ` -- ` + podCmd + res, err = e2e.RunCommand(cmd) + fmt.Println(res) + if err != nil { + return false, err + } + return strings.Contains(res, successMessage), nil +} + +func pingBetweenNode(src, dest e2e.VagrantNode) (bool, error) { + srcIP, err := src.FetchNodeSecondaryIP() + if err != nil { + return false, err + } + fmt.Printf("srcIP: %s\n", srcIP) + + destIP, err := dest.FetchNodeSecondaryIP() + if err != nil { + return false, err + } + fmt.Printf("destIP: %s\n", destIP) + cmd := "ping -c 5 " + destIP + output, err := src.RunCmdOnNode(cmd) + fmt.Printf("output: %s\n", output) + return strings.Contains(output, successMessage), nil + +} + +func runRandomTests(kubeConfigFile, nodeName string) (bool, error) { + //get the name of the multus-demo pod + cmd := `kubectl get pods -l app=multus-demo --field-selector spec.nodeName=` + nodeName + ` -o jsonpath='{.items[0].metadata.name}' --kubeconfig=` + kubeConfigFile + podName, err := e2e.RunCommand(cmd) + if err != nil { + fmt.Printf("error: %s\n", err) + } + fmt.Println(podName) + + cmd = `kubectl exec --kubeconfig=` + kubeConfigFile + ` ` + podName + ` -- ip a` + res, err := e2e.RunCommand(cmd) + fmt.Printf("res: %s", res) + if err != nil { + fmt.Printf("error: %s\n", err) + } + + cmd = `kubectl get pods -o wide -A --kubeconfig=` + kubeConfigFile + res, err = e2e.RunCommand(cmd) + fmt.Printf("res: %s", res) + if err != nil { + fmt.Printf("error: %s\n", err) + } + return true, nil +} + +func Test_E2EMultus(t *testing.T) { + flag.Parse() + RegisterFailHandler(Fail) + suiteConfig, reporterConfig := GinkgoConfiguration() + RunSpecs(t, "Multus config Suite", suiteConfig, reporterConfig) + +} + +var tc *e2e.TestConfig + +var _ = ReportAfterEach(e2e.GenReport) + +var _ = Describe("Verify Multus config", Ordered, func() { + Context("Cluster comes up with Multus enabled", func() { + It("Starts up with no issues", func() { + var err error + if *local { + tc, err = e2e.CreateLocalCluster(*nodeOS, *serverCount, *agentCount) + } else { + tc, err = e2e.CreateCluster(*nodeOS, *serverCount, *agentCount) + } + Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err)) + By("CLUSTER CONFIG") + By("OS: " + *nodeOS) + By(tc.Status()) + }) + + It("Checks Node Status", func() { + Eventually(func() error { + return tests.NodesReady(tc.KubeconfigFile, e2e.VagrantSlice(tc.AllNodes())) + }, "620s", "5s").Should(Succeed()) + e2e.DumpNodes(tc.KubeconfigFile) + }) + + It("Checks pod status", func() { + By("Fetching pod status") + Eventually(func() error { + return tests.AllPodsUp(tc.KubeconfigFile) + }, "620s", "10s").Should(Succeed()) + }) + }) + Context("Deploy workloads to check cluster connectivity of the nodes", func() { + It("Verifies that each node has vagrant IP", func() { + nodeIPs, err := e2e.GetNodeIPs(tc.KubeconfigFile) + fmt.Printf("nodeIPs: %v", nodeIPs) + Expect(err).NotTo(HaveOccurred()) + for _, node := range nodeIPs { + Expect(node.IPv4).Should(ContainSubstring("10.10.")) + } + }) + It("Verifies that each pod has vagrant IP or clusterCIDR IP", func() { + podIPs, err := e2e.GetPodIPs(tc.KubeconfigFile) + Expect(err).NotTo(HaveOccurred()) + for _, pod := range podIPs { + Expect(pod.IPv4).Should(Or(ContainSubstring("10.10."), ContainSubstring("10.42.")), pod.Name) + } + }) + // It("Verifies that nodes can ping each other on secondary network", func() { + // result, err := pingBetweenNode(tc.Servers[0], tc.Agents[0]) + // Expect(err).NotTo(HaveOccurred()) + // Expect(result).To(Equal(true)) + // result, err = pingBetweenNode(tc.Agents[0], tc.Servers[0]) + // Expect(err).NotTo(HaveOccurred()) + // Expect(result).To(Equal(true)) + // }) + It("Deploys Multus NetworkAttachmentDefinition", func() { + _, err := tc.DeployWorkload("multus_network_attach.yaml") + Expect(err).NotTo(HaveOccurred()) + time.Sleep(5 * time.Second) + }) + It("Verifies internode connectivity over multus network", func() { + _, err := tc.DeployWorkload("multus_pod_client.yaml") + Expect(err).NotTo(HaveOccurred()) + + // Wait for each multus-demo pod to have an IP address on the multus network + // then store them + multusIPs := map[string]string{"server-0": "", "agent-0": ""} + + for nodename := range multusIPs { + Eventually(func(g Gomega) { + multusIp, err := getMultusIp(tc.KubeconfigFile, nodename) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(multusIp).Should(ContainSubstring("172.17.0"), "multus IP: "+multusIp) + multusIPs[nodename] = multusIp + }, "40s", "5s").Should(Succeed(), "failed to get Multus IP for node "+nodename) + } + + Eventually(func(g Gomega) { + // result, err := runRandomTests(tc.KubeconfigFile, "server-0") + // g.Expect(err).NotTo(HaveOccurred()) + // g.Expect(result).To(Equal(true)) + //ping pod on agent-0 from pod on server-0 + res, err := pingOverMultusNetwork(tc.KubeconfigFile, "server-0", multusIPs["agent-0"]) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(res).To(Equal(true)) + //ping pod on server-0 from pod on agent-0 + res, err = pingOverMultusNetwork(tc.KubeconfigFile, "agent-0", multusIPs["server-0"]) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(res).To(Equal(true)) + + }, "40s", "5s").Should(Succeed(), "failed to ping between pods on multus network") + + // for _, ip := range clientIPs { + // cmd := "kubectl exec svc/client-curl -- curl -m 5 -s -f http://" + ip.IPv4 + "/name.html" + // Eventually(func() (string, error) { + // return e2e.RunCommand(cmd) + // }, "30s", "10s").Should(ContainSubstring("client-deployment"), "failed cmd: "+cmd) + // } + }) + }) +}) + +var failed bool +var _ = AfterEach(func() { + failed = failed || CurrentSpecReport().Failed() +}) + +var _ = AfterSuite(func() { + if failed { + Expect(e2e.SaveJournalLogs(tc.AllNodes())).To(Succeed()) + Expect(e2e.TailPodLogs(50, tc.AllNodes())).To(Succeed()) + } else { + Expect(e2e.GetCoverageReport(tc.AllNodes())).To(Succeed()) + } + if !failed || *ci { + Expect(e2e.DestroyCluster()).To(Succeed()) + Expect(os.Remove(tc.KubeconfigFile)).To(Succeed()) + } +}) diff --git a/tests/e2e/multus/rke2-multus-config.yaml b/tests/e2e/multus/rke2-multus-config.yaml new file mode 100644 index 000000000000..9cf7bfdc680e --- /dev/null +++ b/tests/e2e/multus/rke2-multus-config.yaml @@ -0,0 +1,24 @@ +apiVersion: helm.cattle.io/v1 +kind: HelmChart +metadata: + name: multus + namespace: kube-system +spec: + repo: https://rke2-charts.rancher.io + chart: rke2-multus + targetNamespace: kube-system + valuesContent: |- + config: + fullnameOverride: multus + cni_conf: + confDir: /var/lib/rancher/k3s/agent/etc/cni/net.d + binDir: /var/lib/rancher/k3s/data/cni/ + kubeconfig: /var/lib/rancher/k3s/agent/etc/cni/net.d/multus.d/multus.kubeconfig + # Comment the following line when using rke2-multus < v4.2.202 + multusAutoconfigDir: /var/lib/rancher/k3s/agent/etc/cni/net.d + rke2-whereabouts: + fullnameOverride: whereabouts + enabled: true + cniConf: + confDir: /var/lib/rancher/k3s/agent/etc/cni/net.d + binDir: /var/lib/rancher/k3s/data/cni/ diff --git a/tests/e2e/testutils.go b/tests/e2e/testutils.go index 2766812fea59..ec111ae030bd 100644 --- a/tests/e2e/testutils.go +++ b/tests/e2e/testutils.go @@ -391,6 +391,19 @@ func (v VagrantNode) FetchNodeExternalIP() (string, error) { return nodeip, nil } +// FetchNodeSecondaryIP returns the IP address of the node on the second network used by Multus +func (v VagrantNode) FetchNodeSecondaryIP() (string, error) { + cmd := "ip -f inet addr show eth2| awk '/inet / {print $2}'|cut -d/ -f1" + ipaddr, err := v.RunCmdOnNode(cmd) + if err != nil { + return "", err + } + ips := strings.Trim(ipaddr, "") + ip := strings.Split(ips, "inet") + nodeip := strings.TrimSpace(ip[1]) + return nodeip, nil +} + // GenKubeconfigFile extracts the kubeconfig from the given node and modifies it for use outside the VM. func GenKubeconfigFile(nodeName string) (string, error) { kubeconfigFile := fmt.Sprintf("kubeconfig-%s", nodeName) @@ -657,6 +670,7 @@ func (v VagrantNode) RunCmdOnNode(cmd string) (string, error) { injectEnv = "GOCOVERDIR=/tmp/k3scov " } runcmd := "vagrant ssh --no-tty " + v.String() + " -c \"sudo " + injectEnv + cmd + "\"" + fmt.Printf("runcmd: %s", runcmd) out, err := RunCommand(runcmd) // On GHA CI we see warnings about "[fog][WARNING] Unrecognized arguments: libvirt_ip_command" // these are added to the command output and need to be removed