Skip to content

Commit f417ef1

Browse files
andrewlockandrewlock
authored
Try fix smoke tests (#5889 -> v2) (#5891)
## Summary of changes - Install v1 docker-compose in the hosted runners used for Windows smoke tests - Cleanup the docker networks in the smoke tests ## Reason for change Microsoft [recently](https://learn.microsoft.com/en-us/azure/devops/release-notes/2024/pipelines/sprint-240-update#dockercompose0-uses-docker-compose-v2-in-v1-compatibility-mode) stopped shipping the v1 of docker compose. That broke all our Windows smoke tests. Unfortunately, for various other reasons, we couldn't just convert to the v2 format, due to how we're reusing various bits and pieces. Additionally, we started getting this on our linux smoke tests: ``` Creating network "ddtrace_20240812_56_default" with the default driver could not find an available, non-overlapping IPv4 address pool among the defaults to assign to the network ``` This is basically because we're creating networks, and never cleaning them up, and for efficiency reasons we're not destroying the smoke test VMs after each run. ## Implementation details - Install Docker-compose v1 in the Windows VMs. A bit hacky, but the quickest solution right now - Clean up the docker network after running the snapshot tests ## Test coverage I did some manual tests to make sure this all works as expected. Also killed and restored the smoke tests machines which solved the docker network problem temporarily. ## Other details This is obviously still all kinda horrible. I'd love to replace all this yaml with just testcontainers. One day.... _stares wistfully into the distance_ Backport of #5889
1 parent 2b89178 commit f417ef1

File tree

3 files changed

+87
-20
lines changed

3 files changed

+87
-20
lines changed

‎.azure-pipelines/steps/install-docker-compose-v1.yml‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
parameters:
2+
- name: isLinux
3+
type: boolean
4+
default: false
5+
6+
- name: 'dockerComposePath'
7+
type: string
8+
default: 'C:/docker-compose/docker-compose.exe'
9+
10+
steps:
11+
- ${{ if eq(parameters.isLinux, true) }}:
12+
- bash: |
13+
sudo mkdir -p "$(dirname "${{ parameters.dockerComposePath }}")"
14+
sudo curl -SL https://github.com/docker/compose/releases/download/1.29.2/docker-compose-linux-x86_64 -o ${{ parameters.dockerComposePath }}
15+
sudo chmod 755 ${{ parameters.dockerComposePath }}
16+
displayName: Download docker-compose
17+
- ${{ else }}:
18+
- powershell: |
19+
$dir= (Split-Path -parent "${{ parameters.dockerComposePath }}")
20+
mkdir -f -p $dir
21+
# GitHub now requires TLS1.2. In PowerShell, run the following
22+
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
23+
Invoke-WebRequest "https://github.com/docker/compose/releases/download/1.29.1/docker-compose-windows-x86_64.exe" -OutFile "${{ parameters.dockerComposePath }}"
24+
displayName: Download docker-compose

‎.azure-pipelines/steps/run-snapshot-test.yml‎

Lines changed: 25 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,10 @@ parameters:
1212
- name: 'apiKey'
1313
type: string
1414
default: ''
15+
16+
- name: 'dockerComposePath'
17+
type: string
18+
default: 'docker-compose'
1519

1620
steps:
1721
- template: ./clean-docker-containers.yml
@@ -56,7 +60,7 @@ steps:
5660
displayName: Set env-specific variables
5761
5862
- bash: |
59-
docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) run --rm $(START_TEST_AGENT_TARGET)
63+
${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) run --rm $(START_TEST_AGENT_TARGET)
6064
env:
6165
dockerTag: $(dockerTag)
6266
DD_LOGGER_DD_API_KEY: ${{ parameters.apiKey }}
@@ -65,54 +69,54 @@ steps:
6569

6670
- script: |
6771
echo "Starting snapshot session"
68-
docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) --fail "http://localhost:8126$(START_ENDPOINT)"
72+
${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) --fail "http://localhost:8126$(START_ENDPOINT)"
6973
displayName: start snapshot session
7074
env:
7175
DD_LOGGER_DD_API_KEY: ${{ parameters.apiKey }}
7276

7377
- bash: |
74-
docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) run -e dockerTag=$(dockerTag) ${{ parameters.target }}
78+
${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) run -e dockerTag=$(dockerTag) ${{ parameters.target }}
7579
env:
7680
dockerTag: $(dockerTag)
7781
DD_LOGGER_DD_API_KEY: ${{ parameters.apiKey }}
78-
displayName: docker-compose run ${{ parameters.target }}
82+
displayName: ${{ parameters.dockerComposePath }} run ${{ parameters.target }}
7983

8084
- script: |
8185
echo "Dumping traces"
82-
docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) -o /debug_snapshots/${{ parameters.snapshotPrefix }}_traces.json "http://localhost:8126$(TRACE_DUMP_ENDPOINT)"
86+
${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) -o /debug_snapshots/${{ parameters.snapshotPrefix }}_traces.json "http://localhost:8126$(TRACE_DUMP_ENDPOINT)"
8387
8488
echo "Dumping stats"
85-
docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) -o /debug_snapshots/${{ parameters.snapshotPrefix }}_stats.json "http://localhost:8126$(STATS_DUMP_ENDPOINT)"
89+
${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) -o /debug_snapshots/${{ parameters.snapshotPrefix }}_stats.json "http://localhost:8126$(STATS_DUMP_ENDPOINT)"
8690
8791
echo "Dumping all requests"
88-
docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) -o /debug_snapshots/${{ parameters.snapshotPrefix }}_requests.json "http://localhost:8126$(REQUESTS_DUMP_ENDPOINT)"
92+
${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) -o /debug_snapshots/${{ parameters.snapshotPrefix }}_requests.json "http://localhost:8126$(REQUESTS_DUMP_ENDPOINT)"
8993
displayName: dump snapshots
9094
env:
9195
DD_LOGGER_DD_API_KEY: ${{ parameters.apiKey }}
9296

9397
- ${{ if eq(parameters.isLinux, true) }}:
9498
- bash: |
9599
echo "Verifying snapshot session (fail on mis-match)"
96-
docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) --w '\nGetting a 400 means there is a diff in snapshots. You can diff the files with the artifacts generated. You can also run the tests locally. Follow the doc in /docs/development/CI/RunSmokeTestsLocally\n' --fail "http://localhost:8126$(VERIFY_ENDPOINT)"
100+
${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) --w '\nGetting a 400 means there is a diff in snapshots. You can diff the files with the artifacts generated. You can also run the tests locally. Follow the doc in /docs/development/CI/RunSmokeTestsLocally\n' --fail "http://localhost:8126$(VERIFY_ENDPOINT)"
97101
displayName: check snapshots
98102
env:
99103
DD_LOGGER_DD_API_KEY: ${{ parameters.apiKey }}
100104
- ${{ else }}:
101105
- bash: |
102106
echo "Verifying snapshot session (fail on mis-match)"
103-
docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) --fail "http://localhost:8126$(VERIFY_ENDPOINT)"
107+
${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) --fail "http://localhost:8126$(VERIFY_ENDPOINT)"
104108
displayName: check snapshots
105109
env:
106110
DD_LOGGER_DD_API_KEY: ${{ parameters.apiKey }}
107111
108-
- script: docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) logs $(TEST_AGENT_TARGET)
112+
- script: ${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) logs $(TEST_AGENT_TARGET)
109113
displayName: dump docker-compose logs for $(TEST_AGENT_TARGET)
110114
env:
111115
DD_LOGGER_DD_API_KEY: ${{ parameters.apiKey }}
112116
condition: succeededOrFailed()
113117
continueOnError: true
114118

115-
- script: docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) down
119+
- script: ${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) down
116120
displayName: docker-compose stop services
117121
env:
118122
DD_LOGGER_DD_API_KEY: ${{ parameters.apiKey }}
@@ -122,7 +126,7 @@ steps:
122126
# Run crash tests
123127
- ${{ if eq(parameters.isLinux, true) }}:
124128
- bash: |
125-
LOGS=$(docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) run -e dockerTag=$(dockerTag) -e CRASH_APP_ON_STARTUP=1 -e COMPlus_DbgEnableMiniDump=0 ${{ parameters.target }})
129+
LOGS=$(${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) run -e dockerTag=$(dockerTag) -e CRASH_APP_ON_STARTUP=1 -e COMPlus_DbgEnableMiniDump=0 ${{ parameters.target }})
126130
echo $LOGS
127131
128132
# check logs for evidence of crash detection in the output
@@ -144,4 +148,12 @@ steps:
144148
- script: |
145149
sudo chmod -R 644 tracer/build_data/dumps/* || true
146150
displayName: Make dumps uploadable to AzDo
147-
condition: succeededOrFailed()
151+
condition: succeededOrFailed()
152+
153+
- script: |
154+
docker network prune -f
155+
displayName: Clean up docker networks
156+
condition: succeededOrFailed()
157+
continueOnError: true
158+
159+

‎.azure-pipelines/ultimate-pipeline.yml‎

Lines changed: 38 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -5454,6 +5454,7 @@ stages:
54545454
variables:
54555455
targetShaId: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.sha']]
54565456
targetBranch: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.branch']]
5457+
dockerComposePath: 'C:/docker-compose/docker-compose.exe'
54575458
jobs:
54585459
- template: steps/update-github-status-jobs.yml
54595460
parameters:
@@ -5469,6 +5470,10 @@ stages:
54695470
vmImage: windows-2022
54705471

54715472
steps:
5473+
- template: steps/install-docker-compose-v1.yml
5474+
parameters:
5475+
dockerComposePath: $(dockerComposePath)
5476+
54725477
- template: steps/clone-repo.yml
54735478
parameters:
54745479
targetShaId: $(targetShaId)
@@ -5492,7 +5497,7 @@ stages:
54925497
displayName: create test data directories
54935498
54945499
- bash: |
5495-
docker-compose -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \
5500+
$(dockerComposePath) -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \
54965501
--build-arg DOTNETSDK_VERSION=$(dotnetCoreSdkLatestVersionShort) \
54975502
--build-arg RUNTIME_IMAGE=$(runtimeImage) \
54985503
--build-arg PUBLISH_FRAMEWORK=$(publishFramework) \
@@ -5509,9 +5514,10 @@ stages:
55095514
target: 'nuget-smoke-tests.windows'
55105515
snapshotPrefix: "smoke_test"
55115516
isLinux: false
5517+
dockerComposePath: $(dockerComposePath)
55125518

55135519
- bash: |
5514-
docker-compose -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \
5520+
$(dockerComposePath) -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \
55155521
--build-arg DOTNETSDK_VERSION=$(dotnetCoreSdkLatestVersionShort) \
55165522
--build-arg RUNTIME_IMAGE=$(runtimeImage) \
55175523
--build-arg PUBLISH_FRAMEWORK=$(publishFramework) \
@@ -5528,6 +5534,7 @@ stages:
55285534
target: 'nuget-dddotnet-smoke-tests.windows'
55295535
snapshotPrefix: "smoke_test"
55305536
isLinux: false
5537+
dockerComposePath: $(dockerComposePath)
55315538

55325539
- publish: tracer/build_data
55335540
artifact: _$(System.StageName)_$(Agent.JobName)_logs_$(System.JobAttempt)
@@ -5544,6 +5551,7 @@ stages:
55445551
variables:
55455552
targetShaId: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.sha']]
55465553
targetBranch: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.branch']]
5554+
dockerComposePath: 'C:/docker-compose/docker-compose.exe'
55475555
jobs:
55485556
- template: steps/update-github-status-jobs.yml
55495557
parameters:
@@ -5559,6 +5567,10 @@ stages:
55595567
vmImage: windows-2022
55605568

55615569
steps:
5570+
- template: steps/install-docker-compose-v1.yml
5571+
parameters:
5572+
dockerComposePath: $(dockerComposePath)
5573+
55625574
- template: steps/clone-repo.yml
55635575
parameters:
55645576
targetShaId: $(targetShaId)
@@ -5579,7 +5591,7 @@ stages:
55795591
displayName: Create test data directories
55805592
55815593
- bash: |
5582-
docker-compose -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \
5594+
$(dockerComposePath) -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \
55835595
--build-arg DOTNETSDK_VERSION=$(dotnetCoreSdkLatestVersionShort) \
55845596
--build-arg RUNTIME_IMAGE=$(runtimeImage) \
55855597
--build-arg PUBLISH_FRAMEWORK=$(publishFramework) \
@@ -5595,6 +5607,7 @@ stages:
55955607
target: 'dotnet-tool-smoke-tests.windows'
55965608
snapshotPrefix: "smoke_test"
55975609
isLinux: false
5610+
dockerComposePath: $(dockerComposePath)
55985611

55995612
- publish: tracer/build_data
56005613
artifact: _$(System.StageName)_$(Agent.JobName)_logs_$(System.JobAttempt)
@@ -5611,6 +5624,7 @@ stages:
56115624
variables:
56125625
targetShaId: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.sha']]
56135626
targetBranch: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.branch']]
5627+
dockerComposePath: 'C:/docker-compose/docker-compose.exe'
56145628
jobs:
56155629
- template: steps/update-github-status-jobs.yml
56165630
parameters:
@@ -5626,6 +5640,10 @@ stages:
56265640
vmImage: windows-2022
56275641

56285642
steps:
5643+
- template: steps/install-docker-compose-v1.yml
5644+
parameters:
5645+
dockerComposePath: $(dockerComposePath)
5646+
56295647
- template: steps/clone-repo.yml
56305648
parameters:
56315649
targetShaId: $(targetShaId)
@@ -5646,7 +5664,7 @@ stages:
56465664
displayName: Create test data directories
56475665
56485666
- bash: |
5649-
docker-compose -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \
5667+
$(dockerComposePath) -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \
56505668
--build-arg DOTNETSDK_VERSION=$(dotnetCoreSdkLatestVersionShort) \
56515669
--build-arg RUNTIME_IMAGE=$(runtimeImage) \
56525670
--build-arg PUBLISH_FRAMEWORK=$(publishFramework) \
@@ -5662,6 +5680,7 @@ stages:
56625680
target: 'smoke-tests.windows'
56635681
snapshotPrefix: "smoke_test"
56645682
isLinux: false
5683+
dockerComposePath: $(dockerComposePath)
56655684

56665685
- publish: tracer/build_data
56675686
artifact: _$(System.StageName)_$(Agent.JobName)_logs_$(System.JobAttempt)
@@ -5678,6 +5697,7 @@ stages:
56785697
variables:
56795698
targetShaId: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.sha']]
56805699
targetBranch: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.branch']]
5700+
dockerComposePath: 'C:/docker-compose/docker-compose.exe'
56815701
jobs:
56825702
- template: steps/update-github-status-jobs.yml
56835703
parameters:
@@ -5694,6 +5714,10 @@ stages:
56945714
vmImage: windows-2022
56955715

56965716
steps:
5717+
- template: steps/install-docker-compose-v1.yml
5718+
parameters:
5719+
dockerComposePath: $(dockerComposePath)
5720+
56975721
- template: steps/clone-repo.yml
56985722
parameters:
56995723
targetShaId: $(targetShaId)
@@ -5717,7 +5741,7 @@ stages:
57175741
displayName: Create test data directories
57185742
57195743
- bash: |
5720-
docker-compose -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \
5744+
$(dockerComposePath) -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \
57215745
--build-arg DOTNETSDK_VERSION=$(dotnetCoreSdkLatestVersionShort) \
57225746
--build-arg RUNTIME_IMAGE=$(runtimeImage) \
57235747
--build-arg PUBLISH_FRAMEWORK=$(publishFramework) \
@@ -5733,6 +5757,7 @@ stages:
57335757
target: 'dd-dotnet-smoke-tests.windows'
57345758
snapshotPrefix: "smoke_test"
57355759
isLinux: false
5760+
dockerComposePath: $(dockerComposePath)
57365761

57375762
- publish: tracer/build_data
57385763
artifact: _$(System.StageName)_$(Agent.JobName)_logs_$(System.JobAttempt)
@@ -5749,6 +5774,7 @@ stages:
57495774
variables:
57505775
targetShaId: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.sha']]
57515776
targetBranch: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.branch']]
5777+
dockerComposePath: 'C:/docker-compose/docker-compose.exe'
57525778
jobs:
57535779
- template: steps/update-github-status-jobs.yml
57545780
parameters:
@@ -5761,9 +5787,13 @@ stages:
57615787
variables:
57625788
smokeTestAppDir: "$(System.DefaultWorkingDirectory)/tracer/test/test-applications/regression/AspNetCoreSmokeTest"
57635789
pool:
5764-
vmImage: windows-2022
5790+
name: azure-windows-scale-set
57655791

57665792
steps:
5793+
- template: steps/install-docker-compose-v1.yml
5794+
parameters:
5795+
dockerComposePath: $(dockerComposePath)
5796+
57675797
- template: steps/clone-repo.yml
57685798
parameters:
57695799
targetShaId: $(targetShaId)
@@ -5782,7 +5812,7 @@ stages:
57825812
path: $(smokeTestAppDir)/artifacts
57835813

57845814
- bash: |
5785-
docker-compose -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \
5815+
$(dockerComposePath) -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \
57865816
--build-arg DOTNETSDK_VERSION=$(dotnetCoreSdkLatestVersionShort) \
57875817
--build-arg RUNTIME_IMAGE=$(runtimeImage) \
57885818
--build-arg PUBLISH_FRAMEWORK=$(publishFramework) \
@@ -5799,6 +5829,7 @@ stages:
57995829
target: 'tracer-home-smoke-tests.windows'
58005830
snapshotPrefix: "smoke_test"
58015831
isLinux: false
5832+
dockerComposePath: $(dockerComposePath)
58025833

58035834
- publish: tracer/build_data
58045835
artifact: _$(System.StageName)_$(Agent.JobName)_logs_$(System.JobAttempt)

0 commit comments

Comments
 (0)