Skip to content

Commit 8318f76

Browse files
andrewlockandrewlock
authored
Try fix smoke tests (#5889)
## Summary of changes - Install v1 docker-compose in the hosted runners used for Windows smoke tests - Cleanup the docker networks in the smoke tests ## Reason for change Microsoft [recently](https://learn.microsoft.com/en-us/azure/devops/release-notes/2024/pipelines/sprint-240-update#dockercompose0-uses-docker-compose-v2-in-v1-compatibility-mode) stopped shipping the v1 of docker compose. That broke all our Windows smoke tests. Unfortunately, for various other reasons, we couldn't just convert to the v2 format, due to how we're reusing various bits and pieces. Additionally, we started getting this on our linux smoke tests: ``` Creating network "ddtrace_20240812_56_default" with the default driver could not find an available, non-overlapping IPv4 address pool among the defaults to assign to the network ``` This is basically because we're creating networks, and never cleaning them up, and for efficiency reasons we're not destroying the smoke test VMs after each run. ## Implementation details - Install Docker-compose v1 in the Windows VMs. A bit hacky, but the quickest solution right now - Clean up the docker network after running the snapshot tests ## Test coverage I did some manual tests to make sure this all works as expected. Also killed and restored the smoke tests machines which solved the docker network problem temporarily. ## Other details This is obviously still all kinda horrible. I'd love to replace all this yaml with just testcontainers. One day.... _stares wistfully into the distance_
1 parent 496d420 commit 8318f76

File tree

3 files changed

+87
-20
lines changed

3 files changed

+87
-20
lines changed

‎.azure-pipelines/steps/install-docker-compose-v1.yml‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
parameters:
2+
- name: isLinux
3+
type: boolean
4+
default: false
5+
6+
- name: 'dockerComposePath'
7+
type: string
8+
default: 'C:/docker-compose/docker-compose.exe'
9+
10+
steps:
11+
- ${{ if eq(parameters.isLinux, true) }}:
12+
- bash: |
13+
sudo mkdir -p "$(dirname "${{ parameters.dockerComposePath }}")"
14+
sudo curl -SL https://github.com/docker/compose/releases/download/1.29.2/docker-compose-linux-x86_64 -o ${{ parameters.dockerComposePath }}
15+
sudo chmod 755 ${{ parameters.dockerComposePath }}
16+
displayName: Download docker-compose
17+
- ${{ else }}:
18+
- powershell: |
19+
$dir= (Split-Path -parent "${{ parameters.dockerComposePath }}")
20+
mkdir -f -p $dir
21+
# GitHub now requires TLS1.2. In PowerShell, run the following
22+
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
23+
Invoke-WebRequest "https://github.com/docker/compose/releases/download/1.29.1/docker-compose-windows-x86_64.exe" -OutFile "${{ parameters.dockerComposePath }}"
24+
displayName: Download docker-compose

‎.azure-pipelines/steps/run-snapshot-test.yml‎

Lines changed: 25 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,10 @@ parameters:
1616
- name: 'apiKey'
1717
type: string
1818
default: ''
19+
20+
- name: 'dockerComposePath'
21+
type: string
22+
default: 'docker-compose'
1923

2024
steps:
2125
- template: ./clean-docker-containers.yml
@@ -60,7 +64,7 @@ steps:
6064
displayName: Set env-specific variables
6165
6266
- bash: |
63-
docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) run --rm $(START_TEST_AGENT_TARGET)
67+
${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) run --rm $(START_TEST_AGENT_TARGET)
6468
env:
6569
dockerTag: $(dockerTag)
6670
DD_LOGGER_DD_API_KEY: ${{ parameters.apiKey }}
@@ -69,35 +73,35 @@ steps:
6973

7074
- script: |
7175
echo "Starting snapshot session"
72-
docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) --fail "http://localhost:8126$(START_ENDPOINT)"
76+
${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) --fail "http://localhost:8126$(START_ENDPOINT)"
7377
displayName: start snapshot session
7478
env:
7579
DD_LOGGER_DD_API_KEY: ${{ parameters.apiKey }}
7680

7781
- bash: |
78-
docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) run -e dockerTag=$(dockerTag) -e PROFILER_IS_NOT_REQUIRED=${{ parameters.isNoop }} ${{ parameters.target }}
82+
${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) run -e dockerTag=$(dockerTag) -e PROFILER_IS_NOT_REQUIRED=${{ parameters.isNoop }} ${{ parameters.target }}
7983
env:
8084
dockerTag: $(dockerTag)
8185
DD_LOGGER_DD_API_KEY: ${{ parameters.apiKey }}
82-
displayName: docker-compose run ${{ parameters.target }}
86+
displayName: ${{ parameters.dockerComposePath }} run ${{ parameters.target }}
8387

8488
- script: |
8589
echo "Dumping traces"
86-
docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) -o /debug_snapshots/${{ parameters.snapshotPrefix }}_traces.json "http://localhost:8126$(TRACE_DUMP_ENDPOINT)"
90+
${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) -o /debug_snapshots/${{ parameters.snapshotPrefix }}_traces.json "http://localhost:8126$(TRACE_DUMP_ENDPOINT)"
8791
8892
echo "Dumping stats"
89-
docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) -o /debug_snapshots/${{ parameters.snapshotPrefix }}_stats.json "http://localhost:8126$(STATS_DUMP_ENDPOINT)"
93+
${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) -o /debug_snapshots/${{ parameters.snapshotPrefix }}_stats.json "http://localhost:8126$(STATS_DUMP_ENDPOINT)"
9094
9195
echo "Dumping all requests"
92-
docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) -o /debug_snapshots/${{ parameters.snapshotPrefix }}_requests.json "http://localhost:8126$(REQUESTS_DUMP_ENDPOINT)"
96+
${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) -o /debug_snapshots/${{ parameters.snapshotPrefix }}_requests.json "http://localhost:8126$(REQUESTS_DUMP_ENDPOINT)"
9397
displayName: dump snapshots
9498
env:
9599
DD_LOGGER_DD_API_KEY: ${{ parameters.apiKey }}
96100

97101
- ${{ if and(eq(parameters.isLinux, true), eq(parameters.isNoop, false)) }}:
98102
- bash: |
99103
echo "Verifying snapshot session (fail on mis-match)"
100-
docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) --w '\nGetting a 400 means there is a diff in snapshots. You can diff the files with the artifacts generated. You can also run the tests locally. Follow the doc in /docs/development/CI/RunSmokeTestsLocally\n' --fail "http://localhost:8126$(VERIFY_ENDPOINT)"
104+
${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) --w '\nGetting a 400 means there is a diff in snapshots. You can diff the files with the artifacts generated. You can also run the tests locally. Follow the doc in /docs/development/CI/RunSmokeTestsLocally\n' --fail "http://localhost:8126$(VERIFY_ENDPOINT)"
101105
displayName: check snapshots
102106
env:
103107
DD_LOGGER_DD_API_KEY: ${{ parameters.apiKey }}
@@ -120,19 +124,19 @@ steps:
120124
- ${{ else }}:
121125
- bash: |
122126
echo "Verifying snapshot session (fail on mis-match)"
123-
docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) --fail "http://localhost:8126$(VERIFY_ENDPOINT)"
127+
${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) --fail "http://localhost:8126$(VERIFY_ENDPOINT)"
124128
displayName: check snapshots
125129
env:
126130
DD_LOGGER_DD_API_KEY: ${{ parameters.apiKey }}
127131
128-
- script: docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) logs $(TEST_AGENT_TARGET)
132+
- script: ${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) logs $(TEST_AGENT_TARGET)
129133
displayName: dump docker-compose logs for $(TEST_AGENT_TARGET)
130134
env:
131135
DD_LOGGER_DD_API_KEY: ${{ parameters.apiKey }}
132136
condition: succeededOrFailed()
133137
continueOnError: true
134138

135-
- script: docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) down
139+
- script: ${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) down
136140
displayName: docker-compose stop services
137141
env:
138142
DD_LOGGER_DD_API_KEY: ${{ parameters.apiKey }}
@@ -142,7 +146,7 @@ steps:
142146
# Run crash tests
143147
- ${{ if and(eq(parameters.isLinux, true), eq(parameters.isNoop, false)) }}:
144148
- bash: |
145-
LOGS=$(docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) run -e dockerTag=$(dockerTag) -e CRASH_APP_ON_STARTUP=1 -e COMPlus_DbgEnableMiniDump=0 ${{ parameters.target }})
149+
LOGS=$(${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) run -e dockerTag=$(dockerTag) -e CRASH_APP_ON_STARTUP=1 -e COMPlus_DbgEnableMiniDump=0 ${{ parameters.target }})
146150
echo $LOGS
147151
148152
# check logs for evidence of crash detection in the output
@@ -164,4 +168,12 @@ steps:
164168
- script: |
165169
sudo chmod -R 644 tracer/build_data/dumps/* || true
166170
displayName: Make dumps uploadable to AzDo
167-
condition: succeededOrFailed()
171+
condition: succeededOrFailed()
172+
173+
- script: |
174+
docker network prune -f
175+
displayName: Clean up docker networks
176+
condition: succeededOrFailed()
177+
continueOnError: true
178+
179+

‎.azure-pipelines/ultimate-pipeline.yml‎

Lines changed: 38 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -5687,6 +5687,7 @@ stages:
56875687
variables:
56885688
targetShaId: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.sha']]
56895689
targetBranch: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.branch']]
5690+
dockerComposePath: 'C:/docker-compose/docker-compose.exe'
56905691
jobs:
56915692
- template: steps/update-github-status-jobs.yml
56925693
parameters:
@@ -5702,6 +5703,10 @@ stages:
57025703
vmImage: windows-2022
57035704

57045705
steps:
5706+
- template: steps/install-docker-compose-v1.yml
5707+
parameters:
5708+
dockerComposePath: $(dockerComposePath)
5709+
57055710
- template: steps/clone-repo.yml
57065711
parameters:
57075712
targetShaId: $(targetShaId)
@@ -5725,7 +5730,7 @@ stages:
57255730
displayName: create test data directories
57265731
57275732
- bash: |
5728-
docker-compose -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \
5733+
$(dockerComposePath) -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \
57295734
--build-arg DOTNETSDK_VERSION=$(dotnetCoreSdkLatestVersionShort) \
57305735
--build-arg RUNTIME_IMAGE=$(runtimeImage) \
57315736
--build-arg PUBLISH_FRAMEWORK=$(publishFramework) \
@@ -5742,9 +5747,10 @@ stages:
57425747
target: 'nuget-smoke-tests.windows'
57435748
snapshotPrefix: "smoke_test"
57445749
isLinux: false
5750+
dockerComposePath: $(dockerComposePath)
57455751

57465752
- bash: |
5747-
docker-compose -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \
5753+
$(dockerComposePath) -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \
57485754
--build-arg DOTNETSDK_VERSION=$(dotnetCoreSdkLatestVersionShort) \
57495755
--build-arg RUNTIME_IMAGE=$(runtimeImage) \
57505756
--build-arg PUBLISH_FRAMEWORK=$(publishFramework) \
@@ -5761,6 +5767,7 @@ stages:
57615767
target: 'nuget-dddotnet-smoke-tests.windows'
57625768
snapshotPrefix: "smoke_test"
57635769
isLinux: false
5770+
dockerComposePath: $(dockerComposePath)
57645771

57655772
- publish: tracer/build_data
57665773
artifact: _$(System.StageName)_$(Agent.JobName)_logs_$(System.JobAttempt)
@@ -5777,6 +5784,7 @@ stages:
57775784
variables:
57785785
targetShaId: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.sha']]
57795786
targetBranch: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.branch']]
5787+
dockerComposePath: 'C:/docker-compose/docker-compose.exe'
57805788
jobs:
57815789
- template: steps/update-github-status-jobs.yml
57825790
parameters:
@@ -5792,6 +5800,10 @@ stages:
57925800
vmImage: windows-2022
57935801

57945802
steps:
5803+
- template: steps/install-docker-compose-v1.yml
5804+
parameters:
5805+
dockerComposePath: $(dockerComposePath)
5806+
57955807
- template: steps/clone-repo.yml
57965808
parameters:
57975809
targetShaId: $(targetShaId)
@@ -5812,7 +5824,7 @@ stages:
58125824
displayName: Create test data directories
58135825
58145826
- bash: |
5815-
docker-compose -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \
5827+
$(dockerComposePath) -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \
58165828
--build-arg DOTNETSDK_VERSION=$(dotnetCoreSdkLatestVersionShort) \
58175829
--build-arg RUNTIME_IMAGE=$(runtimeImage) \
58185830
--build-arg PUBLISH_FRAMEWORK=$(publishFramework) \
@@ -5828,6 +5840,7 @@ stages:
58285840
target: 'dotnet-tool-smoke-tests.windows'
58295841
snapshotPrefix: "smoke_test"
58305842
isLinux: false
5843+
dockerComposePath: $(dockerComposePath)
58315844

58325845
- publish: tracer/build_data
58335846
artifact: _$(System.StageName)_$(Agent.JobName)_logs_$(System.JobAttempt)
@@ -5844,6 +5857,7 @@ stages:
58445857
variables:
58455858
targetShaId: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.sha']]
58465859
targetBranch: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.branch']]
5860+
dockerComposePath: 'C:/docker-compose/docker-compose.exe'
58475861
jobs:
58485862
- template: steps/update-github-status-jobs.yml
58495863
parameters:
@@ -5859,6 +5873,10 @@ stages:
58595873
vmImage: windows-2022
58605874

58615875
steps:
5876+
- template: steps/install-docker-compose-v1.yml
5877+
parameters:
5878+
dockerComposePath: $(dockerComposePath)
5879+
58625880
- template: steps/clone-repo.yml
58635881
parameters:
58645882
targetShaId: $(targetShaId)
@@ -5879,7 +5897,7 @@ stages:
58795897
displayName: Create test data directories
58805898
58815899
- bash: |
5882-
docker-compose -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \
5900+
$(dockerComposePath) -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \
58835901
--build-arg DOTNETSDK_VERSION=$(dotnetCoreSdkLatestVersionShort) \
58845902
--build-arg RUNTIME_IMAGE=$(runtimeImage) \
58855903
--build-arg PUBLISH_FRAMEWORK=$(publishFramework) \
@@ -5895,6 +5913,7 @@ stages:
58955913
target: 'smoke-tests.windows'
58965914
snapshotPrefix: "smoke_test"
58975915
isLinux: false
5916+
dockerComposePath: $(dockerComposePath)
58985917

58995918
- publish: tracer/build_data
59005919
artifact: _$(System.StageName)_$(Agent.JobName)_logs_$(System.JobAttempt)
@@ -5911,6 +5930,7 @@ stages:
59115930
variables:
59125931
targetShaId: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.sha']]
59135932
targetBranch: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.branch']]
5933+
dockerComposePath: 'C:/docker-compose/docker-compose.exe'
59145934
jobs:
59155935
- template: steps/update-github-status-jobs.yml
59165936
parameters:
@@ -5927,6 +5947,10 @@ stages:
59275947
vmImage: windows-2022
59285948

59295949
steps:
5950+
- template: steps/install-docker-compose-v1.yml
5951+
parameters:
5952+
dockerComposePath: $(dockerComposePath)
5953+
59305954
- template: steps/clone-repo.yml
59315955
parameters:
59325956
targetShaId: $(targetShaId)
@@ -5947,7 +5971,7 @@ stages:
59475971
displayName: Create test data directories
59485972
59495973
- bash: |
5950-
docker-compose -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \
5974+
$(dockerComposePath) -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \
59515975
--build-arg DOTNETSDK_VERSION=$(dotnetCoreSdkLatestVersionShort) \
59525976
--build-arg RUNTIME_IMAGE=$(runtimeImage) \
59535977
--build-arg PUBLISH_FRAMEWORK=$(publishFramework) \
@@ -5963,6 +5987,7 @@ stages:
59635987
target: 'dd-dotnet-smoke-tests.windows'
59645988
snapshotPrefix: "smoke_test"
59655989
isLinux: false
5990+
dockerComposePath: $(dockerComposePath)
59665991

59675992
- publish: tracer/build_data
59685993
artifact: _$(System.StageName)_$(Agent.JobName)_logs_$(System.JobAttempt)
@@ -5979,6 +6004,7 @@ stages:
59796004
variables:
59806005
targetShaId: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.sha']]
59816006
targetBranch: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.branch']]
6007+
dockerComposePath: 'C:/docker-compose/docker-compose.exe'
59826008
jobs:
59836009
- template: steps/update-github-status-jobs.yml
59846010
parameters:
@@ -5991,9 +6017,13 @@ stages:
59916017
variables:
59926018
smokeTestAppDir: "$(System.DefaultWorkingDirectory)/tracer/test/test-applications/regression/AspNetCoreSmokeTest"
59936019
pool:
5994-
vmImage: windows-2022
6020+
name: azure-windows-scale-set
59956021

59966022
steps:
6023+
- template: steps/install-docker-compose-v1.yml
6024+
parameters:
6025+
dockerComposePath: $(dockerComposePath)
6026+
59976027
- template: steps/clone-repo.yml
59986028
parameters:
59996029
targetShaId: $(targetShaId)
@@ -6012,7 +6042,7 @@ stages:
60126042
path: $(smokeTestAppDir)/artifacts
60136043

60146044
- bash: |
6015-
docker-compose -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \
6045+
$(dockerComposePath) -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \
60166046
--build-arg DOTNETSDK_VERSION=$(dotnetCoreSdkLatestVersionShort) \
60176047
--build-arg RUNTIME_IMAGE=$(runtimeImage) \
60186048
--build-arg PUBLISH_FRAMEWORK=$(publishFramework) \
@@ -6029,6 +6059,7 @@ stages:
60296059
target: 'tracer-home-smoke-tests.windows'
60306060
snapshotPrefix: "smoke_test"
60316061
isLinux: false
6062+
dockerComposePath: $(dockerComposePath)
60326063

60336064
- publish: tracer/build_data
60346065
artifact: _$(System.StageName)_$(Agent.JobName)_logs_$(System.JobAttempt)

0 commit comments

Comments
 (0)