Switch from yaml.v2 to yaml.v3 (#2558)

dveeden · xhebox · web-flow · commit f6aa0ac09b9d · 2025-08-20T10:41:05.000+08:00
* Switch from yaml.v2 to yaml.v3

* Use KnownFields

* Use KnowFields for a few more places that previously had UnmarshalStrict

* Fix issues found by make check

* Formatting differences in yaml

* update

* fix CI

Signed-off-by: xhe &lt;xw897002528@gmail.com&gt;

---------

Signed-off-by: xhe &lt;xw897002528@gmail.com&gt;
Co-authored-by: xhe &lt;xw897002528@gmail.com&gt;
diff --git a/components/dm/ansible/import.go b/components/dm/ansible/import.go
@@ -33,7 +33,7 @@ import (
 	"github.com/pingcap/tiup/pkg/utils"
 	"github.com/relex/aini"
 	"gopkg.in/ini.v1"
-	"gopkg.in/yaml.v2"
+	"gopkg.in/yaml.v3"
 )
 
 // ref https://docs.ansible.com/ansible/latest/reference_appendices/config.html#the-configuration-file
diff --git a/components/dm/command/import.go b/components/dm/command/import.go
@@ -27,7 +27,7 @@ import (
 	"github.com/pingcap/tiup/pkg/tui"
 	tiuputils "github.com/pingcap/tiup/pkg/utils"
 	"github.com/spf13/cobra"
-	"gopkg.in/yaml.v2"
+	"gopkg.in/yaml.v3"
 )
 
 func newImportCmd() *cobra.Command {
diff --git a/components/dm/spec/topology_dm_test.go b/components/dm/spec/topology_dm_test.go
@@ -19,7 +19,7 @@ import (
 
 	"github.com/pingcap/tiup/pkg/cluster/spec"
 	"github.com/stretchr/testify/require"
-	"gopkg.in/yaml.v2"
+	"gopkg.in/yaml.v3"
 )
 
 func TestDefaultDataDir(t *testing.T) {
diff --git a/go.mod b/go.mod
@@ -4,8 +4,6 @@ go 1.24
 
 toolchain go1.24.1
 
-replace gopkg.in/yaml.v2 => github.com/july2993/yaml v0.0.0-20200423062752-adcfa5abe2ed
-
 require (
 	github.com/AstroProfundis/sysinfo v0.0.0-20240112160158-ed54df16e9ce
 	github.com/BurntSushi/toml v1.5.0
@@ -62,7 +60,6 @@ require (
 	google.golang.org/grpc v1.67.3
 	google.golang.org/protobuf v1.36.1
 	gopkg.in/ini.v1 v1.67.0
-	gopkg.in/yaml.v2 v2.4.0
 	gopkg.in/yaml.v3 v3.0.1
 	software.sslmate.com/src/go-pkcs12 v0.2.0
 )
diff --git a/go.sum b/go.sum
@@ -311,8 +311,6 @@ github.com/joomcode/errorx v1.1.0 h1:dizuSG6yHzlvXOOGHW00gwsmM4Sb9x/yWEfdtPztqcs
 github.com/joomcode/errorx v1.1.0/go.mod h1:eQzdtdlNyN7etw6YCS4W4+lu442waxZYw5yvz0ULrRo=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
-github.com/july2993/yaml v0.0.0-20200423062752-adcfa5abe2ed h1:4dMu41sJad3+E9AMc2hCimGhll184HfPKdu05R6YJtc=
-github.com/july2993/yaml v0.0.0-20200423062752-adcfa5abe2ed/go.mod h1:7FtNkxNa9xDptPhcc7mWT/Ov++hkuMU/NKtgkPZPUwo=
 github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 h1:iQTw/8FWTuc7uiaSepXwyf3o52HaUYcV+Tu66S3F5GA=
 github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8=
 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
@@ -685,6 +683,8 @@ gopkg.in/jcmturner/gokrb5.v6 v6.1.1 h1:n0KFjpbuM5pFMN38/Ay+Br3l91netGSVqHPHEXeWU
 gopkg.in/jcmturner/gokrb5.v6 v6.1.1/go.mod h1:NFjHNLrHQiruory+EmqDXCGv6CrjkeYeA+bR9mIfNFk=
 gopkg.in/jcmturner/rpc.v1 v1.1.0 h1:QHIUxTX1ISuAv9dD2wJ9HWQVuWDX/Zc0PfeC2tjc4rU=
 gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20220512140231-539c8e751b99/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
diff --git a/pkg/cluster/ansible/inventory.go b/pkg/cluster/ansible/inventory.go
@@ -27,7 +27,7 @@ import (
 	"github.com/pingcap/tiup/pkg/tidbver"
 	"github.com/relex/aini"
 	"gopkg.in/ini.v1"
-	"gopkg.in/yaml.v2"
+	"gopkg.in/yaml.v3"
 )
 
 var (
diff --git a/pkg/cluster/manager/edit_config.go b/pkg/cluster/manager/edit_config.go
@@ -27,7 +27,7 @@ import (
 	"github.com/pingcap/tiup/pkg/meta"
 	"github.com/pingcap/tiup/pkg/tui"
 	"github.com/pingcap/tiup/pkg/utils"
-	"gopkg.in/yaml.v2"
+	"gopkg.in/yaml.v3"
 )
 
 // EditConfigOptions contains the options for config edition.
@@ -113,7 +113,9 @@ func (m *Manager) editTopo(origTopo spec.Topology, data []byte, opt EditConfigOp
 	}
 
 	newTopo := m.specManager.NewMetadata().GetTopology()
-	err = yaml.UnmarshalStrict(newData, newTopo)
+	decoder := yaml.NewDecoder(bytes.NewReader(newData))
+	decoder.KnownFields(true)
+	err = decoder.Decode(newTopo)
 	if err != nil {
 		fmt.Print(color.RedString("New topology could not be saved: "))
 		m.logger.Infof("Failed to parse topology file: %v", err)
diff --git a/pkg/cluster/manager/manager_test.go b/pkg/cluster/manager/manager_test.go
@@ -19,7 +19,7 @@ import (
 	"github.com/pingcap/tiup/pkg/cluster/spec"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-	"gopkg.in/yaml.v2"
+	"gopkg.in/yaml.v3"
 )
 
 func TestVersionCompare(t *testing.T) {
diff --git a/pkg/cluster/manager/show_config.go b/pkg/cluster/manager/show_config.go
@@ -20,7 +20,7 @@ import (
 	perrs "github.com/pingcap/errors"
 	"github.com/pingcap/tiup/pkg/cluster/clusterutil"
 	"github.com/pingcap/tiup/pkg/meta"
-	"gopkg.in/yaml.v2"
+	"gopkg.in/yaml.v3"
 )
 
 // ShowConfig shows the cluster's config.
diff --git a/pkg/cluster/spec/parse_topology.go b/pkg/cluster/spec/parse_topology.go
@@ -14,6 +14,7 @@
 package spec
 
 import (
+	"bytes"
 	"os"
 	"path"
 	"reflect"
@@ -23,7 +24,7 @@ import (
 	"github.com/pingcap/tiup/pkg/tui"
 	"github.com/pingcap/tiup/pkg/utils"
 	"go.uber.org/zap"
-	"gopkg.in/yaml.v2"
+	"gopkg.in/yaml.v3"
 )
 
 var (
@@ -86,7 +87,9 @@ func ParseTopologyYaml(file string, out Topology, ignoreGlobal ...bool) error {
 		yamlFile, _ = yaml.Marshal(newTopo)
 	}
 
-	if err = yaml.UnmarshalStrict(yamlFile, out); err != nil {
+	decoder := yaml.NewDecoder(bytes.NewReader(yamlFile))
+	decoder.KnownFields(true)
+	if err = decoder.Decode(out); err != nil {
 		return ErrTopologyParseFailed.
 			Wrap(err, "Failed to parse topology file %s", file).
 			WithProperty(tui.SuggestionFromTemplate(`
diff --git a/pkg/cluster/spec/server_config.go b/pkg/cluster/spec/server_config.go
@@ -30,7 +30,7 @@ import (
 	"github.com/pingcap/tiup/pkg/meta"
 	"github.com/pingcap/tiup/pkg/utils"
 	"go.uber.org/zap"
-	"gopkg.in/yaml.v2"
+	"gopkg.in/yaml.v3"
 )
 
 const (
diff --git a/pkg/cluster/spec/server_config_test.go b/pkg/cluster/spec/server_config_test.go
@@ -5,14 +5,14 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/require"
-	"gopkg.in/yaml.v2"
+	"gopkg.in/yaml.v3"
 )
 
 func TestMerge(t *testing.T) {
 	yamlData := []byte(`
 server_configs:
   tidb:
-    performance.feedback-probability: 0.0
+    performance.feedback-probability: 12.0
 `)
 
 	topo := new(Specification)
@@ -22,13 +22,13 @@ server_configs:
 
 	yamlData, err = yaml.Marshal(topo)
 	require.NoError(t, err)
-	decimal := bytes.Contains(yamlData, []byte("0.0"))
+	decimal := bytes.Contains(yamlData, []byte("12"))
 	require.True(t, decimal)
 
 	get, err := Merge2Toml("tidb", topo.ServerConfigs.TiDB, nil)
 	require.NoError(t, err)
 
-	decimal = bytes.Contains(get, []byte("0.0"))
+	decimal = bytes.Contains(get, []byte("12.0"))
 	require.True(t, decimal)
 }
 
@@ -129,15 +129,15 @@ func TestFoldMap(t *testing.T) {
 
 func TestEncodeRemoteCfg(t *testing.T) {
 	yamlData := []byte(`remote_write:
-- queue_config:
-    batch_send_deadline: 5m
-    capacity: 100000
-    max_samples_per_send: 10000
-    max_shards: 300
-  url: http://127.0.0.1:/8086/write
+    - queue_config:
+        batch_send_deadline: 5m
+        capacity: 100000
+        max_samples_per_send: 10000
+        max_shards: 300
+      url: http://127.0.0.1:/8086/write
 remote_read:
-- url: http://127.0.0.1:/8086/read
-- url: http://127.0.0.1:/8087/read
+    - url: http://127.0.0.1:/8086/read
+    - url: http://127.0.0.1:/8087/read
 `)
 
 	bs, err := encodeRemoteCfg2Yaml(Remote{
diff --git a/pkg/cluster/spec/spec_manager.go b/pkg/cluster/spec/spec_manager.go
@@ -24,7 +24,7 @@ import (
 	"github.com/pingcap/tiup/pkg/tui"
 	"github.com/pingcap/tiup/pkg/utils"
 	"github.com/pingcap/tiup/pkg/version"
-	"gopkg.in/yaml.v2"
+	"gopkg.in/yaml.v3"
 )
 
 var (
diff --git a/pkg/cluster/spec/spec_test.go b/pkg/cluster/spec/spec_test.go
@@ -25,7 +25,7 @@ import (
 	"github.com/pingcap/tiup/pkg/tidbver"
 	"github.com/pingcap/tiup/pkg/utils"
 	"github.com/stretchr/testify/require"
-	"gopkg.in/yaml.v2"
+	"gopkg.in/yaml.v3"
 )
 
 func TestDefaultDataDir(t *testing.T) {
@@ -892,7 +892,7 @@ tiflash_servers:
 
 func TestYAMLAnchor(t *testing.T) {
 	topo := Specification{}
-	err := yaml.UnmarshalStrict([]byte(`
+	decoder := yaml.NewDecoder(bytes.NewReader([]byte(`
 global:
   custom:
     tidb_spec: &tidb_spec
@@ -903,7 +903,9 @@ tidb_servers:
   - <<: *tidb_spec
     host: 172.16.5.138
     deploy_dir: "fake-deploy"
-`), &topo)
+`)))
+	decoder.KnownFields(true)
+	err := decoder.Decode(&topo)
 	require.NoError(t, err)
 	require.Equal(t, "172.16.5.138", topo.TiDBServers[0].Host)
 	require.Equal(t, "fake-deploy", topo.TiDBServers[0].DeployDir)
@@ -912,7 +914,7 @@ tidb_servers:
 
 func TestYAMLAnchorWithUndeclared(t *testing.T) {
 	topo := Specification{}
-	err := yaml.UnmarshalStrict([]byte(`
+	decoder := yaml.NewDecoder(bytes.NewReader([]byte(`
 global:
   custom:
     tidb_spec: &tidb_spec
@@ -923,7 +925,9 @@ global:
 tidb_servers:
   - <<: *tidb_spec
     host: 172.16.5.138
-`), &topo)
+`)))
+	decoder.KnownFields(true)
+	err := decoder.Decode(&topo)
 	require.Error(t, err)
 	require.True(t, strings.Contains(err.Error(), "not found"))
 }
diff --git a/pkg/cluster/spec/tiflash.go b/pkg/cluster/spec/tiflash.go
@@ -35,7 +35,7 @@ import (
 	"github.com/pingcap/tiup/pkg/set"
 	"github.com/pingcap/tiup/pkg/tidbver"
 	"github.com/pingcap/tiup/pkg/utils"
-	"gopkg.in/yaml.v2"
+	"gopkg.in/yaml.v3"
 )
 
 // TiFlashSpec represents the TiFlash topology specification in topology.yaml
diff --git a/pkg/cluster/spec/validate_test.go b/pkg/cluster/spec/validate_test.go
@@ -14,6 +14,7 @@
 package spec
 
 import (
+	"bytes"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -23,7 +24,7 @@ import (
 	"github.com/pingcap/errors"
 	"github.com/pingcap/tiup/pkg/utils"
 	"github.com/stretchr/testify/require"
-	"gopkg.in/yaml.v2"
+	"gopkg.in/yaml.v3"
 )
 
 func TestDirectoryConflicts1(t *testing.T) {
@@ -347,7 +348,9 @@ func TestCountDir2(t *testing.T) {
 	meta := ClusterMeta{}
 	yamlFile, err := os.ReadFile(file)
 	require.NoError(t, err)
-	err = yaml.UnmarshalStrict(yamlFile, &meta)
+	decoder := yaml.NewDecoder(bytes.NewReader(yamlFile))
+	decoder.KnownFields(true)
+	err = decoder.Decode(&meta)
 	require.NoError(t, err)
 	topo := meta.Topology
 
diff --git a/pkg/utils/diff_test.go b/pkg/utils/diff_test.go
@@ -13,8 +13,9 @@ package utils
 import (
 	"testing"
 
+	"gopkg.in/yaml.v3"
+
 	"github.com/stretchr/testify/require"
-	"gopkg.in/yaml.v2"
 )
 
 type sampleDataMeta struct {

Original file line number	Diff line number	Diff line change
`@@ -33,7 +33,7 @@ import (`
`33`	`33`	`"github.com/pingcap/tiup/pkg/utils"`
`34`	`34`	`"github.com/relex/aini"`
`35`	`35`	`"gopkg.in/ini.v1"`
`36`		`- "gopkg.in/yaml.v2"`
	`36`	`+ "gopkg.in/yaml.v3"`
`37`	`37`	`)`
`38`	`38`
`39`	`39`	`// ref https://docs.ansible.com/ansible/latest/reference_appendices/config.html#the-configuration-file`
Original file line number	Diff line number	Diff line change
`@@ -27,7 +27,7 @@ import (`
`27`	`27`	`"github.com/pingcap/tiup/pkg/tui"`
`28`	`28`	`tiuputils "github.com/pingcap/tiup/pkg/utils"`
`29`	`29`	`"github.com/spf13/cobra"`
`30`		`- "gopkg.in/yaml.v2"`
	`30`	`+ "gopkg.in/yaml.v3"`
`31`	`31`	`)`
`32`	`32`
`33`	`33`	`func newImportCmd() *cobra.Command {`
Original file line number	Diff line number	Diff line change
`@@ -19,7 +19,7 @@ import (`
`19`	`19`
`20`	`20`	`"github.com/pingcap/tiup/pkg/cluster/spec"`
`21`	`21`	`"github.com/stretchr/testify/require"`
`22`		`- "gopkg.in/yaml.v2"`
	`22`	`+ "gopkg.in/yaml.v3"`
`23`	`23`	`)`
`24`	`24`
`25`	`25`	`func TestDefaultDataDir(t *testing.T) {`
Original file line number	Diff line number	Diff line change
`@@ -20,7 +20,7 @@ import (`
`20`	`20`	`perrs "github.com/pingcap/errors"`
`21`	`21`	`"github.com/pingcap/tiup/pkg/cluster/clusterutil"`
`22`	`22`	`"github.com/pingcap/tiup/pkg/meta"`
`23`		`- "gopkg.in/yaml.v2"`
	`23`	`+ "gopkg.in/yaml.v3"`
`24`	`24`	`)`
`25`	`25`
`26`	`26`	`// ShowConfig shows the cluster's config.`