diff --git a/src/go/lib/config/commonFlags.go b/src/go/lib/config/commonFlags.go index 85042e115..391ebf4cc 100644 --- a/src/go/lib/config/commonFlags.go +++ b/src/go/lib/config/commonFlags.go @@ -13,6 +13,8 @@ package config +import "github.com/alecthomas/kong" + // Config specifies a list of configuration files to read. // Following the Percona Toolkit specification: // 1. Position: The --config option must be the first argument on the command line. @@ -31,7 +33,7 @@ type ConfigFlag struct { // VersionFlag adds a --version flag that prints the tool version and exits. // Embed this struct into the CLI struct to enable version reporting. type VersionFlag struct { - Version bool `name:"version"` + Version kong.VersionFlag `name:"version" help:"Show version and exit"` } // VersionCheckFlag adds a --version-check / --no-version-check flag that controls diff --git a/src/go/pt-galera-log-explainer/README.rst b/src/go/pt-galera-log-explainer/README.rst index d9405728a..deba46722 100644 --- a/src/go/pt-galera-log-explainer/README.rst +++ b/src/go/pt-galera-log-explainer/README.rst @@ -146,11 +146,29 @@ Available flags ``--version`` Show version and exit. +``--version-check`` + Check for updates (enabled by default). + +``--no-version-check`` + Disable update checks. + ``--custom-regexes`` Add custom regexes, printed in magenta. Format: (golang regex string)=[optional static message to display]. If the static message is left empty, the captured string will be printed instead. Custom regexes are separated using semi-colon. Example: ``--custom-regexes="Page cleaner took [0-9]*ms to flush [0-9]* pages=;doesn't recommend.*pxc_strict_mode=unsafe query used"`` +Command flags +~~~~~~~~~~~~~ + +``list`` + ``--skip-state-colored-column``, ``--all``, ``--states``, ``--views``, ``--events``, ``--sst``, ``--applicative`` + +``whois`` + ``--type {nodename|ip|uuid|auto}``, ``--json`` + +``conflicts`` + ``--yaml``, ``--json`` + Example outputs =============== @@ -286,4 +304,3 @@ VERSION ======= :program:`pt-galera-log-explainer` 3.7.1 - diff --git a/src/go/pt-k8s-debug-collector/README.rst b/src/go/pt-k8s-debug-collector/README.rst index 95ceaaf69..0163d7fc1 100644 --- a/src/go/pt-k8s-debug-collector/README.rst +++ b/src/go/pt-k8s-debug-collector/README.rst @@ -129,6 +129,10 @@ Supported Flags List of Percona Toolkit configuration file(s) separated by a comma without an equal sign. Must be a first flag. Uses default config file locations if not specified. +``--help`` + +Show help and exit. + ``--resource`` Targeted custom resource name. Supported values: @@ -177,6 +181,18 @@ Default: ``auto`` ``--skip-pod-summary`` Skip the collection of pod-specific summary data. +``--skip-pod-summary`` + +Skip pod summary collection. + +``--version-check`` + +Check for updates (enabled by default). + +``--no-version-check`` + +Disable update checks. + ``--version`` Print version info. @@ -244,4 +260,3 @@ VERSION ======= :program:`pt-k8s-debug-collector` 3.7.1 - diff --git a/src/go/pt-k8s-debug-collector/main.go b/src/go/pt-k8s-debug-collector/main.go index f31ccd5d6..61e4ba67c 100644 --- a/src/go/pt-k8s-debug-collector/main.go +++ b/src/go/pt-k8s-debug-collector/main.go @@ -17,6 +17,7 @@ import ( "fmt" "os" + "github.com/alecthomas/kong" log "github.com/sirupsen/logrus" "github.com/percona/percona-toolkit/src/go/lib/config" @@ -52,14 +53,6 @@ type cliOptions struct { } func (c *cliOptions) AfterApply() error { - if c.Version { - fmt.Println(toolname) - fmt.Printf("Version %s\n", Version) - fmt.Printf("Build: %s using %s\n", Build, GoVersion) - fmt.Printf("Commit: %s\n", Commit) - return nil - } - if c.VersionCheck { advice, err := versioncheck.CheckUpdates(toolname, Version) if err != nil { @@ -90,7 +83,15 @@ func (c *cliOptions) AfterApply() error { func main() { opts := &cliOptions{} - _, _, err := config.Setup(toolname, opts) + _, _, err := config.Setup(toolname, opts, + kong.Description("Collects debug data (logs, resource statuses etc.) from a k8s/OpenShift cluster"), + kong.Vars{ + "version": fmt.Sprintf( + "%s\nVersion %s\nBuild: %s using %s\nCommit: %s", + toolname, Version, Build, GoVersion, Commit, + ), + }, + ) if err != nil { log.Printf("cannot get parameters: %s", err.Error()) os.Exit(1) diff --git a/src/go/pt-mongodb-index-check/README.rst b/src/go/pt-mongodb-index-check/README.rst index 78d7ce39b..5237762de 100644 --- a/src/go/pt-mongodb-index-check/README.rst +++ b/src/go/pt-mongodb-index-check/README.rst @@ -37,38 +37,49 @@ Run the program as ``pt-mongodb-index-check [flags]`` Available commands ~~~~~~~~~~~~~~~~~~ -================ ================================== +================ ========================================== Command Description -================ ================================== -check-duplicated Run checks for duplicated indexes. -check-unused Run check for unused indexes. -check-all Run all checks -================ ================================== +================ ========================================== +check-duplicates Run checks for duplicated indexes. +check-unused Run checks for unused indexes. +check-all Run checks for unused and duplicated indexes. +================ ========================================== Available flags ~~~~~~~~~~~~~~~ -+----------------------------+----------------------------------------+ -| Flag | Description | -+============================+========================================+ -| –all-databases | Check in all databases excluding | -| | system dbs. | -+----------------------------+----------------------------------------+ -| –databases=DATABASES,… | Comma separated list of databases to | -| | check. | -+----------------------------+----------------------------------------+ -| –all-collections | Check in all collections in the | -| | selected databases. | -+----------------------------+----------------------------------------+ -| –collections=COLLECTIONS,… | Comma separated list of collections to | -| | check. | -+----------------------------+----------------------------------------+ -| –mongodb.uri= | Connection URI | -+----------------------------+----------------------------------------+ -| –json | Show output as JSON | -+----------------------------+----------------------------------------+ -| –version | Show version information | -+----------------------------+----------------------------------------+ +``--config=FILE[,FILE,...]`` + List of Percona Toolkit config files. Must be the first flag. + +``--all-databases`` + Check in all databases excluding system DBs. + +``--databases=DATABASES,...`` + Comma-separated list of databases to check. + +``--all-collections`` + Check in all collections in selected databases. + +``--collections=COLLECTIONS,...`` + Comma-separated list of collections to check. + +``--mongodb.uri=...`` + Connection URI. + +``--json`` + Show output as JSON. + +``--version-check`` + Check for updates (enabled by default). + +``--no-version-check`` + Disable update checks. + +``--help`` + Show help and exit. + +``--version`` + Show version information and exit. Authors ======= @@ -108,4 +119,3 @@ VERSION ======= :program:`pt-mongodb-index-check` 3.7.1 - diff --git a/src/go/pt-mongodb-index-check/main.go b/src/go/pt-mongodb-index-check/main.go index b9d44e7f5..3865046ee 100644 --- a/src/go/pt-mongodb-index-check/main.go +++ b/src/go/pt-mongodb-index-check/main.go @@ -18,6 +18,7 @@ import ( "context" "encoding/json" "fmt" + "os" "strings" "text/template" "time" @@ -29,16 +30,29 @@ import ( "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" + "github.com/percona/percona-toolkit/src/go/lib/config" + "github.com/percona/percona-toolkit/src/go/lib/versioncheck" "github.com/percona/percona-toolkit/src/go/pt-mongodb-index-check/indexes" "github.com/percona/percona-toolkit/src/go/pt-mongodb-index-check/templates" ) -type cmdlineArgs struct { +const ( + toolname = "pt-mongodb-index-check" +) + +// We do not set anything here, these variables are defined by the Makefile +var ( + Build string //nolint + GoVersion string //nolint + Version string //nolint + Commit string //nolint +) + +type CmdlineArgs struct { + config.ConfigFlag CheckUnused struct{} `cmd:"" name:"check-unused" help:"Check for unused indexes."` CheckDuplicated struct{} `cmd:"" name:"check-duplicates" help:"Check for duplicated indexes."` CheckAll struct{} `cmd:"" name:"check-all" help:"Check for unused and duplicated indexes."` - ShowHelp struct{} `cmd:"" default:"1"` - Version kong.VersionFlag AllDatabases bool `name:"all-databases" xor:"db" help:"Check in all databases excluding system dbs"` Databases []string `name:"databases" xor:"db" help:"Comma separated list of databases to check"` @@ -47,6 +61,22 @@ type cmdlineArgs struct { Collections []string `name:"collections" xor:"colls" help:"Comma separated list of collections to check"` URI string `name:"mongodb.uri" required:"" placeholder:"mongodb://host:port/admindb?options" help:"Connection URI"` JSON bool `name:"json" help:"Show output as JSON"` + + config.VersionFlag + config.VersionCheckFlag +} + +func (c *CmdlineArgs) AfterApply() error { + if c.VersionCheck { + advice, err := versioncheck.CheckUpdates(toolname, Version) + if err != nil { + log.Errorf("cannot check version updates: %s", err.Error()) + } else if advice != "" { + log.Infof("%s", advice) + } + } + + return nil } type response struct { @@ -54,23 +84,27 @@ type response struct { Duplicated []indexes.Duplicate } -const ( - toolname = "pt-mongodb-index-check" -) - -// We do not set anything here, these variables are defined by the Makefile -var ( - Build string //nolint - GoVersion string //nolint - Version string //nolint - Commit string //nolint -) - func main() { - var args cmdlineArgs - kongctx := kong.Parse(&args, kong.UsageOnError(), - kong.Vars{"version": fmt.Sprintf("%s\nVersion %s\nBuild: %s using %s\nCommit: %s", - toolname, Version, Build, GoVersion, Commit)}) + var args CmdlineArgs + kongctx, _, err := config.Setup( + toolname, + &args, + kong.UsageOnError(), + kong.Vars{ + "version": fmt.Sprintf( + "%s\nVersion %s\nBuild: %s using %s\nCommit: %s", + toolname, Version, Build, GoVersion, Commit, + ), + }, + ) + if err != nil { + log.Errorf("cannot get parameters: %s", err.Error()) + os.Exit(1) + } + + if args.Version { + return + } ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) defer cancel() diff --git a/src/go/pt-mongodb-index-check/main_test.go b/src/go/pt-mongodb-index-check/main_test.go index 866885263..a21b44964 100644 --- a/src/go/pt-mongodb-index-check/main_test.go +++ b/src/go/pt-mongodb-index-check/main_test.go @@ -37,13 +37,13 @@ func TestVersionOption(t *testing.T) { func TestNoCommand(t *testing.T) { mockMongo := "mongodb://127.0.0.1:27017" - out, err := exec.Command("../../../bin/"+toolname, "--mongodb.uri", mockMongo).Output() - if err != nil { - t.Errorf("error executing %s with no command: %s", toolname, err.Error()) + out, err := exec.Command("../../../bin/"+toolname, "--mongodb.uri", mockMongo).CombinedOutput() + if err == nil { + t.Fatalf("expected non-zero exit code when no command is provided") } - want := "Usage: pt-mongodb-index-check show-help" + want := "Usage: pt-mongodb-index-check --mongodb.uri=" if !strings.Contains(string(out), want) { - t.Errorf("Output missmatch. Output %q should contain %q", string(out), want) + t.Errorf("Output mismatch. Output %q should contain %q", string(out), want) } } diff --git a/src/go/pt-mongodb-query-digest/README.rst b/src/go/pt-mongodb-query-digest/README.rst index f7594b8c6..76a76f2f7 100644 --- a/src/go/pt-mongodb-query-digest/README.rst +++ b/src/go/pt-mongodb-query-digest/README.rst @@ -12,7 +12,7 @@ Usage .. code-block:: bash - pt-mongodb-query-digest [OPTIONS] + pt-mongodb-query-digest [OPTIONS] [host[:port]] It runs the following command:: @@ -27,7 +27,10 @@ By default, the results are sorted by ascending query count. Options ------- -``-?``, ``--help`` +``--config`` +List of Percona Toolkit configuration file(s) separated by a comma without an equal sign. Must be a first flag. Uses default config file locations if not specified. + +``--help`` Show help and exit ``-a``, ``--authenticationDatabase`` @@ -35,26 +38,32 @@ Options with a MongoDB server. By default, the ``admin`` database is used. -``-c``, ``--no-version-check`` - Don't check for updates +``--version-check`` + Check for updates (enabled by default). + +``--no-version-check`` + Disable update checks. ``-d``, ``--database`` Specifies which database to profile +``host[:port]`` + Optional positional host. If no scheme is provided, ``mongodb://`` is prepended automatically. + ``-f``, ``--output-format`` Specifies the report output format. Valid options are: ``text``, ``json``. The default value is ``text``. ``-l``, ``--log-level`` Specifies the log level: - ``panic``, ``fatal``, ``error``, ``warn``, ``info``, ``debug error`` + ``panic``, ``fatal``, ``error``, ``warn``, ``info``, ``debug`` ``-n``, ``--limit`` Limits the number of queries to show ``-o``, ``--order-by`` Specifies the sorting order using fields: - ``count``, ``ratio``, ``query-time``, ``docs-examined``, ``docs-returned``. + ``count``, ``ratio``, ``query-time``, ``docs-scanned``, ``docs-returned``. Adding a hyphen (``-``) in front of a field denotes reverse order. For example: ``--order-by="count,-ratio"``. @@ -81,11 +90,11 @@ Options ``--sslPEMKeyFile`` Specifies SSL client PEM file used for authentication. -``-u``, ``--user`` +``-u``, ``--username`` Specifies the user name for connecting to a server with authentication enabled. -``-v``, ``--version`` +``--version`` Show version and exit Output Example @@ -147,4 +156,3 @@ VERSION ======= :program:`pt-mongodb-query-digest` 3.7.1 - diff --git a/src/go/pt-mongodb-query-digest/main.go b/src/go/pt-mongodb-query-digest/main.go index fee0cc041..9bdc53cea 100644 --- a/src/go/pt-mongodb-query-digest/main.go +++ b/src/go/pt-mongodb-query-digest/main.go @@ -25,7 +25,7 @@ import ( "text/template" "time" - "github.com/pborman/getopt" + "github.com/alecthomas/kong" log "github.com/sirupsen/logrus" "go.mongodb.org/mongo-driver/bson/primitive" "go.mongodb.org/mongo-driver/mongo" @@ -61,76 +61,112 @@ var ( ) type cliOptions struct { - AuthDB string - Database string - Debug bool - Help bool - Host string - Limit int - LogLevel string - NoVersionCheck bool - OrderBy []string - OutputFormat string - Password string - SkipCollections []string - SSLCAFile string - SSLPEMKeyFile string - User string - Version bool + config.ConfigFlag + AuthDB string `name:"authenticationDatabase" short:"a" help:"Database to use for optional MongoDB authentication" default:"admin"` + Database string `name:"database" short:"d" help:"MongoDB database to profile"` + Host string `arg:"" name:"host" help:"host[:port]" default:"localhost:27017"` + Limit int `name:"limit" short:"n" help:"Show the first n queries"` + LogLevel string `name:"log-level" short:"l" help:"panic, fatal, error, warn, info, debug" default:"error"` + OrderBy []string `name:"order-by" short:"o" help:"Comma separated list of order by fields (count, ratio, query-time, docs-scanned, docs-returned). Prefix '-' for reverse order." default:"-count"` + SkipCollections []string `name:"skip-collections" short:"s" help:"A comma separated list of collections (namespaces) to skip." default:"system.profile"` + OutputFormat string `name:"output-format" short:"f" help:"Output format: text, json." default:"text"` + User string `name:"username" short:"u" help:"Username to use for optional MongoDB authentication"` + Password config.StdinRequestString `name:"password" short:"p" help:"Password to use for optional MongoDB authentication"` + SSLCAFile string `name:"sslCAFile" help:"SSL CA cert file used for authentication"` + SSLPEMKeyFile string `name:"sslPEMKeyFile" help:"SSL client PEM file used for authentication"` + config.VersionFlag + config.VersionCheckFlag } -type report struct { - Headers []string - QueryStats []stats.QueryStats - QueryTotals stats.QueryStats -} +func (c *cliOptions) AfterApply() error { + if len(c.OrderBy) > 0 { + validFields := []string{"count", "ratio", "query-time", "docs-scanned", "docs-returned"} + for _, field := range c.OrderBy { + valid := false + for _, vf := range validFields { + if field == vf || field == "-"+vf { + valid = true + } + } + if !valid { + return fmt.Errorf("invalid sort field '%q'", field) + } + } + } -func main() { - opts, err := getOptions() + err := c.Password.Request(func() (string, error) { + print("Password: ") + pass, err := term.ReadPassword(0) + return string(pass), err + }) if err != nil { - log.Errorf("error processing command line arguments: %s", err) - os.Exit(1) + return err } - if opts == nil && err == nil { - return + + if c.OutputFormat != "json" && c.OutputFormat != "text" { + log.Infof("Invalid output format '%s'. Using text format", c.OutputFormat) + c.OutputFormat = "text" } - logLevel, err := log.ParseLevel(opts.LogLevel) - if err != nil { - fmt.Printf("Cannot set log level: %s", err.Error()) - os.Exit(1) + if !strings.HasPrefix(c.Host, "mongodb://") { + c.Host = "mongodb://" + c.Host } - log.SetLevel(logLevel) - if opts.Version { - fmt.Println(toolname) - fmt.Printf("Version %s\n", Version) - fmt.Printf("Build: %s using %s\n", Build, GoVersion) - fmt.Printf("Commit: %s\n", Commit) - return + if c.Database == "" { + return fmt.Errorf("must indicate a database to profile with the --database parameter") } - conf := config.DefaultConfig(toolname) - if !conf.GetBool("no-version-check") && !opts.NoVersionCheck { + logLevel, err := log.ParseLevel(c.LogLevel) + if err != nil { + return fmt.Errorf("Cannot set log level: %w", err) + } + log.SetLevel(logLevel) + + if c.VersionCheck { advice, err := versioncheck.CheckUpdates(toolname, Version) if err != nil { - log.Infof("cannot check version updates: %s", err.Error()) + log.Errorf("cannot check version updates: %s", err.Error()) } else if advice != "" { - log.Warn(advice) + log.Infof("%s", advice) } } - log.Debugf("Command line options:\n%+v\n", opts) + return nil +} - clientOptions, err := getClientOptions(opts) +type report struct { + Headers []string + QueryStats []stats.QueryStats + QueryTotals stats.QueryStats +} + +func main() { + var opts cliOptions + _, _, err := config.Setup( + toolname, + &opts, + kong.UsageOnError(), + kong.Vars{ + "version": fmt.Sprintf( + "%s\nVersion %s\nBuild: %s using %s\nCommit: %s", + toolname, Version, Build, GoVersion, Commit, + ), + }, + ) if err != nil { - log.Errorf("Cannot get a MongoDB client: %s", err) - os.Exit(2) + log.Errorf("cannot get parameters: %s", err.Error()) + os.Exit(1) } - if opts.Database == "" { - log.Errorln("must indicate a database to profile with the --database parameter") - getopt.PrintUsage(os.Stderr) + if opts.Version { + return + } + + log.Debugf("Command line options:\n%+v\n", opts) + + clientOptions, err := getClientOptions(&opts) + if err != nil { + log.Errorf("Cannot get a MongoDB client: %s", err) os.Exit(2) } @@ -196,7 +232,7 @@ func main() { return } rep := report{ - Headers: getHeaders(opts), + Headers: getHeaders(&opts), QueryTotals: queries.CalcTotalQueriesStats(uptime), QueryStats: sortedQueryStats, } @@ -276,87 +312,6 @@ func uptime(ctx context.Context, client *mongo.Client) int64 { return ss.Uptime } -func getOptions() (*cliOptions, error) { - opts := &cliOptions{ - Host: DEFAULT_HOST, - LogLevel: DEFAULT_LOGLEVEL, - OrderBy: strings.Split(DEFAULT_ORDERBY, ","), - SkipCollections: strings.Split(DEFAULT_SKIPCOLLECTIONS, ","), - AuthDB: DEFAULT_AUTHDB, - OutputFormat: "text", - } - - gop := getopt.New() - gop.BoolVarLong(&opts.Help, "help", '?', "Show help") - gop.BoolVarLong(&opts.Version, "version", 'v', "Show version & exit") - gop.BoolVarLong(&opts.NoVersionCheck, "no-version-check", 'c', "Default: Don't check for updates") - - gop.IntVarLong(&opts.Limit, "limit", 'n', "Show the first n queries") - - gop.ListVarLong(&opts.OrderBy, "order-by", 'o', - "Comma separated list of order by fields (max values): "+ - "count,ratio,query-time,docs-scanned,docs-returned. "+ - "- in front of the field name denotes reverse order. Default: "+DEFAULT_ORDERBY) - gop.ListVarLong(&opts.SkipCollections, "skip-collections", 's', "A comma separated list of collections (namespaces) to skip."+ - " Default: "+DEFAULT_SKIPCOLLECTIONS) - - gop.StringVarLong(&opts.AuthDB, "authenticationDatabase", 'a', "admin", "Database to use for optional MongoDB authentication. Default: admin") - gop.StringVarLong(&opts.Database, "database", 'd', "", "MongoDB database to profile") - gop.StringVarLong(&opts.LogLevel, "log-level", 'l', "Log level: error", "panic, fatal, error, warn, info, debug. Default: error") - gop.StringVarLong(&opts.OutputFormat, "output-format", 'f', "text", "Output format: text, json. Default: text") - gop.StringVarLong(&opts.Password, "password", 'p', "", "Password to use for optional MongoDB authentication").SetOptional() - gop.StringVarLong(&opts.User, "username", 'u', "Username to use for optional MongoDB authentication") - gop.StringVarLong(&opts.SSLCAFile, "sslCAFile", 0, "SSL CA cert file used for authentication") - gop.StringVarLong(&opts.SSLPEMKeyFile, "sslPEMKeyFile", 0, "SSL client PEM file used for authentication") - - gop.SetParameters("host[:port]") - - gop.Parse(os.Args) - if gop.NArgs() > 0 { - opts.Host = gop.Arg(0) - gop.Parse(gop.Args()) - } - if opts.Help { - gop.PrintUsage(os.Stdout) - return nil, nil - } - - if gop.IsSet("order-by") { - validFields := []string{"count", "ratio", "query-time", "docs-scanned", "docs-returned"} - for _, field := range opts.OrderBy { - valid := false - for _, vf := range validFields { - if field == vf || field == "-"+vf { - valid = true - } - } - if !valid { - return nil, fmt.Errorf("invalid sort field '%q'", field) - } - } - } - - if opts.OutputFormat != "json" && opts.OutputFormat != "text" { - log.Infof("Invalid output format '%s'. Using text format", opts.OutputFormat) - opts.OutputFormat = "text" - } - - if gop.IsSet("password") && opts.Password == "" { - print("Password: ") - pass, err := term.ReadPassword(0) - if err != nil { - return nil, err - } - opts.Password = string(pass) - } - - if !strings.HasPrefix(opts.Host, "mongodb://") { - opts.Host = "mongodb://" + opts.Host - } - - return opts, nil -} - func getClientOptions(opts *cliOptions) (*options.ClientOptions, error) { clientOptions := options.Client().ApplyURI(opts.Host) credential := options.Credential{} @@ -365,7 +320,7 @@ func getClientOptions(opts *cliOptions) (*options.ClientOptions, error) { clientOptions.SetAuth(credential) } if opts.Password != "" { - credential.Password = opts.Password + credential.Password = string(opts.Password) credential.PasswordSet = true clientOptions.SetAuth(credential) } diff --git a/src/go/pt-mongodb-query-digest/main_test.go b/src/go/pt-mongodb-query-digest/main_test.go index 014a2aeee..47b528815 100644 --- a/src/go/pt-mongodb-query-digest/main_test.go +++ b/src/go/pt-mongodb-query-digest/main_test.go @@ -31,10 +31,11 @@ import ( "text/template" "time" + "github.com/alecthomas/kong" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" - "github.com/pborman/getopt" + "github.com/percona/percona-toolkit/src/go/lib/config" "github.com/percona/percona-toolkit/src/go/lib/profiling" "github.com/percona/percona-toolkit/src/go/lib/tutil" "github.com/percona/percona-toolkit/src/go/mongolib/stats" @@ -59,12 +60,17 @@ var client *mongo.Client func TestMain(m *testing.M) { var err error + mongoDSN := os.Getenv("PT_TEST_MONGODB_DSN") if vars.RootPath, err = tutil.RootPath(); err != nil { log.Printf("cannot get root path: %s", err.Error()) os.Exit(1) } - client, err = mongo.Connect(context.TODO(), options.Client().ApplyURI(os.Getenv("PT_TEST_MONGODB_DSN"))) + if mongoDSN == "" { + os.Exit(m.Run()) + } + + client, err = mongo.Connect(context.TODO(), options.Client().ApplyURI(mongoDSN)) if err != nil { log.Printf("Cannot connect: %s", err.Error()) os.Exit(1) @@ -113,54 +119,52 @@ func TestIsProfilerEnabled(t *testing.T) { } } -func TestParseArgs(t *testing.T) { - tests := []struct { - args []string - want *cliOptions - }{ - { - args: []string{toolname}, // arg[0] is the command itself - want: &cliOptions{ - Host: "mongodb://" + DEFAULT_HOST, - LogLevel: DEFAULT_LOGLEVEL, - OrderBy: strings.Split(DEFAULT_ORDERBY, ","), - SkipCollections: strings.Split(DEFAULT_SKIPCOLLECTIONS, ","), - AuthDB: DEFAULT_AUTHDB, - OutputFormat: "text", - }, - }, - { - args: []string{toolname, "zapp.brannigan.net:27018/samples", "--help"}, - want: nil, - }, - { - args: []string{toolname, "zapp.brannigan.net:27018/samples"}, - want: &cliOptions{ - Host: "mongodb://zapp.brannigan.net:27018/samples", - LogLevel: DEFAULT_LOGLEVEL, - OrderBy: strings.Split(DEFAULT_ORDERBY, ","), - SkipCollections: strings.Split(DEFAULT_SKIPCOLLECTIONS, ","), - AuthDB: DEFAULT_AUTHDB, - Help: false, - OutputFormat: "text", - }, - }, - } - for i, test := range tests { - getopt.Reset() - os.Args = test.args - //disabling Stdout to avoid printing help message to the screen - sout := os.Stdout - os.Stdout = nil - got, err := getOptions() - os.Stdout = sout - if err != nil { - t.Errorf("error parsing command line arguments: %s", err.Error()) - } - if !reflect.DeepEqual(got, test.want) { - t.Errorf("invalid command line options test %d\ngot %+v\nwant %+v\n", i, got, test.want) - } - } +// NOTE: legacy getopt-based parsing tests were removed after the CLI migration to Kong. + +func withArgs(args []string, fn func()) { + oldArgs := os.Args + os.Args = args + defer func() { os.Args = oldArgs }() + fn() +} + +func TestParseArgsKong(t *testing.T) { + t.Run("default host", func(t *testing.T) { + withArgs([]string{toolname, "--database=test", "--no-version-check"}, func() { + var opts cliOptions + _, _, err := config.Setup(toolname, &opts, kong.UsageOnError()) + if err != nil { + t.Fatalf("config.Setup() error = %v", err) + } + + if opts.Host != "mongodb://localhost:27017" { + t.Fatalf("opts.Host = %q, want %q", opts.Host, "mongodb://localhost:27017") + } + if opts.Database != "test" { + t.Fatalf("opts.Database = %q, want %q", opts.Database, "test") + } + if opts.AuthDB != "admin" { + t.Fatalf("opts.AuthDB = %q, want %q", opts.AuthDB, "admin") + } + if opts.OutputFormat != "text" { + t.Fatalf("opts.OutputFormat = %q, want %q", opts.OutputFormat, "text") + } + }) + }) + + t.Run("positional host without scheme", func(t *testing.T) { + withArgs([]string{toolname, "example.org:27018", "--database=test", "--no-version-check"}, func() { + var opts cliOptions + _, _, err := config.Setup(toolname, &opts, kong.UsageOnError()) + if err != nil { + t.Fatalf("config.Setup() error = %v", err) + } + + if opts.Host != "mongodb://example.org:27018" { + t.Fatalf("opts.Host = %q, want %q", opts.Host, "mongodb://example.org:27018") + } + }) + }) } func TestPTMongoDBQueryDigest(t *testing.T) { @@ -208,6 +212,9 @@ func TestPTMongoDBQueryDigest(t *testing.T) { url: os.Getenv("PT_TEST_MONGODB_DSN"), db: "test", } + if data.url == "" { + t.Skip("Skipping integration test: PT_TEST_MONGODB_DSN is not set") + } tests := []func(*testing.T, Data){ testVersion, testEmptySystemProfile, diff --git a/src/go/pt-mongodb-summary/README.rst b/src/go/pt-mongodb-summary/README.rst index 80161e880..bb6876bcd 100644 --- a/src/go/pt-mongodb-summary/README.rst +++ b/src/go/pt-mongodb-summary/README.rst @@ -37,8 +37,15 @@ Options with a MongoDB server. By default, the ``admin`` database is used. -``-c``, ``--no-version-check`` - Disables checking the version of MongoDB before running the report. +``--config`` + List of Percona Toolkit configuration file(s) separated by a comma without an equal sign. + Must be the first flag. Uses default config file locations if not specified. + +``--version-check`` + Check for updates (enabled by default). + +``--no-version-check`` + Disable update checks. ``-f``, ``--output-format`` Specifies the report output format. Valid options are: ``text``, ``json``. @@ -57,7 +64,7 @@ Options ``-l``, ``--log-level`` Specifies the logging level. Valid options: ``panic``, ``fatal``, ``error``, ``warn``, ``info``, ``debug``. - Default: ``error``. + Default: ``warn``. ``-p``, ``--password`` Specifies the password to use when connecting to a server @@ -91,7 +98,7 @@ Options Specifies the username to use when connecting to a server with authentication enabled. -``-v``, ``--version`` +``--version`` Show version information and exit. Output example @@ -250,4 +257,3 @@ VERSION ======= :program:`pt-mongodb-summary` 3.7.1 - diff --git a/src/go/pt-mongodb-summary/main.go b/src/go/pt-mongodb-summary/main.go index 020356521..393cc3a0e 100644 --- a/src/go/pt-mongodb-summary/main.go +++ b/src/go/pt-mongodb-summary/main.go @@ -31,8 +31,8 @@ import ( "strings" "time" + "github.com/alecthomas/kong" version "github.com/hashicorp/go-version" - "github.com/pborman/getopt" "github.com/pkg/errors" "github.com/shirou/gopsutil/process" log "github.com/sirupsen/logrus" @@ -53,14 +53,9 @@ import ( const ( toolname = "pt-mongodb-summary" - DefaultAuthDB = "admin" - DefaultHost = "localhost" - DefaultPort = "27017" - DefaultLogLevel = "warn" - DefaultRunningOpsInterval = 1000 // milliseconds - DefaultRunningOpsSamples = 5 - DefaultOutputFormat = "text" - typeMongos = "mongos" + DefaultHost = "localhost" + DefaultPort = "27017" + typeMongos = "mongos" // Exit Codes. cannotFormatResults = 1 @@ -187,22 +182,71 @@ func (t mongosInfo) MaxNameLen() int { } type cliOptions struct { - Host string - Port string - User string - Password string - AuthDB string - LogLevel string - OutputFormat string - SSLCAFile string - SSLPEMKeyFile string - RunningOpsSamples int - RunningOpsInterval int - URI string - Help bool - Version bool - NoVersionCheck bool - NoRunningOps bool + config.ConfigFlag + Host string `name:"host" help:"Host" default:"${default_host}"` + Port string `name:"port" help:"Port" default:"${default_port}"` + HostParams string `arg:"" name:"host" help:"host[:port]" default:""` + User string `name:"username" short:"u" help:"Username to use for optional MongoDB authentication"` + Password config.StdinRequestString `name:"password" short:"p" help:"Password to use for optional MongoDB authentication"` + AuthDB string `name:"authenticationDatabase" short:"a" help:"Database to use for optional MongoDB authentication." default:"admin"` + LogLevel string `name:"log-level" short:"l" help:"Log level: panic, fatal, error, warn, info, debug." default:"warn"` + OutputFormat string `name:"output-format" short:"f" help:"Output format: text, json." default:"text"` + SSLCAFile string `name:"sslCAFile" help:"SSL CA cert file used for authentication"` + SSLPEMKeyFile string `name:"sslPEMKeyFile" help:"SSL client PEM file used for authentication"` + RunningOpsSamples int `name:"running-ops-samples" short:"s" help:"Number of samples to collect for running ops." default:"5"` + RunningOpsInterval int `name:"running-ops-interval" short:"i" help:"Interval to wait between running ops samples in milliseconds." default:"1000"` + URI string `name:"uri" help:"URI describes the hosts to be used and options. Flags has higher priority. If a full URI is provided, you cannot also specify \"--host\" or \"--port\"."` + config.VersionFlag + config.VersionCheckFlag +} + +func (c *cliOptions) AfterApply() error { + if c.HostParams != "" { + if c.Host != "" || c.Port != "" || c.URI != "" { + return fmt.Errorf(`argument host[:port] is not compatible with "--uri", "--host" and "--port" flags set`) + } + var err error + c.Host, c.Port, err = net.SplitHostPort(c.HostParams) + if err != nil { + return err + } + } + + err := c.Password.Request(func() (string, error) { + print("Password: ") + pass, err := term.ReadPassword(0) + return string(pass), err + }) + if err != nil { + return err + } + + if c.URI != "" && (c.Host != "" || c.Port != "") { + return fmt.Errorf("If a full URI is provided, you cannot also specify --host or --port") + } + + if c.OutputFormat != "json" && c.OutputFormat != "text" { + log.Infof("Invalid output format '%s'. Using text format", c.OutputFormat) + c.OutputFormat = "text" + } + + logLevel, err := log.ParseLevel(c.LogLevel) + if err != nil { + fmt.Printf("cannot set log level: %s", err.Error()) + } + + log.SetLevel(logLevel) + + if c.VersionCheck { + advice, err := versioncheck.CheckUpdates(toolname, Version) + if err != nil { + log.Errorf("cannot check version updates: %s", err.Error()) + } else if advice != "" { + log.Infof("%s", advice) + } + } + + return nil } type collectedInfo struct { @@ -218,45 +262,31 @@ type collectedInfo struct { } func main() { - opts, err := parseFlags() + var opts cliOptions + _, _, err := config.Setup( + toolname, + &opts, + kong.UsageOnError(), + kong.Vars{ + "version": fmt.Sprintf( + "%s\nVersion %s\nBuild: %s using %s\nCommit: %s", + toolname, Version, Build, GoVersion, Commit, + ), + "default_host": DefaultHost, + "default_port": DefaultPort, + }, + ) if err != nil { log.Errorf("cannot get parameters: %s", err.Error()) - - os.Exit(cannotParseCommandLineParameters) - } - - if opts == nil && err == nil { - return + os.Exit(1) } - logLevel, err := log.ParseLevel(opts.LogLevel) - if err != nil { - fmt.Printf("cannot set log level: %s", err.Error()) - } - - log.SetLevel(logLevel) - if opts.Version { - fmt.Println(toolname) - fmt.Printf("Version %s\n", Version) - fmt.Printf("Build: %s using %s\n", Build, GoVersion) - fmt.Printf("Commit: %s\n", Commit) - return } - conf := config.DefaultConfig(toolname) - if !conf.GetBool("no-version-check") && !opts.NoVersionCheck { - advice, err := versioncheck.CheckUpdates(toolname, Version) - if err != nil { - log.Infof("cannot check version updates: %s", err.Error()) - } else if advice != "" { - log.Infof("%s", advice) - } - } - ctx := context.Background() - clientOptions, err := getClientOptions(opts) + clientOptions, err := getClientOptions(&opts) if err != nil { log.Error(err) @@ -979,88 +1009,6 @@ func externalIP() (string, error) { return "", errors.New("are you connected to the network?") } -func parseFlags() (*cliOptions, error) { - opts := &cliOptions{ - LogLevel: DefaultLogLevel, - RunningOpsSamples: DefaultRunningOpsSamples, - RunningOpsInterval: DefaultRunningOpsInterval, // milliseconds - AuthDB: DefaultAuthDB, - OutputFormat: DefaultOutputFormat, - } - - gop := getopt.New() - gop.BoolVarLong(&opts.Help, "help", 'h', "Show help") - gop.BoolVarLong(&opts.Version, "version", 'v', "", "Show version & exit") - gop.BoolVarLong(&opts.NoVersionCheck, "no-version-check", 'c', "", "Default: Don't check for updates") - - gop.StringVarLong(&opts.URI, "uri", 0, `URI describes the hosts to be used and options. Flags has higher priority. If a full URI is provided, you cannot also specify "--host" or "--port".`) - gop.StringVarLong(&opts.Host, "host", 0, "Host") - gop.StringVarLong(&opts.Port, "port", 0, "Port") - - gop.StringVarLong(&opts.User, "username", 'u', "", "Username to use for optional MongoDB authentication") - gop.StringVarLong(&opts.Password, "password", 'p', "", "Password to use for optional MongoDB authentication"). - SetOptional() - gop.StringVarLong(&opts.AuthDB, "authenticationDatabase", 'a', "admin", - "Database to use for optional MongoDB authentication. Default: admin") - gop.StringVarLong(&opts.LogLevel, "log-level", 'l', "error", - "Log level: panic, fatal, error, warn, info, debug. Default: error") - gop.StringVarLong(&opts.OutputFormat, "output-format", 'f', "text", "Output format: text, json. Default: text") - - gop.IntVarLong(&opts.RunningOpsSamples, "running-ops-samples", 's', - fmt.Sprintf("Number of samples to collect for running ops. Default: %d", opts.RunningOpsSamples), - ) - - gop.IntVarLong(&opts.RunningOpsInterval, "running-ops-interval", 'i', - fmt.Sprintf("Interval to wait between running ops samples in milliseconds. Default %d milliseconds", - opts.RunningOpsInterval), - ) - - gop.StringVarLong(&opts.SSLCAFile, "sslCAFile", 0, "SSL CA cert file used for authentication") - gop.StringVarLong(&opts.SSLPEMKeyFile, "sslPEMKeyFile", 0, "SSL client PEM file used for authentication") - - gop.SetParameters("host[:port]") - gop.Parse(os.Args) - - if gop.NArgs() > 0 { - if gop.IsSet("host") || gop.IsSet("port") || gop.IsSet("uri") { - return nil, fmt.Errorf(`parameter host[:port] is not compatible with "--uri", "--host" and "--port" flags set`) - } - var err error - opts.Host, opts.Port, err = net.SplitHostPort(gop.Arg(0)) - if err != nil { - return nil, err - } - gop.Parse(gop.Args()) - } - - if gop.IsSet("password") && opts.Password == "" { - print("Password: ") - - pass, err := term.ReadPassword(0) - if err != nil { - return opts, err - } - - opts.Password = string(pass) - } - - if gop.IsSet("uri") && (gop.IsSet("host") || gop.IsSet("port")) { - return nil, fmt.Errorf("If a full URI is provided, you cannot also specify --host or --port") - } - - if opts.Help { - gop.PrintUsage(os.Stdout) - - return nil, nil - } - - if opts.OutputFormat != "json" && opts.OutputFormat != "text" { - log.Infof("Invalid output format '%s'. Using text format", opts.OutputFormat) - } - - return opts, nil -} - func getChunksCount(ctx context.Context, client *mongo.Client) ([]proto.ChunksByCollection, error) { var result []proto.ChunksByCollection @@ -1112,7 +1060,7 @@ func getClientOptions(opts *cliOptions) (*options.ClientOptions, error) { auth.Username = opts.User } if opts.Password != "" { - auth.Password = opts.Password + auth.Password = string(opts.Password) auth.PasswordSet = true } diff --git a/src/go/pt-mongodb-summary/main_test.go b/src/go/pt-mongodb-summary/main_test.go index 795453572..c96b12abb 100644 --- a/src/go/pt-mongodb-summary/main_test.go +++ b/src/go/pt-mongodb-summary/main_test.go @@ -14,17 +14,16 @@ package main import ( - "testing" - "time" - "context" "os" - "reflect" + "testing" + "time" + "github.com/alecthomas/kong" + "github.com/percona/percona-toolkit/src/go/lib/config" "github.com/stretchr/testify/assert" "go.mongodb.org/mongo-driver/mongo/options" - "github.com/pborman/getopt" "github.com/stretchr/testify/require" tu "github.com/percona/percona-toolkit/src/go/internal/testutils" @@ -114,97 +113,82 @@ func TestClusterWideInfo(t *testing.T) { } } -func TestParseFlags(t *testing.T) { - tests := []struct { - name string - args []string - want *cliOptions - wantErr bool - }{ - { - name: "Default values", - args: []string{toolname}, - want: &cliOptions{ - Host: "", - LogLevel: DefaultLogLevel, - AuthDB: DefaultAuthDB, - RunningOpsSamples: DefaultRunningOpsSamples, - RunningOpsInterval: DefaultRunningOpsInterval, - OutputFormat: "text", - }, - }, - { - name: "URI only", - args: []string{toolname, "--uri", "mongodb://test:27017"}, - want: &cliOptions{ - URI: "mongodb://test:27017", - LogLevel: DefaultLogLevel, - AuthDB: DefaultAuthDB, - RunningOpsSamples: DefaultRunningOpsSamples, - RunningOpsInterval: DefaultRunningOpsInterval, - OutputFormat: "text", - }, - }, - { - name: "Legacy positional host:port", - args: []string{toolname, "test.example.com:27019"}, - want: &cliOptions{ - Host: "test.example.com", - Port: "27019", - LogLevel: DefaultLogLevel, - AuthDB: DefaultAuthDB, - RunningOpsSamples: DefaultRunningOpsSamples, - RunningOpsInterval: DefaultRunningOpsInterval, - OutputFormat: "text", - }, - }, - { - name: "Error: URI and Host together", - args: []string{toolname, "--uri", "mongodb://test", "--host", "localhost"}, - wantErr: true, - }, - { - name: "Error: Positional arg and Host flag together", - args: []string{toolname, "--host", "newhost", "legacy:27017"}, - wantErr: true, - }, - { - name: "Help flag returns nil options", - args: []string{toolname, "--help"}, - want: nil, - }, - } - - // Backup and silence stdout - oldStdout := os.Stdout - _, w, _ := os.Pipe() - os.Stdout = w - defer func() { os.Stdout = oldStdout }() +// NOTE: legacy getopt-based parsing tests were removed after the CLI migration to Kong. - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - getopt.Reset() - os.Args = tt.args - - got, err := parseFlags() +func withArgs(args []string, fn func()) { + oldArgs := os.Args + os.Args = args + defer func() { os.Args = oldArgs }() + fn() +} - if tt.wantErr { - if err == nil { - t.Errorf("expected error but got none") - } - return +func TestParseFlagsKong(t *testing.T) { + t.Run("default values", func(t *testing.T) { + withArgs([]string{toolname, "--no-version-check"}, func() { + var opts cliOptions + _, _, err := config.Setup( + toolname, + &opts, + kong.UsageOnError(), + kong.Vars{ + "default_host": DefaultHost, + "default_port": DefaultPort, + "version": "test-version", + }, + ) + if err != nil { + t.Fatalf("config.Setup() error = %v", err) } + if opts.Host != DefaultHost { + t.Fatalf("opts.Host = %q, want %q", opts.Host, DefaultHost) + } + if opts.Port != DefaultPort { + t.Fatalf("opts.Port = %q, want %q", opts.Port, DefaultPort) + } + if opts.AuthDB != "admin" { + t.Fatalf("opts.AuthDB = %q, want %q", opts.AuthDB, "admin") + } + if opts.LogLevel != "warn" { + t.Fatalf("opts.LogLevel = %q, want %q", opts.LogLevel, "warn") + } + if opts.OutputFormat != "text" { + t.Fatalf("opts.OutputFormat = %q, want %q", opts.OutputFormat, "text") + } + if opts.RunningOpsSamples != 5 { + t.Fatalf("opts.RunningOpsSamples = %d, want %d", opts.RunningOpsSamples, 5) + } + if opts.RunningOpsInterval != 1000 { + t.Fatalf("opts.RunningOpsInterval = %d, want %d", opts.RunningOpsInterval, 1000) + } + }) + }) + + t.Run("explicit host and port flags", func(t *testing.T) { + withArgs([]string{toolname, "--host", "test.example.com", "--port", "27019", "--no-version-check"}, func() { + var opts cliOptions + _, _, err := config.Setup( + toolname, + &opts, + kong.UsageOnError(), + kong.Vars{ + "default_host": DefaultHost, + "default_port": DefaultPort, + "version": "test-version", + }, + ) if err != nil { - t.Errorf("unexpected error: %v", err) - return + t.Fatalf("config.Setup() error = %v", err) } - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("mismatch:\ngot: %+v\nwant: %+v", got, tt.want) + if opts.Host != "test.example.com" { + t.Fatalf("opts.Host = %q, want %q", opts.Host, "test.example.com") + } + if opts.Port != "27019" { + t.Fatalf("opts.Port = %q, want %q", opts.Port, "27019") } }) - } + }) } func TestGetClientOptions(t *testing.T) { diff --git a/src/go/pt-pg-summary/README.rst b/src/go/pt-pg-summary/README.rst index bf2689a58..6917900a3 100644 --- a/src/go/pt-pg-summary/README.rst +++ b/src/go/pt-pg-summary/README.rst @@ -11,16 +11,26 @@ Usage .. code-block:: bash - pt-pg-summary [OPTIONS] [HOST:[PORT]] + pt-pg-summary [OPTIONS] Options ------- -``--help``, ``--help-long``, ``--help-man`` - Shows context-sensitive help. ``--help-long`` and ``--help-man`` provide more verbose output. +``--config`` + +List of Percona Toolkit configuration file(s) separated by a comma without an equal sign. Must be a first flag. Uses default config file locations if not specified. + +``--help`` + Show help message and exit. ``--version`` - Show application version and exit. | + Show application version and exit. + +``--version-check`` + Check for updates (enabled by default). + +``--no-version-check`` + Disable update checks. ``--databases`` Summarizes this comma-separated list of databases. @@ -30,30 +40,27 @@ Options ``-h``, ``--host`` Host or local Unix socket for connection. -``-W``, ``--password`` - Password to use when connecting. | +``-W``, ``--passwrord`` + Password to use when connecting. ``-p``, ``--port`` - Port number to use for connection. | + Port number to use for connection. ``--sleep`` Seconds to sleep when gathering status counters. Sleeps 10 seconds if not provided. -``-U``, ``--username`` +``-u``, ``--username`` User for login if not current user. -``--disable-ssl`` - Disable SSL for the connection. - - Enabled by default. +``--ssl`` / ``--no-ssl`` + Enable or disable SSL for the connection. + Default: disabled. -``--verbose`` - Show verbose log. - -``--debug`` - Show debug information in the logs. +``-l``, ``--log-level`` + Specifies the log level: + ``panic``, ``fatal``, ``error``, ``warn``, ``info``, ``debug`` Experimental Options @@ -62,12 +69,6 @@ Experimental Options ``--list-encrypted-tables`` Include a list of the encrypted tables in all databases. -``--ask-pass`` - Prompt for a password when connecting to PostgreSQL. - -``--config`` - Configuration file. - ``--defaults-file`` Only read PostgreSQL options from the given file. @@ -547,4 +548,3 @@ VERSION ======= :program:`pt-pg-summary` 3.7.1 - diff --git a/src/go/pt-pg-summary/main.go b/src/go/pt-pg-summary/main.go index d1ddfaa4e..2668dce1b 100644 --- a/src/go/pt-pg-summary/main.go +++ b/src/go/pt-pg-summary/main.go @@ -20,13 +20,15 @@ import ( "strings" "text/template" - "github.com/alecthomas/kingpin" + "github.com/alecthomas/kong" _ "github.com/lib/pq" "github.com/pkg/errors" - "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus" + "golang.org/x/term" + "github.com/percona/percona-toolkit/src/go/lib/config" "github.com/percona/percona-toolkit/src/go/lib/pginfo" + "github.com/percona/percona-toolkit/src/go/lib/versioncheck" "github.com/percona/percona-toolkit/src/go/pt-pg-summary/templates" ) @@ -43,50 +45,86 @@ var ( ) type connOpts struct { - Host string - Port int - User string - Password string - DisableSSL bool + Host string `name:"host" short:"h" help:"Host to connect to"` + Port int `name:"port" short:"p" help:"Port number to use for connection"` + User string `name:"username" short:"u" help:"User for login if not current user"` + Password config.StdinRequestString `name:"passwrord" short:"W" help:"Password to use when connecting"` + SSL bool `name:"ssl" help:"Enable SSL for the connection" default:"false" negatable:""` } type cliOptions struct { - app *kingpin.Application - connOpts connOpts - Config string - DefaultsFile string - ReadSamples string - SaveSamples string - Databases []string - Seconds int - AllDatabases bool - AskPass bool - ListEncryptedTables bool - Verbose bool - Debug bool + config.ConfigFlag + connOpts + ReadSamples string `name:"read-samples" hidden:"" help:"Create a report from the files found in this directory"` + SaveSamples string `name:"save-samples" hidden:"" help:"Save the data files used to generate the summary in this directory"` + Databases []string `name:"databases" help:"Summarize this comma-separated list of databases. All if not specified"` + Seconds int `name:"sleep" help:"Seconds to sleep when gathering status counters" default:"10"` + DefaultsFile string `name:"defaults-file" hidden:"" help:"Only read PostgreSQL options from the given file"` + ListEncryptedTables bool `name:"list-encrypted-tables" hidden:"" help:"Include a list of the encrypted tables in all databases"` + LogLevel string `name:"log-level" short:"l" help:"Log level: panic, fatal, error, warn, info, debug." default:"warn"` + config.VersionFlag + config.VersionCheckFlag +} + +func (c *cliOptions) AfterApply() error { + err := c.connOpts.Password.Request(func() (string, error) { + print("Password: ") + pass, err := term.ReadPassword(0) + return string(pass), err + }) + if err != nil { + return err + } + + logLevel, err := log.ParseLevel(c.LogLevel) + if err != nil { + fmt.Printf("cannot set log level: %s", err.Error()) + } + + log.SetLevel(logLevel) + + if c.VersionCheck { + advice, err := versioncheck.CheckUpdates(toolname, Version) + if err != nil { + log.Errorf("cannot check version updates: %s", err.Error()) + } else if advice != "" { + log.Infof("%s", advice) + } + } + + return nil } func main() { - opts, err := parseCommandLineOpts(os.Args[1:]) + var opts cliOptions + kongCtx, _, err := config.Setup( + toolname, + &opts, + kong.UsageOnError(), + kong.Vars{ + "version": fmt.Sprintf( + "%s\nVersion %s\nBuild: %s using %s\nCommit: %s", + toolname, Version, Build, GoVersion, Commit, + ), + }, + ) if err != nil { - fmt.Printf("Cannot parse command line arguments: %s", err) + log.Errorf("cannot get parameters: %s", err.Error()) os.Exit(1) } - logger := logrus.New() - if opts.Verbose { - logger.SetLevel(logrus.InfoLevel) - } - if opts.Debug { - logger.SetLevel(logrus.DebugLevel) + + if opts.Version { + return } dsn := buildConnString(opts.connOpts, "postgres") + logger := log.New() logger.Infof("Connecting to the database server using: %s", safeConnString(opts.connOpts, "postgres")) db, err := connect(dsn) if err != nil { logger.Errorf("Cannot connect to the database: %s\n", err) - opts.app.Usage(os.Args[1:]) + kongCtx.PrintUsage(true) os.Exit(1) } logger.Infof("Connection OK") @@ -189,7 +227,9 @@ func buildConnString(opts connOpts, dbName string) string { if opts.Password != "" { parts = append(parts, fmt.Sprintf("password=%s", opts.Password)) } - if opts.DisableSSL { + if opts.SSL { + parts = append(parts, "sslmode=enable") + } else { parts = append(parts, "sslmode=disable") } if dbName == "" { @@ -216,7 +256,9 @@ func safeConnString(opts connOpts, dbName string) string { if opts.Password != "" { parts = append(parts, "password=******") } - if opts.DisableSSL { + if opts.SSL { + parts = append(parts, "sslmode=enable") + } else { parts = append(parts, "sslmode=disable") } if dbName == "" { @@ -226,56 +268,3 @@ func safeConnString(opts connOpts, dbName string) string { return strings.Join(parts, " ") } - -func parseCommandLineOpts(args []string) (cliOptions, error) { - app := kingpin.New(toolname, "Percona Toolkit - PostgreSQL Summary") - app.UsageWriter(os.Stdout) - // version, commit and date will be set at build time by the compiler -ldflags param - app.Version(fmt.Sprintf("%s\nVersion %s\nBuild: %s using %s\nCommit: %s", - app.Name, Version, Build, GoVersion, Commit)) - opts := cliOptions{app: app} - - app.Flag("ask-pass", "Prompt for a password when connecting to PostgreSQL"). - Hidden().BoolVar(&opts.AskPass) // hidden because it is not implemented yet - app.Flag("config", "Config file"). - Hidden().StringVar(&opts.Config) // hidden because it is not implemented yet - app.Flag("databases", "Summarize this comma-separated list of databases. All if not specified"). - StringsVar(&opts.Databases) - app.Flag("defaults-file", "Only read PostgreSQL options from the given file"). - Hidden().StringVar(&opts.DefaultsFile) // hidden because it is not implemented yet - app.Flag("host", "Host to connect to"). - Short('h'). - StringVar(&opts.connOpts.Host) - app.Flag("list-encrypted-tables", "Include a list of the encrypted tables in all databases"). - Hidden().BoolVar(&opts.ListEncryptedTables) - app.Flag("password", "Password to use when connecting"). - Short('W'). - StringVar(&opts.connOpts.Password) - app.Flag("port", "Port number to use for connection"). - Short('p'). - IntVar(&opts.connOpts.Port) - app.Flag("read-samples", "Create a report from the files found in this directory"). - Hidden().StringVar(&opts.ReadSamples) // hidden because it is not implemented yet - app.Flag("save-samples", "Save the data files used to generate the summary in this directory"). - Hidden().StringVar(&opts.SaveSamples) // hidden because it is not implemented yet - app.Flag("sleep", "Seconds to sleep when gathering status counters"). - Default("10").IntVar(&opts.Seconds) - app.Flag("username", "User for login if not current user"). - Short('U'). - StringVar(&opts.connOpts.User) - app.Flag("disable-ssl", "Disable SSL for the connection"). - Default("true").BoolVar(&opts.connOpts.DisableSSL) - app.Flag("verbose", "Show verbose log"). - Default("false").BoolVar(&opts.Verbose) - app.Flag("debug", "Show debug information in the logs"). - Default("false").BoolVar(&opts.Debug) - _, err := app.Parse(args) - - dbs := []string{} - for _, databases := range opts.Databases { - ds := strings.Split(databases, ",") - dbs = append(dbs, ds...) - } - opts.Databases = dbs - return opts, err -} diff --git a/src/go/pt-pg-summary/main_test.go b/src/go/pt-pg-summary/main_test.go index 67613a43f..030b0ee9c 100644 --- a/src/go/pt-pg-summary/main_test.go +++ b/src/go/pt-pg-summary/main_test.go @@ -50,8 +50,11 @@ func TestMain(m *testing.M) { } func TestConnection(t *testing.T) { - // use an "external" IP to simulate a remote host - tests := append(tests, Test{"remote_host", tu.PG9DockerIP, tu.DefaultPGPort, tu.Username, tu.Password}) + tests := append([]Test{}, tests...) + // use an "external" IP to simulate a remote host when available. + if tu.PG9DockerIP != "" { + tests = append(tests, Test{"remote_host", tu.PG9DockerIP, tu.DefaultPGPort, tu.Username, tu.Password}) + } // use IPV6 for PostgreSQL 9 // tests := append(tests, Test{"IPV6", tu.IPv6Host, tu.IPv6PG9Port, tu.Username, tu.Password}) for _, test := range tests { diff --git a/src/go/pt-secure-collect/README.rst b/src/go/pt-secure-collect/README.rst index 39507a300..148f32b0c 100644 --- a/src/go/pt-secure-collect/README.rst +++ b/src/go/pt-secure-collect/README.rst @@ -26,20 +26,33 @@ By default, :program:`pt-secure-collect` will collect the output of: Global flags ------------ +.. option:: --config + + List of Percona Toolkit configuration file(s) separated by a comma without an equal sign. Must be a first flag. Uses default config file locations if not specified. + .. option:: --help - Show context-sensitive help (also try --help-long and --help-man). + Show help and exit. .. option:: --debug Enable debug log level. -COMMANDS -======== +.. option:: --version + + Show version and exit. + +.. option:: --version-check + + Check for updates (enabled by default). -* **Help command** +.. option:: --no-version-check - Show help + Disable update checks. + + +COMMANDS +======== * **Collect command** @@ -82,10 +95,6 @@ COMMANDS MySQL password. - .. option:: --ask-mysql-pass - - Ask MySQL password. - .. option:: --extra-cmd Also run this command as part of the data collection. This parameter can @@ -96,29 +105,29 @@ COMMANDS Encrypt the output file using this password. If omitted, it will be asked in the command line. - .. option:: --no-collect + .. option:: --collect / --no-collect - Do not collect data + Enable or disable archive encryption stage. - .. option:: --no-sanitize + .. option:: --sanitize / --no-sanitize - Do not sanitize data + Enable or disable sanitization stage. - .. option:: --no-encrypt + .. option:: --encrypt / --no-encrypt - Do not encrypt the output file. + Enable or disable hostname sanitization. - .. option:: --no-sanitize-hostnames + .. option:: --sanitize-hostnames / --no-sanitize-hostnames - Do not sanitize hostnames. + Enable or disable query fingerprint sanitization. - .. option:: --no-sanitize-queries + .. option:: --sanitize-queries / --no-sanitize-queries - Do not replace queries by their fingerprints. + Enable or disable command data collection. - .. option:: --no-remove-temp-files + .. option:: --remove-temp-files / --no-remove-temp-files - Do not remove temporary files. + Keep or remove temporary files. * **Decrypt command** @@ -127,12 +136,20 @@ COMMANDS :: - pt-secure-collect decrypt [flags] + pt-secure-collect decrypt --infile= [flags] + + .. option:: --infile + + Encrypted input file (required). .. option:: --outfile Write the output to this file. If omitted, the output file - name will be the same as the input file, adding the ``.aes`` extension. + name is inferred from input file by removing ``.aes`` extension. + + .. option:: --encrypt-password + + Password used for decryption. If omitted, it will be requested interactively. * **Encrypt command** @@ -140,12 +157,20 @@ COMMANDS :: - pt-secure-collect encrypt [flags] + pt-secure-collect encrypt --infile= [flags] + + .. option:: --infile + + Unencrypted input file (required). .. option:: --outfile Write the output to this file. If omitted, the output file - name will be the same as the input file, without the ``.aes`` extension. + name will be the same as the input file, adding the ``.aes`` extension. + + .. option:: --encrypt-password + + Password used for encryption. If omitted, it will be requested interactively. * **Sanitize command** @@ -164,13 +189,13 @@ COMMANDS Output file. If not specified, the input will be Stdout. - .. option:: --no-sanitize-hostnames + .. option:: --sanitize-hostnames / --no-sanitize-hostnames - Do not sanitize host names. + Enable or disable hostname sanitization. - .. option:: --no-sanitize-queries + .. option:: --sanitize-queries / --no-sanitize-queries - Do not replace queries by their fingerprints. + Enable or disable query fingerprint sanitization. Authors ======= @@ -210,4 +235,3 @@ VERSION ======= :program:`pt-secure-collect` 3.7.1 - diff --git a/src/go/pt-secure-collect/collect.go b/src/go/pt-secure-collect/collect.go index 076542bab..bcd0d7b35 100644 --- a/src/go/pt-secure-collect/collect.go +++ b/src/go/pt-secure-collect/collect.go @@ -31,37 +31,139 @@ import ( "github.com/pkg/errors" log "github.com/sirupsen/logrus" + "github.com/percona/percona-toolkit/src/go/lib/config" "github.com/percona/percona-toolkit/src/go/pt-secure-collect/sanitize" "github.com/percona/percona-toolkit/src/go/pt-secure-collect/sanitize/util" ) -func collectData(opts *cliOptions) error { - log.Infof("Temp directory is %q", *opts.TempDir) +type CollectCmd struct { + BinDir string `name:"bin-dir" help:"Directory having the Percona Toolkit binaries (if they are not in PATH)."` + TempDir string `name:"temp-dir" help:"Temporary directory used for the data collection." default:"${default_temp_dir}"` // in case Percona Toolkit is not in the PATH + IncludeDirs []string `name:"include-dir" help:"Include this dir into the sanitized tar file"` + ConfigFile string `name:"config-file" help:"Path to the config file." default:"~/.my.cnf"` // .my.cnf file + MySQLHost string `name:"mysql-host" help:"MySQL host."` + MySQLPort int `name:"mysql-port" help:"MySQL port."` + MySQLUser string `name:"mysql-user" help:"MySQL user name."` + MySQLPass config.StdinRequestString `name:"mysql-password" help:"MySQL password."` //TODO: list in changed + + AdditionalCmds []string `name:"extra-cmd" help:"Also run this command as part of the data collection. This parameter can be used more than once."` + EncryptPassword config.StdinRequestString `name:"encrypt-password" help:"Encrypt the output file using this password. If omitted, the file won't be encrypted."` // if set, it will produce an encrypted .aes file + + Encrypt bool `name:"collect" negatable:"" default:"true"` + Sanitize bool `name:"sanitize" negatable:"" default:"true"` + SanitizeHostnames bool `name:"encrypt" negatable:"" default:"true"` + SanitizeQueries bool `name:"sanitize-hostnames" negatable:"" default:"true"` + Collect bool `name:"sanitize-queries" negatable:"" default:"true"` + RemoveTempFiles bool `name:"remove-temp-files" negatable:"" default:"true"` +} + +func (c *CollectCmd) AfterApply(args ...any) error { + err := c.ParseMySQLConfig() + if err != nil { + return err + } + + err = c.MySQLPass.Request(func() (string, error) { + return askMysqlPassword(c.MySQLUser) + }) + if err != nil { + return err + } + + err = c.EncryptPassword.Request(func() (string, error) { + if !c.Encrypt { + return "", nil + } + return askEncryptionPassword(true) + }) + if err != nil { + return err + } + + c.BinDir = expandHomeDir(c.BinDir) + c.ConfigFile = expandHomeDir(c.ConfigFile) + c.TempDir = expandHomeDir(c.TempDir) + for _, incDir := range c.IncludeDirs { + incDir = expandHomeDir(incDir) + } + + if c.BinDir != "" { + os.Setenv("PATH", fmt.Sprintf("%s%s%s", c.BinDir, string(os.PathListSeparator), os.Getenv("PATH"))) + } + + lp, err := exec.LookPath("pt-summary") + if (err != nil || lp == "") && c.BinDir == "" && c.Collect { + return errors.New("Cannot find Percona Toolkit binaries. Please run this tool again using --bin-dir parameter") + } + + return nil +} + +func (c *CollectCmd) ParseMySQLConfig() error { + mycnf, err := getParamsFromMyCnf(c.ConfigFile) + if err != nil { + return err + } + + if c.MySQLPort == 0 && mycnf.MySQLPort > 0 { + log.Debugf("Setting default port from config file") + c.MySQLPort = mycnf.MySQLPort + } + if c.MySQLHost == "" && mycnf.MySQLHost != "" { + c.MySQLHost = mycnf.MySQLHost + log.Debugf("Setting default host from config file") + } + if c.MySQLUser == "" && mycnf.MySQLUser != "" { + log.Debugf("Setting default user from config file") + c.MySQLUser = mycnf.MySQLUser + } + if c.MySQLPass == "" && mycnf.MySQLPass != "" { + log.Debugf("Setting default password from config file") + c.MySQLPass = config.StdinRequestString(mycnf.MySQLPass) + } + + if c.MySQLHost == "" { + log.Debugf("MySQL host is empty. Setting it to %s", defaultMySQLHost) + c.MySQLHost = defaultMySQLHost + } + if c.MySQLPort == 0 { + log.Debugf("MySQL port is empty. Setting it to %d", defaultMySQLPort) + c.MySQLPort = defaultMySQLPort + } + if c.MySQLUser == "" { + return fmt.Errorf("MySQL user cannot be empty") + } + + return nil +} + +func (c *CollectCmd) Run() error { + log.Infof("Temp directory is %q", c.TempDir) - if !*opts.NoCollect { - cmds, safeCmds, err := getCommandsToRun(defaultCmds, opts) + if c.Collect { + cmds, safeCmds, err := c.getCommandsToRun(defaultCmds) // Run the commands - if err = runCommands(cmds, safeCmds, *opts.TempDir); err != nil { + if err = runCommands(cmds, safeCmds, c.TempDir); err != nil { return errors.Wrap(err, "Cannot run data collection commands") } } - if !*opts.NoSanitize { + if c.Sanitize { log.Infof("Sanitizing output collected data") - err := processFiles(*opts.TempDir, *opts.IncludeDirs, *opts.TempDir, !*opts.NoSanitizeHostnames, !*opts.NoSanitizeQueries) + err := processFiles(c.TempDir, c.IncludeDirs, c.TempDir, c.SanitizeHostnames, c.SanitizeQueries) if err != nil { - return errors.Wrapf(err, "Cannot sanitize files in %q", *opts.TempDir) + return errors.Wrapf(err, "Cannot sanitize files in %q", c.TempDir) } } - tarFile := fmt.Sprintf(path.Join(*opts.TempDir, path.Base(*opts.TempDir)+".tar.gz")) + tarFile := path.Join(c.TempDir, path.Base(c.TempDir)+".tar.gz") log.Infof("Creating tar file %q", tarFile) - if err := tarit(tarFile, []string{*opts.TempDir}); err != nil { + if err := tarit(tarFile, []string{c.TempDir}); err != nil { return err } - if !*opts.NoEncrypt && *opts.EncryptPassword != "" { - key, err := deriveKey(*opts.EncryptPassword) + if c.Encrypt { + key, err := deriveKey(string(c.EncryptPassword)) if err != nil { return errors.WithStack(err) } @@ -156,7 +258,7 @@ func tarit(outfile string, srcPaths []string) error { return nil } -func getCommandsToRun(defaultCmds []string, opts *cliOptions) ([]*exec.Cmd, []string, error) { +func (c *CollectCmd) getCommandsToRun(defaultCmds []string) ([]*exec.Cmd, []string, error) { log.Debug("Default commands to run:") for i, cmd := range defaultCmds { log.Debugf("%02d) %s", i, cmd) @@ -166,22 +268,22 @@ func getCommandsToRun(defaultCmds []string, opts *cliOptions) ([]*exec.Cmd, []st safeCmds := []string{} notAllowedCmdsRe := regexp.MustCompile("(rm|fdisk|rmdir)") - if !*opts.NoCollect { + if c.Collect { cmdList = append(cmdList, defaultCmds...) } - if *opts.AdditionalCmds != nil { - cmdList = append(cmdList, *opts.AdditionalCmds...) + if c.AdditionalCmds != nil { + cmdList = append(cmdList, c.AdditionalCmds...) } for _, cmdstr := range cmdList { - cmdstr = strings.Replace(cmdstr, "$mysql-host", *opts.MySQLHost, -1) - cmdstr = strings.Replace(cmdstr, "$mysql-port", fmt.Sprintf("%d", *opts.MySQLPort), -1) - cmdstr = strings.Replace(cmdstr, "$mysql-user", *opts.MySQLUser, -1) - cmdstr = strings.Replace(cmdstr, "$temp-dir", *opts.TempDir, -1) + cmdstr = strings.Replace(cmdstr, "$mysql-host", c.MySQLHost, -1) + cmdstr = strings.Replace(cmdstr, "$mysql-port", fmt.Sprintf("%d", c.MySQLPort), -1) + cmdstr = strings.Replace(cmdstr, "$mysql-user", c.MySQLUser, -1) + cmdstr = strings.Replace(cmdstr, "$temp-dir", c.TempDir, -1) safeCmd := cmdstr safeCmd = strings.Replace(safeCmd, "$mysql-pass", "********", -1) - cmdstr = strings.Replace(cmdstr, "$mysql-pass", *opts.MySQLPass, -1) + cmdstr = strings.Replace(cmdstr, "$mysql-pass", string(c.MySQLPass), -1) args, err := shellwords.Parse(cmdstr) if err != nil { diff --git a/src/go/pt-secure-collect/ecncrypt_test.go b/src/go/pt-secure-collect/ecncrypt_test.go index ac6103cd3..8300b8a36 100644 --- a/src/go/pt-secure-collect/ecncrypt_test.go +++ b/src/go/pt-secure-collect/ecncrypt_test.go @@ -17,7 +17,6 @@ import ( "os" "testing" - "github.com/AlekSi/pointer" "github.com/stretchr/testify/require" ) @@ -39,12 +38,12 @@ func TestEncrypt(t *testing.T) { require.NoError(t, err) t.Cleanup(func() { os.Remove(encrypted.Name()) }) - err = encryptorCmd(&cliOptions{ - Command: "encrypt", - EncryptPassword: pointer.ToString("password"), - EncryptInFile: pointer.ToString(input.Name()), - EncryptOutFile: pointer.ToString(encrypted.Name()), - }) + encryptCmd := &EncryptCmd{ + EncryptInFile: input.Name(), + EncryptOutFile: encrypted.Name(), + EncryptPassword: "password", + } + err = encryptCmd.Run() require.NoError(t, err) encryptedData, err := os.ReadFile(encrypted.Name()) @@ -53,12 +52,12 @@ func TestEncrypt(t *testing.T) { // Check that the encrypted data is different from the original data require.NotEqual(t, data, encryptedData) - err = encryptorCmd(&cliOptions{ - Command: "decrypt", - EncryptPassword: pointer.ToString("password"), - DecryptInFile: pointer.ToString(encrypted.Name()), - DecryptOutFile: pointer.ToString(output.Name()), - }) + decryptCmd := &DecryptCmd{ + DecryptInFile: encrypted.Name(), + DecryptOutFile: output.Name(), + EncryptPassword: "password", + } + err = decryptCmd.Run() require.NoError(t, err) decryptedData, err := os.ReadFile(output.Name()) diff --git a/src/go/pt-secure-collect/encrypt.go b/src/go/pt-secure-collect/encrypt.go index ed8f14768..03b4df0c3 100644 --- a/src/go/pt-secure-collect/encrypt.go +++ b/src/go/pt-secure-collect/encrypt.go @@ -17,11 +17,13 @@ import ( "crypto/aes" "crypto/cipher" "crypto/sha256" + "fmt" "io" "os" "path/filepath" "strings" + "github.com/percona/percona-toolkit/src/go/lib/config" "github.com/pkg/errors" log "github.com/sirupsen/logrus" "golang.org/x/crypto/hkdf" @@ -52,6 +54,68 @@ var ( } ) +type EncryptCmd struct { + EncryptInFile string `name:"infile" help:"Unencrypted file." required:""` + EncryptOutFile string `name:"outfile" help:"Encrypted file. Default: .aes"` + EncryptPassword config.StdinRequestString `name:"encrypt-password" help:"Encrypt the output file using this password. If omitted, the file won't be encrypted."` // if set, it will produce an encrypted .aes file +} + +func (c *EncryptCmd) AfterApply(args ...any) error { + if c.EncryptOutFile == "" { + c.EncryptOutFile = filepath.Base(c.EncryptInFile) + ".aes" + } + + err := c.EncryptPassword.Request(func() (string, error) { + return askEncryptionPassword(true) + }) + if err != nil { + return err + } + return nil +} + +func (c *EncryptCmd) Run() error { + key, err := deriveKey(string(c.EncryptPassword)) + if err != nil { + return errors.WithStack(err) + } + + log.Infof("Encrypting file %q into %q", c.EncryptInFile, c.EncryptOutFile) + return encrypt(c.EncryptInFile, c.EncryptOutFile, key) +} + +type DecryptCmd struct { + DecryptInFile string `name:"infile" help:"Encrypted file." required:""` + DecryptOutFile string `name:"outfile" help:"Unencrypted file. Default: same name without .aes extension"` + EncryptPassword config.StdinRequestString `name:"encrypt-password" help:"Encrypt the output file using this password. If omitted, the file won't be encrypted."` // if set, it will produce an encrypted .aes file +} + +func (c *DecryptCmd) AfterApply(args ...any) error { + if c.DecryptOutFile == "" && strings.HasSuffix(c.DecryptInFile, ".aes") { + c.DecryptOutFile = strings.TrimSuffix(filepath.Base(c.DecryptInFile), ".aes") + } else if !strings.HasSuffix(c.DecryptInFile, ".aes") { + return fmt.Errorf("Input file does not have .aes extension. I cannot infer the output file") + } + + err := c.EncryptPassword.Request(func() (string, error) { + return askEncryptionPassword(false) + }) + if err != nil { + return err + } + return nil +} + +func (c *DecryptCmd) Run() error { + key, err := deriveKey(string(c.EncryptPassword)) + if err != nil { + return errors.WithStack(err) + } + + log.Infof("Decrypting file %q into %q", c.DecryptInFile, c.DecryptOutFile) + return decrypt(c.DecryptInFile, c.DecryptOutFile, key) +} + // deriveKey derives a cryptographically strong key from password. func deriveKey(password string) ([]byte, error) { hkdf := hkdf.New(sha256.New, []byte(password), salt[:], hkdfInfo) @@ -63,29 +127,6 @@ func deriveKey(password string) ([]byte, error) { return key, nil } -func encryptorCmd(opts *cliOptions) (err error) { - key, err := deriveKey(*opts.EncryptPassword) - if err != nil { - return errors.WithStack(err) - } - - switch opts.Command { - case "decrypt": - if *opts.DecryptOutFile == "" && strings.HasSuffix(*opts.DecryptInFile, ".aes") { - *opts.DecryptOutFile = strings.TrimSuffix(filepath.Base(*opts.DecryptInFile), ".aes") - } - log.Infof("Decrypting file %q into %q", *opts.DecryptInFile, *opts.DecryptOutFile) - err = decrypt(*opts.DecryptInFile, *opts.DecryptOutFile, key) - case "encrypt": - if *opts.EncryptOutFile == "" { - *opts.EncryptOutFile = filepath.Base(*opts.EncryptInFile) + ".aes" - } - log.Infof("Encrypting file %q into %q", *opts.EncryptInFile, *opts.EncryptOutFile) - err = encrypt(*opts.EncryptInFile, *opts.EncryptOutFile, key) - } - return -} - func encrypt(infile, outfile string, key []byte) error { inFile, err := os.Open(infile) if err != nil { diff --git a/src/go/pt-secure-collect/main.go b/src/go/pt-secure-collect/main.go index 887a861e0..82b9df1f4 100644 --- a/src/go/pt-secure-collect/main.go +++ b/src/go/pt-secure-collect/main.go @@ -15,18 +15,18 @@ package main import ( "fmt" - "io" "io/ioutil" "os" - "os/exec" "os/user" "path" "path/filepath" "strings" "time" - "github.com/alecthomas/kingpin" + "github.com/alecthomas/kong" "github.com/go-ini/ini" + "github.com/percona/percona-toolkit/src/go/lib/config" + "github.com/percona/percona-toolkit/src/go/lib/versioncheck" "github.com/pkg/errors" "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus" @@ -34,42 +34,35 @@ import ( ) type cliOptions struct { - Command string - Debug *bool - - DecryptCommand *kingpin.CmdClause - DecryptInFile *string - DecryptOutFile *string - - EncryptCommand *kingpin.CmdClause - EncryptInFile *string - EncryptOutFile *string - - CollectCommand *kingpin.CmdClause - BinDir *string - TempDir *string // in case Percona Toolkit is not in the PATH - IncludeDirs *[]string - ConfigFile *string // .my.cnf file - EncryptPassword *string // if set, it will produce an encrypted .aes file - AdditionalCmds *[]string - AskMySQLPass *bool - MySQLHost *string - MySQLPort *int - MySQLUser *string - MySQLPass *string - - NoEncrypt *bool - NoSanitize *bool - NoSanitizeHostnames *bool - NoSanitizeQueries *bool - NoCollect *bool - NoRemoveTempFiles *bool - - SanitizeCommand *kingpin.CmdClause - SanitizeInputFile *string - SanitizeOutputFile *string - DontSanitizeHostnames *bool - DontSanitizeQueries *bool + config.ConfigFlag + Debug bool `name:"debug" help:"Enable debug log level."` + + DecryptCommand DecryptCmd `name:"decrypt" cmd:"" help:"Decrypt an encrypted file. The password will be requested from the terminal."` + + EncryptCommand EncryptCmd `name:"encrypt" cmd:"" help:"Encrypt a file. The password will be requested from the terminal."` + + CollectCommand CollectCmd `name:"collect" cmd:"" help:"Collect, sanitize, pack and encrypt data from pt-tools."` + + SanitizeCommand SanitizeCmd `name:"sanitize" cmd:"" help:"Replace queries in a file by their fingerprints and obfuscate hostnames."` + config.VersionFlag + config.VersionCheckFlag +} + +func (c *cliOptions) AfterApply(args ...any) error { + if c.VersionCheck { + advice, err := versioncheck.CheckUpdates(toolname, Version) + if err != nil { + log.Infof("cannot check version updates: %s", err.Error()) + } else if advice != "" { + log.Infof("%s", advice) + } + } + + if c.Debug { + log.SetLevel(log.DebugLevel) + } + + return nil } type myDefaults struct { @@ -91,6 +84,7 @@ const ( ) var ( + CLI = &cliOptions{} defaultCmds = []string{ "pt-stalk --no-stalk --iterations=2 --sleep=30 --host=$mysql-host --dest=$temp-dir --port=$mysql-port --user=$mysql-user --password=$mysql-pass", "pt-summary", @@ -115,36 +109,27 @@ func main() { log.Fatalf("Cannot get current user: %s", err) } - opts, err := processCliParams(u.HomeDir, nil) + kCtx, _, err := config.Setup(toolname, CLI, + kong.Description("Collect, sanitize, pack and encrypt data.\nBy default, this program will collect the output of:\n "+strings.Join(defaultCmds, "\n ")), + kong.Vars{ + "default_temp_dir": path.Join(u.HomeDir, fmt.Sprintf("data_collection_%s", time.Now().Format("2006-01-02_15_04_05"))), + "version": fmt.Sprintf( + "%s\nVersion %s\nBuild: %s using %s\nCommit: %s", + toolname, Version, Build, GoVersion, Commit, + ), + }, + ) //TODO fix help if err != nil { - log.Fatal(err) + log.Errorf("cannot get parameters: %s", err.Error()) + os.Exit(1) } - switch opts.Command { - case collectCmd: - if _, err = os.Stat(*opts.TempDir); os.IsNotExist(err) { - log.Infof("Creating temporary directory: %s", *opts.TempDir) - if err = os.Mkdir(*opts.TempDir, os.ModePerm); err != nil { - log.Fatalf("Cannot create temporary dirextory %q: %s", *opts.TempDir, err) - } - } - err = collectData(opts) - if err != nil && !*opts.NoRemoveTempFiles { - log.Fatal(err) - } - if !*opts.NoRemoveTempFiles { - if err = removeTempFiles(*opts.TempDir, !*opts.NoEncrypt); err != nil { - log.Fatal(err) - } - } - case encryptCmd, decryptCmd: - err = encryptorCmd(opts) - case sanitizeCmd: - err = sanitizeFile(opts) - } - if err != nil { - log.Fatal(err) + if CLI.Version { + return } + + err = kCtx.Run() + kCtx.FatalIfErrorf(err) } func removeTempFiles(tempDir string, removeTarFile bool) error { @@ -174,196 +159,34 @@ func removeTempFiles(tempDir string, removeTarFile bool) error { return nil } -func processCliParams(baseTempPath string, usageWriter io.Writer) (*cliOptions, error) { - var err error - tmpdir := path.Join(baseTempPath, fmt.Sprintf("data_collection_%s", time.Now().Format("2006-01-02_15_04_05"))) - - // Do not remove the extra space after \n. That's to trick the help template to not to remove the new line - msg := "Collect, sanitize, pack and encrypt data.\nBy default, this program will collect the output of:" - for _, cmd := range defaultCmds { - msg += "\n " + cmd - } - msg += "\n " - - app := kingpin.New(toolname, msg) - if usageWriter != nil { - app.UsageWriter(usageWriter) - app.Terminate(nil) - } else { - app.UsageWriter(os.Stdout) - } - - // Add support for --version flag - app.Version(toolname + "\nVersion " + Version + "\nBuild: " + Build + " using " + GoVersion + - "\nCommit:" + Commit) - - opts := &cliOptions{ - CollectCommand: app.Command(collectCmd, "Collect, sanitize, pack and encrypt data from pt-tools."), - DecryptCommand: app.Command(decryptCmd, "Decrypt an encrypted file. The password will be requested from the terminal."), - EncryptCommand: app.Command(encryptCmd, "Encrypt a file. The password will be requested from the terminal."), - SanitizeCommand: app.Command(sanitizeCmd, "Replace queries in a file by their fingerprints and obfuscate hostnames."), - Debug: app.Flag("debug", "Enable debug log level.").Bool(), - } - // Decrypt command flags - opts.DecryptInFile = opts.DecryptCommand.Arg("infile", "Encrypted file.").Required().String() - opts.DecryptOutFile = opts.DecryptCommand.Flag("outfile", "Unencrypted file. Default: same name without .aes extension").String() - - // Encrypt command flags - opts.EncryptInFile = opts.EncryptCommand.Arg("infile", "Unencrypted file.").Required().String() - opts.EncryptOutFile = opts.EncryptCommand.Flag("outfile", "Encrypted file. Default: .aes").String() - - // Collect command flags - opts.BinDir = opts.CollectCommand.Flag("bin-dir", "Directory having the Percona Toolkit binaries (if they are not in PATH).").String() - opts.TempDir = opts.CollectCommand.Flag("temp-dir", "Temporary directory used for the data collection.").Default(tmpdir).String() - opts.IncludeDirs = opts.CollectCommand.Flag("include-dir", "Include this dir into the sanitized tar file").Strings() - // MySQL related flags - opts.ConfigFile = opts.CollectCommand.Flag("config-file", "Path to the config file.").Default("~/.my.cnf").String() - opts.MySQLHost = opts.CollectCommand.Flag("mysql-host", "MySQL host.").String() - opts.MySQLPort = opts.CollectCommand.Flag("mysql-port", "MySQL port.").Int() - opts.MySQLUser = opts.CollectCommand.Flag("mysql-user", "MySQL user name.").String() - opts.MySQLPass = opts.CollectCommand.Flag("mysql-password", "MySQL password.").String() - opts.AskMySQLPass = opts.CollectCommand.Flag("ask-mysql-pass", "Ask MySQL password.").Bool() - // Additional flags - opts.AdditionalCmds = opts.CollectCommand.Flag("extra-cmd", - "Also run this command as part of the data collection. This parameter can be used more than once.").Strings() - opts.EncryptPassword = opts.CollectCommand.Flag("encrypt-password", "Encrypt the output file using this password."+ - " If omitted, the file won't be encrypted.").String() - // No-Flags - opts.NoCollect = opts.CollectCommand.Flag("no-collect", "Do not collect data").Bool() - opts.NoSanitize = opts.CollectCommand.Flag("no-sanitize", "Sanitize data").Bool() - opts.NoEncrypt = opts.CollectCommand.Flag("no-encrypt", "Do not encrypt the output file.").Bool() - opts.NoSanitizeHostnames = opts.CollectCommand.Flag("no-sanitize-hostnames", "Don't sanitize host names.").Bool() - opts.NoSanitizeQueries = opts.CollectCommand.Flag("no-sanitize-queries", "Do not replace queries by their fingerprints.").Bool() - opts.NoRemoveTempFiles = opts.CollectCommand.Flag("no-remove-temp-files", "Do not remove temporary files.").Bool() - - // Sanitize command flags - opts.SanitizeInputFile = opts.SanitizeCommand.Flag("input-file", "Input file. If not specified, the input will be Stdin.").String() - opts.SanitizeOutputFile = opts.SanitizeCommand.Flag("output-file", "Output file. If not specified, the input will be Stdout.").String() - opts.DontSanitizeHostnames = opts.SanitizeCommand.Flag("no-sanitize-hostnames", "Don't sanitize host names.").Bool() - opts.DontSanitizeQueries = opts.SanitizeCommand.Flag("no-sanitize-queries", "Don't replace queries by their fingerprints.").Bool() - - opts.Command, err = app.Parse(os.Args[1:]) +func askMysqlPassword(user string) (string, error) { + fmt.Printf("MySQL password for user %q:", user) + passb, err := term.ReadPassword(0) if err != nil { - return nil, err - } - - if *opts.Debug { - log.SetLevel(log.DebugLevel) - } - - *opts.BinDir = expandHomeDir(*opts.BinDir) - *opts.ConfigFile = expandHomeDir(*opts.ConfigFile) - *opts.TempDir = expandHomeDir(*opts.TempDir) - for _, incDir := range *opts.IncludeDirs { - incDir = expandHomeDir(incDir) + return "", errors.Wrap(err, "Cannot read MySQL password from the terminal") } - - if *opts.BinDir != "" { - os.Setenv("PATH", fmt.Sprintf("%s%s%s", *opts.BinDir, string(os.PathListSeparator), os.Getenv("PATH"))) - } - - lp, err := exec.LookPath("pt-summary") - if (err != nil || lp == "") && *opts.BinDir == "" && opts.Command == "collect" && !*opts.NoCollect { - return nil, errors.New("Cannot find Percona Toolkit binaries. Please run this tool again using --bin-dir parameter") - } - - switch opts.Command { - case collectCmd: - mycnf, err := getParamsFromMyCnf(*opts.ConfigFile) - if err == nil { - if err = validateMySQLParams(opts, mycnf); err != nil { - return nil, err - } - } - if *opts.AskMySQLPass { - if err = askMySQLPassword(opts); err != nil { - return nil, err - } - } - err = askEncryptionPassword(opts, true) - case encryptCmd: - err = askEncryptionPassword(opts, true) - case decryptCmd: - if !strings.HasSuffix(*opts.DecryptInFile, ".aes") && *opts.DecryptOutFile == "" { - return nil, fmt.Errorf("Input file does not have .aes extension. I cannot infer the output file") - } - err = askEncryptionPassword(opts, false) - } - - if err != nil { - return nil, err - } - - return opts, nil + return string(passb), nil } -func validateMySQLParams(opts *cliOptions, mycnf *myDefaults) error { - if *opts.MySQLPort == 0 && mycnf.MySQLPort > 0 { - log.Debugf("Setting default port from config file") - *opts.MySQLPort = mycnf.MySQLPort - } - if *opts.MySQLHost == "" && mycnf.MySQLHost != "" { - *opts.MySQLHost = mycnf.MySQLHost - log.Debugf("Setting default host from config file") - } - if *opts.MySQLUser == "" && mycnf.MySQLUser != "" { - log.Debugf("Setting default user from config file") - *opts.MySQLUser = mycnf.MySQLUser - } - if *opts.MySQLPass == "" && mycnf.MySQLPass != "" { - log.Debugf("Setting default password from config file") - *opts.MySQLPass = mycnf.MySQLPass - } - - if *opts.MySQLHost == "" { - log.Debugf("MySQL host is empty. Setting it to %s", defaultMySQLHost) - *opts.MySQLHost = defaultMySQLHost - } - if *opts.MySQLPort == 0 { - log.Debugf("MySQL port is empty. Setting it to %d", defaultMySQLPort) - *opts.MySQLPort = defaultMySQLPort - } - if *opts.MySQLUser == "" { - return fmt.Errorf("MySQL user cannot be empty") +func askEncryptionPassword(requireConfirmation bool) (string, error) { + fmt.Print("Encryption password: ") + passa, err := term.ReadPassword(0) + if err != nil { + return "", errors.Wrap(err, "Cannot read encryption password from the terminal") } - - return nil -} - -func askMySQLPassword(opts *cliOptions) error { - if *opts.AskMySQLPass { - fmt.Printf("MySQL password for user %q:", *opts.MySQLUser) + fmt.Println("") + if requireConfirmation { + fmt.Print("Re type password: ") passb, err := term.ReadPassword(0) if err != nil { - return errors.Wrap(err, "Cannot read MySQL password from the terminal") - } - *opts.MySQLPass = string(passb) - } - return nil -} - -func askEncryptionPassword(opts *cliOptions, requireConfirmation bool) error { - if !*opts.NoEncrypt && *opts.EncryptPassword == "" { - fmt.Print("Encryption password: ") - passa, err := term.ReadPassword(0) - if err != nil { - return errors.Wrap(err, "Cannot read encryption password from the terminal") + return "", errors.Wrap(err, "Cannot read encryption password confirmation from the terminal") } fmt.Println("") - if requireConfirmation { - fmt.Print("Re type password: ") - passb, err := term.ReadPassword(0) - if err != nil { - return errors.Wrap(err, "Cannot read encryption password confirmation from the terminal") - } - fmt.Println("") - if string(passa) != string(passb) { - return errors.New("Passwords don't match") - } + if string(passa) != string(passb) { + return "", errors.New("Passwords don't match") } - *opts.EncryptPassword = string(passa) } - return nil + return string(passa), nil } func getParamsFromMyCnf(configFile string) (*myDefaults, error) { diff --git a/src/go/pt-secure-collect/main_test.go b/src/go/pt-secure-collect/main_test.go index 4a028cafb..7cc969763 100644 --- a/src/go/pt-secure-collect/main_test.go +++ b/src/go/pt-secure-collect/main_test.go @@ -14,43 +14,11 @@ package main import ( - "bufio" - "bytes" - "os" "os/exec" - "reflect" "regexp" "testing" ) -func TestProcessCliParams(t *testing.T) { - var output bytes.Buffer - writer := bufio.NewWriter(&output) - - tests := []struct { - Args []string - WantOpts *cliOptions - WantErr bool - }{ - { - Args: []string{"pt-sanitize-data", "llll"}, - WantOpts: nil, - WantErr: true, - }, - } - - for i, test := range tests { - os.Args = test.Args - opts, err := processCliParams(os.TempDir(), writer) - writer.Flush() - if test.WantErr && err == nil { - t.Errorf("Test #%d expected error, have nil", i) - } - if !reflect.DeepEqual(opts, test.WantOpts) { - } - } -} - func TestCollect(t *testing.T) { } diff --git a/src/go/pt-secure-collect/sanitize.go b/src/go/pt-secure-collect/sanitize.go index 73c86a0b8..5fba3805b 100644 --- a/src/go/pt-secure-collect/sanitize.go +++ b/src/go/pt-secure-collect/sanitize.go @@ -22,34 +22,41 @@ import ( "github.com/percona/percona-toolkit/src/go/pt-secure-collect/sanitize/util" ) -func sanitizeFile(opts *cliOptions) error { +type SanitizeCmd struct { + SanitizeInputFile string `name:"input-file" help:"Input file. If not specified, the input will be Stdin."` + SanitizeOutputFile string `name:"output-file" help:"Output file. If not specified, the input will be Stdout."` + SanitizeHostnames bool `name:"sanitize-hostnames" negatable:"" default:"true"` + SanitizeQueries bool `name:"sanitize-queries" negatable:"" default:"true"` +} + +func (c *SanitizeCmd) Run() error { var err error ifh := os.Stdin ofh := os.Stdout - if *opts.SanitizeInputFile != "" { - ifh, err = os.Open(*opts.SanitizeInputFile) + if c.SanitizeInputFile != "" { + ifh, err = os.Open(c.SanitizeInputFile) if err != nil { - return errors.Wrapf(err, "Cannot open %q for reading", *opts.SanitizeInputFile) + return errors.Wrapf(err, "Cannot open %q for reading", c.SanitizeInputFile) } } - if *opts.SanitizeOutputFile != "" { - ofh, err = os.Create(*opts.SanitizeOutputFile) + if c.SanitizeOutputFile != "" { + ofh, err = os.Create(c.SanitizeOutputFile) if err != nil { - return errors.Wrapf(err, "Cannot create output file %q", *opts.SanitizeOutputFile) + return errors.Wrapf(err, "Cannot create output file %q", c.SanitizeOutputFile) } } lines, err := util.ReadLinesFromFile(ifh) if err != nil { - return errors.Wrapf(err, "Cannot read input file %q", *opts.SanitizeInputFile) + return errors.Wrapf(err, "Cannot read input file %q", c.SanitizeInputFile) } - sanitized := sanitize.Sanitize(lines, !*opts.DontSanitizeHostnames, !*opts.DontSanitizeQueries) + sanitized := sanitize.Sanitize(lines, c.SanitizeHostnames, c.SanitizeQueries) if err = util.WriteLinesToFile(ofh, sanitized); err != nil { - return errors.Wrapf(err, "Cannot write output file %q", *opts.SanitizeOutputFile) + return errors.Wrapf(err, "Cannot write output file %q", c.SanitizeOutputFile) } return nil