diff --git a/Makefile b/Makefile index 7bb0439a..81a603a6 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help build build-all build-gitlab build-github build-bitbucket build-devops build-gitea test test-unit test-e2e lint clean coverage coverage-html serve-docs +.PHONY: help build build-all build-gitlab build-github build-bitbucket build-devops build-gitea build-circle test test-unit test-e2e lint clean coverage coverage-html serve-docs # Default target help: @@ -12,6 +12,7 @@ help: @echo " make build-bitbucket - Build BitBucket-specific binary" @echo " make build-devops - Build Azure DevOps-specific binary" @echo " make build-gitea - Build Gitea-specific binary" + @echo " make build-circle - Build CircleCI-specific binary" @echo " make test - Run all tests (unit + e2e)" @echo " make test-unit - Run unit tests only" @echo " make test-e2e - Run e2e tests (builds binary first)" @@ -54,8 +55,13 @@ build-gitea: @echo "Building pipeleek-gitea..." CGO_ENABLED=0 go build $(GO_BUILD_FLAGS) -o pipeleek-gitea ./cmd/pipeleek-gitea +# Build CircleCI-specific binary +build-circle: + @echo "Building pipeleek-circle..." + CGO_ENABLED=0 go build $(GO_BUILD_FLAGS) -o pipeleek-circle ./cmd/pipeleek-circle + # Build all binaries -build-all: build build-gitlab build-github build-bitbucket build-devops build-gitea +build-all: build build-gitlab build-github build-bitbucket build-devops build-gitea build-circle @echo "All binaries built successfully" # Run all tests @@ -92,6 +98,10 @@ test-e2e-gitea: build @echo "Running Gitea e2e tests..." PIPELEEK_BINARY=$$(pwd)/pipeleek go test ./tests/e2e/gitea/... -tags=e2e -v +test-e2e-circle: build + @echo "Running CircleCI e2e tests..." + PIPELEEK_BINARY=$$(pwd)/pipeleek go test ./tests/e2e/circle/... -tags=e2e -v + # Generate test coverage report coverage: @echo "Generating coverage report..." @@ -139,4 +149,5 @@ clean: rm -f pipeleek-bitbucket pipeleek-bitbucket.exe rm -f pipeleek-devops pipeleek-devops.exe rm -f pipeleek-gitea pipeleek-gitea.exe + rm -f pipeleek-circle pipeleek-circle.exe go clean -cache -testcache diff --git a/cmd/pipeleek-circle/main.go b/cmd/pipeleek-circle/main.go new file mode 100644 index 00000000..f41aa16f --- /dev/null +++ b/cmd/pipeleek-circle/main.go @@ -0,0 +1,28 @@ +package main + +import ( + "github.com/CompassSecurity/pipeleek/internal/cmd/circle" + "github.com/CompassSecurity/pipeleek/internal/cmd/common" + "github.com/spf13/cobra" +) + +func main() { + common.Run(newRootCmd()) +} + +func newRootCmd() *cobra.Command { + circleCmd := circle.NewCircleRootCmd() + circleCmd.Use = "pipeleek-circle" + circleCmd.Short = "Scan CircleCI logs and artifacts for secrets" + circleCmd.Long = `Pipeleek-Circle scans CircleCI pipelines, logs, test results, and artifacts to detect leaked secrets.` + circleCmd.Version = common.Version + circleCmd.GroupID = "" + + common.SetupPersistentPreRun(circleCmd) + common.AddCommonFlags(circleCmd) + + circleCmd.SetVersionTemplate(`{{.Version}} +`) + + return circleCmd +} diff --git a/docs/introduction/configuration.md b/docs/introduction/configuration.md index b15cc4cc..a5f8dba2 100644 --- a/docs/introduction/configuration.md +++ b/docs/introduction/configuration.md @@ -200,6 +200,32 @@ jenkins: max_builds: 25 # jenkins scan --max-builds ``` +### CircleCI + +```yaml +circle: + url: https://circleci.com + token: circleci-token + + scan: + project: [my-org/my-repo] # circle scan --project (optional if org is set) + vcs: github # circle scan --vcs + org: my-org # circle scan --org (also enables org-wide discovery when project is omitted) + # --org accepts: my-org, github/my-org, or app URL forms like + # https://app.circleci.com/pipelines/github/my-org/my-repo + # Note: org-wide discovery requires token visibility to that org. If not, + # use explicit --project selectors instead. + branch: main # circle scan --branch + status: [success, failed] # circle scan --status + workflow: [build, deploy] # circle scan --workflow + job: [unit-tests, release] # circle scan --job + since: 2026-01-01T00:00:00Z # circle scan --since (RFC3339) + until: 2026-01-31T23:59:59Z # circle scan --until (RFC3339) + max_pipelines: 0 # circle scan --max-pipelines (0 = no limit) + tests: true # circle scan --tests + insights: true # circle scan --insights +``` + ### Common Settings Scan commands inherit from `common`: diff --git a/go.mod b/go.mod index 196382d5..245c1f39 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.26 require ( atomicgo.dev/keyboard v0.2.9 code.gitea.io/sdk/gitea v0.24.1 + github.com/CircleCI-Public/circleci-cli v0.1.34950 github.com/PuerkitoBio/goquery v1.12.0 github.com/bndr/gojenkins v1.2.0 github.com/docker/go-units v0.5.0 @@ -35,13 +36,13 @@ require ( cloud.google.com/go/compute/metadata v0.9.0 // indirect cloud.google.com/go/iam v1.5.3 // indirect cloud.google.com/go/secretmanager v1.16.0 // indirect - dario.cat/mergo v1.0.0 // indirect + dario.cat/mergo v1.0.2 // indirect filippo.io/edwards25519 v1.1.1 // indirect github.com/42wim/httpsig v1.2.4 // indirect github.com/Azure/go-ntlmssp v0.1.0 // indirect github.com/BobuSumisu/aho-corasick v1.0.3 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect - github.com/ProtonMail/go-crypto v1.1.6 // indirect + github.com/ProtonMail/go-crypto v1.3.0 // indirect github.com/TheZeroSlave/zapsentry v1.23.0 // indirect github.com/Unpackerr/iso9660 v0.0.3 // indirect github.com/andybalholm/brotli v1.2.0 // indirect @@ -120,9 +121,9 @@ require ( github.com/jlaffaye/ftp v0.2.0 // indirect github.com/joho/godotenv v1.5.1 // indirect github.com/josharian/intern v1.0.0 // indirect - github.com/kevinburke/ssh_config v1.2.0 // indirect + github.com/kevinburke/ssh_config v1.4.0 // indirect github.com/klauspost/compress v1.18.4 // indirect - github.com/klauspost/cpuid/v2 v2.2.9 // indirect + github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/lestrrat-go/blackmagic v1.0.4 // indirect github.com/lestrrat-go/httpcc v1.0.1 // indirect github.com/lestrrat-go/httprc/v3 v3.0.1 // indirect @@ -147,12 +148,12 @@ require ( github.com/pelletier/go-toml/v2 v2.2.4 // indirect github.com/peterebden/ar v0.0.0-20241106141004-20dc11b778e8 // indirect github.com/pierrec/lz4/v4 v4.1.26 // indirect - github.com/pjbgf/sha1cd v0.3.2 // indirect + github.com/pjbgf/sha1cd v0.5.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/prometheus/client_golang v1.20.5 // indirect + github.com/prometheus/client_golang v1.21.1 // indirect github.com/prometheus/client_model v0.6.2 // indirect - github.com/prometheus/common v0.55.0 // indirect + github.com/prometheus/common v0.62.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect github.com/rabbitmq/amqp091-go v1.10.0 // indirect github.com/robfig/cron/v3 v3.0.1 // indirect @@ -160,7 +161,7 @@ require ( github.com/sagikazarmark/locafero v0.11.0 // indirect github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect github.com/segmentio/asm v1.2.1 // indirect - github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect + github.com/sergi/go-diff v1.4.0 // indirect github.com/skeema/knownhosts v1.3.1 // indirect github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect github.com/spf13/afero v1.15.0 // indirect @@ -183,9 +184,9 @@ require ( go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.62.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect - go.opentelemetry.io/otel v1.39.0 // indirect - go.opentelemetry.io/otel/metric v1.39.0 // indirect - go.opentelemetry.io/otel/trace v1.39.0 // indirect + go.opentelemetry.io/otel v1.40.0 // indirect + go.opentelemetry.io/otel/metric v1.40.0 // indirect + go.opentelemetry.io/otel/trace v1.40.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect diff --git a/go.sum b/go.sum index ba04adea..ca6c236d 100644 --- a/go.sum +++ b/go.sum @@ -15,8 +15,8 @@ cloud.google.com/go/secretmanager v1.16.0 h1:19QT7ZsLJ8FSP1k+4esQvuCD7npMJml6hYz cloud.google.com/go/secretmanager v1.16.0/go.mod h1://C/e4I8D26SDTz1f3TQcddhcmiC3rMEl0S1Cakvs3Q= code.gitea.io/sdk/gitea v0.24.1 h1:hpaqcdGcBmfMpV7JSbBJVwE99qo+WqGreJYKrDKEyW8= code.gitea.io/sdk/gitea v0.24.1/go.mod h1:5/77BL3sHneCMEiZaMT9lfTvnnibsYxyO48mceCF3qA= -dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= -dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= +dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= +dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= filippo.io/edwards25519 v1.1.1 h1:YpjwWWlNmGIDyXOn8zLzqiD+9TyIlPhGFG96P39uBpw= filippo.io/edwards25519 v1.1.1/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= github.com/42wim/httpsig v1.2.4 h1:mI5bH0nm4xn7K18fo1K3okNDRq8CCJ0KbBYWyA6r8lU= @@ -40,6 +40,8 @@ github.com/AzureAD/microsoft-authentication-library-for-go v1.3.2/go.mod h1:wP83 github.com/BobuSumisu/aho-corasick v1.0.3 h1:uuf+JHwU9CHP2Vx+wAy6jcksJThhJS9ehR8a+4nPE9g= github.com/BobuSumisu/aho-corasick v1.0.3/go.mod h1:hm4jLcvZKI2vRF2WDU1N4p/jpWtpOzp3nLmi9AzX/XE= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/CircleCI-Public/circleci-cli v0.1.34950 h1:W3ZyGfyKL35PPaJcCShW5H6t+GGS0HzBjkOlKG6LmU0= +github.com/CircleCI-Public/circleci-cli v0.1.34950/go.mod h1:uUr25Nqz9pq9QoFYimDGsWpG5D8Q3+FDldKTiWdHRmA= github.com/MarvinJWendt/testza v0.1.0/go.mod h1:7AxNvlfeHP7Z/hDQ5JtE3OKYT3XFUeLCDE2DQninSqs= github.com/MarvinJWendt/testza v0.2.1/go.mod h1:God7bhG8n6uQxwdScay+gjm9/LnO4D3kkcZX4hv9Rp8= github.com/MarvinJWendt/testza v0.2.8/go.mod h1:nwIcjmr0Zz+Rcwfh3/4UhBp7ePKVhuBExvZqnKYWlII= @@ -51,8 +53,8 @@ github.com/MarvinJWendt/testza v0.4.2/go.mod h1:mSdhXiKH8sg/gQehJ63bINcCKp7RtYew github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= -github.com/ProtonMail/go-crypto v1.1.6 h1:ZcV+Ropw6Qn0AX9brlQLAUXfqLBc7Bl+f/DmNxpLfdw= -github.com/ProtonMail/go-crypto v1.1.6/go.mod h1:rA3QumHc/FZ8pAHreoekgiAbzpNsfQAosU5td4SnOrE= +github.com/ProtonMail/go-crypto v1.3.0 h1:ILq8+Sf5If5DCpHQp4PbZdS1J7HDFRXz/+xKBiRGFrw= +github.com/ProtonMail/go-crypto v1.3.0/go.mod h1:9whxjD8Rbs29b4XWbB8irEcE8KHMqaR2e7GWU1R+/PE= github.com/PuerkitoBio/goquery v1.12.0 h1:pAcL4g3WRXekcB9AU/y1mbKez2dbY2AajVhtkO8RIBo= github.com/PuerkitoBio/goquery v1.12.0/go.mod h1:802ej+gV2y7bbIhOIoPY5sT183ZW0YFofScC4q/hIpQ= github.com/TheZeroSlave/zapsentry v1.23.0 h1:TKyzfEL7LRlRr+7AvkukVLZ+jZPC++ebCUv7ZJHl1AU= @@ -324,8 +326,8 @@ github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= -github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= +github.com/kevinburke/ssh_config v1.4.0 h1:6xxtP5bZ2E4NF5tuQulISpTO2z8XbtH8cg1PWkxoFkQ= +github.com/kevinburke/ssh_config v1.4.0/go.mod h1:q2RIzfka+BXARoNexmF9gkxEX7DmvbW9P4hIVx2Kg4M= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.18.4 h1:RPhnKRAQ4Fh8zU2FY/6ZFDwTVTxgJ/EMydqSTzE9a2c= @@ -333,8 +335,8 @@ github.com/klauspost/compress v1.18.4/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxh github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.10/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= -github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY= -github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8= +github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= +github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -437,8 +439,8 @@ github.com/pierrec/lz4/v4 v4.1.26 h1:GrpZw1gZttORinvzBdXPUXATeqlJjqUG/D87TKMnhjY github.com/pierrec/lz4/v4 v4.1.26/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= -github.com/pjbgf/sha1cd v0.3.2 h1:a9wb0bp1oC2TGwStyn0Umc/IGKQnEgF0vVaZ8QF8eo4= -github.com/pjbgf/sha1cd v0.3.2/go.mod h1:zQWigSxVmsHEZow5qaLtPYxpcKMMQpa09ixqBxuCS6A= +github.com/pjbgf/sha1cd v0.5.0 h1:a+UkboSi1znleCDUNT3M5YxjOnN1fz2FhN48FlwCxs0= +github.com/pjbgf/sha1cd v0.5.0/go.mod h1:lhpGlyHLpQZoxMv8HcgXvZEhcGs0PG/vsZnEJ7H0iCM= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -451,13 +453,13 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= -github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= -github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= +github.com/prometheus/client_golang v1.21.1 h1:DOvXXTqVzvkIewV/CDPFdejpMCGeMcbGCQ8YOmu+Ibk= +github.com/prometheus/client_golang v1.21.1/go.mod h1:U9NM32ykUErtVBxdvD3zfi+EuFkkaBvMb09mIfe0Zgg= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= -github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= -github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= +github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= +github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/pterm/pterm v0.12.27/go.mod h1:PhQ89w4i95rhgE+xedAoqous6K9X+r6aSOI2eFF7DZI= @@ -490,8 +492,8 @@ github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj github.com/segmentio/asm v1.2.1 h1:DTNbBqs57ioxAD4PrArqftgypG4/qNpXoJx8TVXxPR0= github.com/segmentio/asm v1.2.1/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= -github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= -github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= +github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw= +github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/shirou/gopsutil/v3 v3.23.12 h1:z90NtUkp3bMtmICZKpC4+WaknU1eXtp5vtbQ11DgpE4= github.com/shirou/gopsutil/v3 v3.23.12/go.mod h1:1FrWgea594Jp7qmjHUUPlJDTPgcsb9mGnXDxavtikzM= github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= @@ -608,16 +610,16 @@ go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.6 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.62.0/go.mod h1:ru6KHrNtNHxM4nD/vd6QrLVWgKhxPYgblq4VAtNawTQ= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q= -go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48= -go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8= -go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF0Ls5Q0= -go.opentelemetry.io/otel/metric v1.39.0/go.mod h1:jrZSWL33sD7bBxg1xjrqyDjnuzTUB0x1nBERXd7Ftcs= -go.opentelemetry.io/otel/sdk v1.39.0 h1:nMLYcjVsvdui1B/4FRkwjzoRVsMK8uL/cj0OyhKzt18= -go.opentelemetry.io/otel/sdk v1.39.0/go.mod h1:vDojkC4/jsTJsE+kh+LXYQlbL8CgrEcwmt1ENZszdJE= +go.opentelemetry.io/otel v1.40.0 h1:oA5YeOcpRTXq6NN7frwmwFR0Cn3RhTVZvXsP4duvCms= +go.opentelemetry.io/otel v1.40.0/go.mod h1:IMb+uXZUKkMXdPddhwAHm6UfOwJyh4ct1ybIlV14J0g= +go.opentelemetry.io/otel/metric v1.40.0 h1:rcZe317KPftE2rstWIBitCdVp89A2HqjkxR3c11+p9g= +go.opentelemetry.io/otel/metric v1.40.0/go.mod h1:ib/crwQH7N3r5kfiBZQbwrTge743UDc7DTFVZrrXnqc= +go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8= +go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE= go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2WKg+sEJTtB8= go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew= -go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI= -go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA= +go.opentelemetry.io/otel/trace v1.40.0 h1:WA4etStDttCSYuhwvEa8OP8I5EWu24lkOzp+ZYblVjw= +go.opentelemetry.io/otel/trace v1.40.0/go.mod h1:zeAhriXecNGP/s2SEG3+Y8X9ujcJOTqQ5RgdEJcawiA= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -817,6 +819,8 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= +gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= resty.dev/v3 v3.0.0-beta.6 h1:ghRdNpoE8/wBCv+kTKIOauW1aCrSIeTq7GxtfYgtevU= diff --git a/internal/cmd/circle/circle.go b/internal/cmd/circle/circle.go new file mode 100644 index 00000000..289cb241 --- /dev/null +++ b/internal/cmd/circle/circle.go @@ -0,0 +1,18 @@ +package circle + +import ( + "github.com/CompassSecurity/pipeleek/internal/cmd/circle/scan" + "github.com/spf13/cobra" +) + +func NewCircleRootCmd() *cobra.Command { + circleCmd := &cobra.Command{ + Use: "circle [command]", + Short: "CircleCI related commands", + GroupID: "CircleCI", + } + + circleCmd.AddCommand(scan.NewScanCmd()) + + return circleCmd +} diff --git a/internal/cmd/circle/circle_test.go b/internal/cmd/circle/circle_test.go new file mode 100644 index 00000000..18e4695b --- /dev/null +++ b/internal/cmd/circle/circle_test.go @@ -0,0 +1,30 @@ +package circle + +import "testing" + +func TestNewCircleRootCmd(t *testing.T) { + cmd := NewCircleRootCmd() + if cmd == nil { + t.Fatal("Expected non-nil command") + } + + if cmd.Use != "circle [command]" { + t.Errorf("Expected Use to be 'circle [command]', got %q", cmd.Use) + } + + if cmd.Short == "" { + t.Error("Expected non-empty Short description") + } + + if cmd.GroupID != "CircleCI" { + t.Errorf("Expected GroupID 'CircleCI', got %q", cmd.GroupID) + } + + if len(cmd.Commands()) != 1 { + t.Errorf("Expected 1 subcommand, got %d", len(cmd.Commands())) + } + + if len(cmd.Commands()) == 1 && cmd.Commands()[0].Use != "scan" { + t.Errorf("Expected subcommand 'scan', got %q", cmd.Commands()[0].Use) + } +} diff --git a/internal/cmd/circle/scan/scan.go b/internal/cmd/circle/scan/scan.go new file mode 100644 index 00000000..d744650b --- /dev/null +++ b/internal/cmd/circle/scan/scan.go @@ -0,0 +1,169 @@ +package scan + +import ( + "time" + + "github.com/CompassSecurity/pipeleek/internal/cmd/flags" + circlescan "github.com/CompassSecurity/pipeleek/pkg/circle/scan" + "github.com/CompassSecurity/pipeleek/pkg/config" + "github.com/CompassSecurity/pipeleek/pkg/logging" + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" + "github.com/spf13/cobra" +) + +type CircleScanOptions struct { + config.CommonScanOptions + Token string + CircleURL string + Organization string + Projects []string + VCS string + Branch string + Statuses []string + Workflows []string + Jobs []string + Since string + Until string + MaxPipelines int + IncludeTests bool + Insights bool +} + +var options = CircleScanOptions{ + CommonScanOptions: config.DefaultCommonScanOptions(), +} + +var maxArtifactSize string + +func NewScanCmd() *cobra.Command { + scanCmd := &cobra.Command{ + Use: "scan", + Short: "Scan CircleCI logs and artifacts", + Long: `Scan CircleCI pipelines, workflows, jobs, logs, test results, and optional artifacts for secrets.`, + Example: ` +# Scan explicit project(s) +pipeleek circle scan --token --project org/repo + +# Restrict by branch and statuses +pipeleek circle scan --token --project org/repo --branch main --status success --status failed + +# Include artifacts and tests with time window +pipeleek circle scan --token --project org/repo --artifacts --since 2026-01-01T00:00:00Z --until 2026-01-31T23:59:59Z + `, + Run: Scan, + } + + flags.AddCommonScanFlagsNoOwned(scanCmd, &options.CommonScanOptions, &maxArtifactSize) + scanCmd.Flags().StringVarP(&options.Token, "token", "t", "", "CircleCI API token") + scanCmd.Flags().StringVarP(&options.CircleURL, "circle", "c", "https://circleci.com", "CircleCI base URL") + scanCmd.Flags().StringVarP(&options.Organization, "org", "", "", "CircleCI organization slug (used to filter projects)") + scanCmd.Flags().StringSliceVarP(&options.Projects, "project", "p", []string{}, "Project selector. Format: org/repo or vcs/org/repo") + scanCmd.Flags().StringVarP(&options.VCS, "vcs", "", "github", "VCS provider for project selectors without prefix (github or bitbucket)") + scanCmd.Flags().StringVarP(&options.Branch, "branch", "b", "", "Filter pipelines by branch") + scanCmd.Flags().StringSliceVarP(&options.Statuses, "status", "", []string{}, "Filter by pipeline/workflow/job status") + scanCmd.Flags().StringSliceVarP(&options.Workflows, "workflow", "", []string{}, "Filter by workflow name") + scanCmd.Flags().StringSliceVarP(&options.Jobs, "job", "", []string{}, "Filter by job name") + scanCmd.Flags().StringVarP(&options.Since, "since", "", "", "Include items created after this RFC3339 timestamp") + scanCmd.Flags().StringVarP(&options.Until, "until", "", "", "Include items created before this RFC3339 timestamp") + scanCmd.Flags().IntVarP(&options.MaxPipelines, "max-pipelines", "", 0, "Maximum number of pipelines to scan per project (0 = no limit)") + scanCmd.Flags().BoolVarP(&options.IncludeTests, "tests", "", true, "Scan CircleCI test results per job") + scanCmd.Flags().BoolVarP(&options.Insights, "insights", "", true, "Scan CircleCI workflow insights endpoints") + + return scanCmd +} + +func Scan(cmd *cobra.Command, args []string) { + if err := config.AutoBindFlags(cmd, map[string]string{ + "circle": "circle.url", + "token": "circle.token", + "org": "circle.scan.org", + "project": "circle.scan.project", + "vcs": "circle.scan.vcs", + "branch": "circle.scan.branch", + "status": "circle.scan.status", + "workflow": "circle.scan.workflow", + "job": "circle.scan.job", + "since": "circle.scan.since", + "until": "circle.scan.until", + "max-pipelines": "circle.scan.max_pipelines", + "tests": "circle.scan.tests", + "insights": "circle.scan.insights", + "threads": "common.threads", + "truffle-hog-verification": "common.trufflehog_verification", + "max-artifact-size": "common.max_artifact_size", + "confidence": "common.confidence_filter", + "hit-timeout": "common.hit_timeout", + }); err != nil { + log.Fatal().Err(err).Msg("Failed to bind command flags to configuration keys") + } + + if err := config.RequireConfigKeys("circle.token"); err != nil { + log.Fatal().Err(err).Msg("required configuration missing") + } + + options.Token = config.GetString("circle.token") + options.CircleURL = config.GetString("circle.url") + options.Organization = config.GetString("circle.scan.org") + options.Projects = config.GetStringSlice("circle.scan.project") + options.VCS = config.GetString("circle.scan.vcs") + options.Branch = config.GetString("circle.scan.branch") + options.Statuses = config.GetStringSlice("circle.scan.status") + options.Workflows = config.GetStringSlice("circle.scan.workflow") + options.Jobs = config.GetStringSlice("circle.scan.job") + options.Since = config.GetString("circle.scan.since") + options.Until = config.GetString("circle.scan.until") + options.MaxPipelines = config.GetInt("circle.scan.max_pipelines") + options.IncludeTests = config.GetBool("circle.scan.tests") + options.Insights = config.GetBool("circle.scan.insights") + options.MaxScanGoRoutines = config.GetInt("common.threads") + options.TruffleHogVerification = config.GetBool("common.trufflehog_verification") + options.ConfidenceFilter = config.GetStringSlice("common.confidence_filter") + maxArtifactSize = config.GetString("common.max_artifact_size") + if hitTimeoutSeconds := config.GetInt("common.hit_timeout"); hitTimeoutSeconds > 0 { + options.HitTimeout = time.Duration(hitTimeoutSeconds) * time.Second + } + + if err := config.ValidateURL(options.CircleURL, "CircleCI URL"); err != nil { + log.Fatal().Err(err).Msg("Invalid CircleCI URL") + } + if err := config.ValidateToken(options.Token, "CircleCI API token"); err != nil { + log.Fatal().Err(err).Msg("Invalid CircleCI API token") + } + if err := config.ValidateThreadCount(options.MaxScanGoRoutines); err != nil { + log.Fatal().Err(err).Msg("Invalid thread count") + } + + scanOpts, err := circlescan.InitializeOptions(circlescan.InitializeOptionsInput{ + Token: options.Token, + CircleURL: options.CircleURL, + Organization: options.Organization, + Projects: options.Projects, + VCS: options.VCS, + Branch: options.Branch, + Statuses: options.Statuses, + WorkflowNames: options.Workflows, + JobNames: options.Jobs, + Since: options.Since, + Until: options.Until, + MaxPipelines: options.MaxPipelines, + IncludeTests: options.IncludeTests, + IncludeInsights: options.Insights, + Artifacts: options.Artifacts, + MaxArtifactSize: maxArtifactSize, + ConfidenceFilter: options.ConfidenceFilter, + MaxScanGoRoutines: options.MaxScanGoRoutines, + TruffleHogVerification: options.TruffleHogVerification, + HitTimeout: options.HitTimeout, + }) + if err != nil { + log.Fatal().Err(err).Msg("Failed initializing CircleCI scan options") + } + + scanner := circlescan.NewScanner(scanOpts) + logging.RegisterStatusHook(func() *zerolog.Event { return scanner.Status() }) + + if err := scanner.Scan(); err != nil { + log.Fatal().Err(err).Msg("Scan failed") + } +} diff --git a/internal/cmd/circle/scan/scan_test.go b/internal/cmd/circle/scan/scan_test.go new file mode 100644 index 00000000..01cd97fc --- /dev/null +++ b/internal/cmd/circle/scan/scan_test.go @@ -0,0 +1,41 @@ +package scan + +import "testing" + +func TestNewScanCmd(t *testing.T) { + cmd := NewScanCmd() + if cmd == nil { + t.Fatal("expected non-nil command") + } + + if cmd.Use != "scan" { + t.Fatalf("expected use 'scan', got %q", cmd.Use) + } + + flags := cmd.Flags() + for _, name := range []string{ + "token", + "circle", + "org", + "project", + "vcs", + "branch", + "status", + "workflow", + "job", + "since", + "until", + "max-pipelines", + "tests", + "insights", + "threads", + "truffle-hog-verification", + "confidence", + "artifacts", + "max-artifact-size", + } { + if flags.Lookup(name) == nil { + t.Errorf("expected flag %q to exist", name) + } + } +} diff --git a/internal/cmd/root.go b/internal/cmd/root.go index c6bdfcdd..fb8a06c2 100644 --- a/internal/cmd/root.go +++ b/internal/cmd/root.go @@ -8,6 +8,7 @@ import ( "time" "github.com/CompassSecurity/pipeleek/internal/cmd/bitbucket" + "github.com/CompassSecurity/pipeleek/internal/cmd/circle" "github.com/CompassSecurity/pipeleek/internal/cmd/devops" "github.com/CompassSecurity/pipeleek/internal/cmd/docs" "github.com/CompassSecurity/pipeleek/internal/cmd/gitea" @@ -79,6 +80,7 @@ func init() { rootCmd.AddCommand(devops.NewAzureDevOpsRootCmd()) rootCmd.AddCommand(gitea.NewGiteaRootCmd()) rootCmd.AddCommand(jenkins.NewJenkinsRootCmd()) + rootCmd.AddCommand(circle.NewCircleRootCmd()) rootCmd.AddCommand(docs.NewDocsCmd(rootCmd)) rootCmd.PersistentFlags().StringVar(&ConfigFile, "config", "", "Config file path. Example: ~/.config/pipeleek/pipeleek.yaml") rootCmd.PersistentFlags().BoolVarP(&JsonLogoutput, "json", "", false, "Use JSON as log output format") @@ -99,6 +101,7 @@ func init() { rootCmd.AddGroup(&cobra.Group{ID: "AzureDevOps", Title: "Azure DevOps Commands"}) rootCmd.AddGroup(&cobra.Group{ID: "Gitea", Title: "Gitea Commands"}) rootCmd.AddGroup(&cobra.Group{ID: "Jenkins", Title: "Jenkins Commands"}) + rootCmd.AddGroup(&cobra.Group{ID: "CircleCI", Title: "CircleCI Commands"}) } type CustomWriter struct { diff --git a/pipeleek.example.yaml b/pipeleek.example.yaml index d663de5a..c42ab8cf 100644 --- a/pipeleek.example.yaml +++ b/pipeleek.example.yaml @@ -200,3 +200,27 @@ jenkins: job: "team-a/service-a" # Optional: scan a single job path max_builds: 25 # Maximum builds to scan per job (0 = all) # Inherits common.* settings + +#------------------------------------------------------------------------------ +# CircleCI Platform Configuration +#------------------------------------------------------------------------------ +circle: + url: https://circleci.com + token: circleci_token_REPLACE_ME + + # scan - Scan CircleCI pipelines, logs, test results and optional artifacts + scan: + project: ["example-org/example-repo"] # Optional project selector(s): org/repo or vcs/org/repo + vcs: "github" # Default VCS used when project entries omit prefix + org: "example-org" # Optional org filter; supports my-org, github/my-org, or app.circleci.com/pipelines URLs + # Org-wide discovery requires token visibility to that org. If discovery fails, use explicit --project entries. + branch: "main" # Optional branch filter + status: ["success", "failed"] # Optional pipeline/workflow/job status filter + workflow: ["build", "deploy"] # Optional workflow name filter + job: ["unit-tests", "release"] # Optional job name filter + since: "2026-01-01T00:00:00Z" # Optional RFC3339 start timestamp + until: "2026-01-31T23:59:59Z" # Optional RFC3339 end timestamp + max_pipelines: 0 # Maximum number of pipelines to scan per project (0 = no limit) + tests: true # Scan job test results + insights: true # Scan workflow insights endpoints + # Inherits common.* settings diff --git a/pkg/circle/scan/normalize.go b/pkg/circle/scan/normalize.go new file mode 100644 index 00000000..0aea75cf --- /dev/null +++ b/pkg/circle/scan/normalize.go @@ -0,0 +1,231 @@ +package scan + +import ( + "fmt" + "net/url" + "strings" +) + +func normalizeProjectSlug(value, defaultVCS string) (string, error) { + parts := strings.Split(strings.Trim(value, " /"), "/") + if len(parts) == 2 { + return fmt.Sprintf("%s/%s/%s", defaultVCS, parts[0], parts[1]), nil + } + if len(parts) == 3 { + return strings.Join(parts, "/"), nil + } + return "", fmt.Errorf("invalid project selector %q (expected org/repo or vcs/org/repo)", value) +} + +func belongsToOrg(projectSlug, org string) bool { + parts := strings.Split(projectSlug, "/") + return len(parts) >= 3 && strings.EqualFold(parts[1], org) +} + +func normalizedOrgName(org string) string { + trimmed := strings.Trim(strings.TrimSpace(org), "/") + if trimmed == "" { + return "" + } + parts := strings.Split(trimmed, "/") + if len(parts) >= 2 { + return parts[len(parts)-1] + } + return trimmed +} + +func toFilterSet(values []string) map[string]struct{} { + out := make(map[string]struct{}, len(values)) + for _, value := range values { + trimmed := strings.TrimSpace(strings.ToLower(value)) + if trimmed == "" { + continue + } + out[trimmed] = struct{}{} + } + return out +} + +func matchesFilter(filter map[string]struct{}, value string) bool { + if len(filter) == 0 { + return true + } + _, ok := filter[strings.ToLower(strings.TrimSpace(value))] + return ok +} + +func vcsFromURL(raw string) string { + value := strings.ToLower(raw) + switch { + case strings.Contains(value, "bitbucket"): + return "bitbucket" + case strings.Contains(value, "github"): + return "github" + default: + return "" + } +} + +func normalizeVCSName(vcs string) string { + switch strings.ToLower(strings.TrimSpace(vcs)) { + case "github", "gh": + return "github" + case "bitbucket", "bb": + return "bitbucket" + case "circleci": + return "circleci" + default: + return strings.ToLower(strings.TrimSpace(vcs)) + } +} + +func projectSlugFromV1(item v1ProjectItem, defaultVCS string) (string, bool) { + vcs := normalizeVCSName(item.VCSType) + if vcs == "circleci" { + if slug, ok := circleciUUIDSlug(item.VCSURL); ok { + return slug, true + } + } + + org := strings.TrimSpace(item.Username) + repo := strings.TrimSpace(item.Reponame) + if org == "" || repo == "" { + return "", false + } + + if vcs == "" { + vcs = normalizeVCSName(vcsFromURL(item.VCSURL)) + } + if vcs == "" { + vcs = normalizeVCSName(defaultVCS) + } + if vcs == "" { + vcs = "github" + } + + normalized, err := normalizeProjectSlug(fmt.Sprintf("%s/%s/%s", vcs, org, repo), defaultVCS) + if err != nil { + return "", false + } + + return normalized, true +} + +func circleciUUIDSlug(raw string) (string, bool) { + trimmed := strings.TrimSpace(raw) + if trimmed == "" { + return "", false + } + trimmed = strings.TrimPrefix(trimmed, "https://") + trimmed = strings.TrimPrefix(trimmed, "http://") + trimmed = strings.TrimPrefix(trimmed, "//") + trimmed = strings.TrimPrefix(trimmed, "circleci.com/") + trimmed = strings.Trim(trimmed, "/") + + parts := strings.Split(trimmed, "/") + if len(parts) < 2 { + return "", false + } + + orgID := strings.TrimSpace(parts[0]) + projectID := strings.TrimSpace(parts[1]) + if orgID == "" || projectID == "" { + return "", false + } + + return fmt.Sprintf("circleci/%s/%s", orgID, projectID), true +} + +func vcsSlugCandidates(vcs string) []string { + v := strings.ToLower(strings.TrimSpace(vcs)) + switch v { + case "gh", "github": + return []string{"github", "gh"} + case "bb", "bitbucket": + return []string{"bitbucket", "bb"} + case "gitlab", "gl": + return []string{"gitlab", "gl"} + case "": + return []string{"github", "gh", "bitbucket", "bb"} + default: + return []string{v} + } +} + +func uniqueStrings(values []string) []string { + out := make([]string, 0, len(values)) + seen := make(map[string]struct{}, len(values)) + for _, value := range values { + key := strings.TrimSpace(value) + if key == "" { + continue + } + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + out = append(out, key) + } + return out +} + +func orgSlugCandidates(value, defaultVCS string) []string { + trimmed := strings.TrimSpace(value) + if trimmed == "" { + return nil + } + + candidates := []string{trimmed} + + // Support org/project URLs like: + // https://app.circleci.com/pipelines/github/storybookjs/storybook + if parsed, err := url.Parse(trimmed); err == nil && parsed.Host != "" { + parts := strings.Split(strings.Trim(parsed.Path, "/"), "/") + if len(parts) >= 3 && parts[0] == "pipelines" { + vcs := normalizeVCSName(parts[1]) + org := strings.TrimSpace(parts[2]) + if vcs != "" && org != "" { + candidates = append(candidates, fmt.Sprintf("%s/%s", vcs, org), org) + } + } + } + + orgName := normalizedOrgName(trimmed) + if orgName != "" && !strings.EqualFold(orgName, trimmed) { + candidates = append(candidates, orgName) + } + + if !strings.Contains(orgName, "/") && orgName != "" { + for _, vcsSlug := range vcsSlugCandidates(defaultVCS) { + candidates = append(candidates, fmt.Sprintf("%s/%s", vcsSlug, orgName)) + } + } + + return uniqueStrings(candidates) +} + +func orgDiscoveryHint(value string) string { + trimmed := strings.TrimSpace(value) + if trimmed == "" { + return "" + } + + if parsed, err := url.Parse(trimmed); err == nil && parsed.Host != "" { + parts := strings.Split(strings.Trim(parsed.Path, "/"), "/") + if len(parts) >= 4 && parts[0] == "pipelines" { + vcs := normalizeVCSName(parts[1]) + org := strings.TrimSpace(parts[2]) + repo := strings.TrimSpace(parts[3]) + if vcs != "" && org != "" && repo != "" { + return fmt.Sprintf("--org appears to be a project URL; use --project %s/%s/%s instead", vcs, org, repo) + } + } + } + + parts := strings.Split(strings.Trim(trimmed, "/"), "/") + if len(parts) == 3 && normalizeVCSName(parts[0]) != "" { + return fmt.Sprintf("--org appears to be a project selector; use --project %s instead", strings.Join(parts, "/")) + } + + return "org-wide discovery requires token visibility to that CircleCI org; if discovery fails, scan explicit projects with --project" +} diff --git a/pkg/circle/scan/scanner.go b/pkg/circle/scan/scanner.go new file mode 100644 index 00000000..95e660c7 --- /dev/null +++ b/pkg/circle/scan/scanner.go @@ -0,0 +1,717 @@ +package scan + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/CompassSecurity/pipeleek/pkg/format" + "github.com/CompassSecurity/pipeleek/pkg/logging" + "github.com/CompassSecurity/pipeleek/pkg/scan/logline" + "github.com/CompassSecurity/pipeleek/pkg/scan/result" + "github.com/CompassSecurity/pipeleek/pkg/scan/runner" + pkgscanner "github.com/CompassSecurity/pipeleek/pkg/scanner" + "github.com/h2non/filetype" + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" +) + +type InitializeOptionsInput struct { + Token string + CircleURL string + Organization string + Projects []string + VCS string + Branch string + Statuses []string + WorkflowNames []string + JobNames []string + Since string + Until string + MaxPipelines int + IncludeTests bool + IncludeInsights bool + Artifacts bool + MaxArtifactSize string + ConfidenceFilter []string + MaxScanGoRoutines int + TruffleHogVerification bool + HitTimeout time.Duration +} + +type ScanOptions struct { + Token string + CircleURL string + Organization string + Projects []string + Branch string + Statuses map[string]struct{} + WorkflowNames map[string]struct{} + JobNames map[string]struct{} + Since *time.Time + Until *time.Time + MaxPipelines int + IncludeTests bool + IncludeInsights bool + Artifacts bool + MaxArtifactSize int64 + ConfidenceFilter []string + MaxScanGoRoutines int + TruffleHogVerification bool + HitTimeout time.Duration + Context context.Context + APIClient CircleClient + HTTPClient *http.Client +} + +type Scanner interface { + pkgscanner.BaseScanner + Status() *zerolog.Event +} + +type circleScanner struct { + options ScanOptions + + pipelinesScanned atomic.Int64 + jobsScanned atomic.Int64 + artifactsScanned atomic.Int64 + currentProject string + mu sync.RWMutex +} + +var _ pkgscanner.BaseScanner = (*circleScanner)(nil) + +func NewScanner(opts ScanOptions) Scanner { + return &circleScanner{options: opts} +} + +func (s *circleScanner) Status() *zerolog.Event { + s.mu.RLock() + project := s.currentProject + s.mu.RUnlock() + + return log.Info(). + Int64("pipelinesScanned", s.pipelinesScanned.Load()). + Int64("jobsScanned", s.jobsScanned.Load()). + Int64("artifactsScanned", s.artifactsScanned.Load()). + Str("currentProject", project) +} + +func (s *circleScanner) Scan() error { + runner.InitScanner(s.options.ConfidenceFilter) + + for _, project := range s.options.Projects { + s.mu.Lock() + s.currentProject = project + s.mu.Unlock() + + log.Info().Str("project", project).Msg("Scanning CircleCI project") + if err := s.scanProject(project); err != nil { + log.Warn().Err(err).Str("project", project).Msg("Project scan failed, continuing") + } + } + + log.Info().Msg("Scan Finished, Bye Bye 🏳️‍🌈🔥") + return nil +} + +func (s *circleScanner) scanProject(project string) error { + var pageToken string + scanned := 0 + + for { + log.Debug(). + Str("project", project). + Str("pageToken", pageToken). + Int("pipelinesScannedForProject", scanned). + Msg("Fetching pipeline page") + + pipelines, nextToken, err := s.options.APIClient.ListPipelines(s.options.Context, project, s.options.Branch, pageToken) + if err != nil { + return err + } + + log.Debug(). + Str("project", project). + Int("pipelinesReturned", len(pipelines)). + Str("nextPageToken", nextToken). + Msg("Fetched pipeline page") + + for _, pipeline := range pipelines { + if s.options.MaxPipelines > 0 && scanned >= s.options.MaxPipelines { + log.Debug(). + Str("project", project). + Int("maxPipelines", s.options.MaxPipelines). + Int("pipelinesScannedForProject", scanned). + Msg("Reached max pipeline limit for project") + return nil + } + + if !s.inTimeWindow(parseRFC3339Ptr(pipeline.CreatedAt)) { + continue + } + + if !matchesFilter(s.options.Statuses, pipeline.State) { + continue + } + + s.pipelinesScanned.Add(1) + scanned++ + + log.Debug(). + Str("project", project). + Str("pipelineID", pipeline.ID). + Str("pipelineState", pipeline.State). + Msg("Scanning pipeline") + + if err := s.scanPipeline(project, pipeline); err != nil { + log.Warn().Err(err).Str("project", project).Str("pipeline", pipeline.ID).Msg("Pipeline scan failed, continuing") + } + } + + if nextToken == "" { + break + } + pageToken = nextToken + } + + if s.options.IncludeInsights { + if err := s.scanProjectInsights(project); err != nil { + log.Debug().Err(err).Str("project", project).Msg("Failed scanning project insights") + } + } + + return nil +} + +func (s *circleScanner) scanProjectInsights(project string) error { + log.Debug().Str("project", project).Msg("Scanning project insights") + + workflows, err := s.options.APIClient.ListProjectInsightsWorkflows(s.options.Context, project, s.options.Branch) + if err != nil { + return err + } + + log.Debug(). + Str("project", project). + Int("insightWorkflows", len(workflows)). + Msg("Fetched project insights workflows") + + for _, workflowName := range workflows { + if !matchesFilter(s.options.WorkflowNames, workflowName) { + continue + } + + details, err := s.options.APIClient.GetProjectInsightsWorkflow(s.options.Context, project, workflowName, s.options.Branch) + if err != nil { + continue + } + + payload, err := json.Marshal(details) + if err != nil { + continue + } + + findings, err := pkgscanner.DetectHits(payload, s.options.MaxScanGoRoutines, s.options.TruffleHogVerification, s.options.HitTimeout) + if err != nil { + continue + } + + result.ReportFindings(findings, result.ReportOptions{ + LocationURL: strings.TrimRight(s.options.CircleURL, "/") + "/pipelines/" + project, + JobName: workflowName, + BuildName: "insights", + Type: logging.SecretTypeLog, + }) + } + + return nil +} + +func (s *circleScanner) scanPipeline(project string, pipeline pipelineItem) error { + workflows, err := s.options.APIClient.ListPipelineWorkflows(s.options.Context, pipeline.ID) + if err != nil { + return err + } + + log.Debug(). + Str("project", project). + Str("pipelineID", pipeline.ID). + Int("workflowsReturned", len(workflows)). + Msg("Fetched pipeline workflows") + + for _, wf := range workflows { + if !matchesFilter(s.options.WorkflowNames, wf.Name) { + continue + } + if !matchesFilter(s.options.Statuses, wf.Status) { + continue + } + if !s.inTimeWindow(parseRFC3339Ptr(wf.CreatedAt)) { + continue + } + + log.Debug(). + Str("project", project). + Str("pipelineID", pipeline.ID). + Str("workflowID", wf.ID). + Str("workflowName", wf.Name). + Str("workflowStatus", wf.Status). + Msg("Scanning workflow") + + if err := s.scanWorkflow(project, pipeline, wf); err != nil { + log.Warn().Err(err).Str("project", project).Str("workflow", wf.ID).Msg("Workflow scan failed, continuing") + } + } + + return nil +} + +func (s *circleScanner) scanWorkflow(project string, pipeline pipelineItem, workflow workflowItem) error { + jobs, err := s.options.APIClient.ListWorkflowJobs(s.options.Context, workflow.ID) + if err != nil { + return err + } + + log.Debug(). + Str("project", project). + Str("pipelineID", pipeline.ID). + Str("workflowID", workflow.ID). + Int("jobsReturned", len(jobs)). + Msg("Fetched workflow jobs") + + for _, job := range jobs { + if !matchesFilter(s.options.JobNames, job.Name) { + continue + } + if !matchesFilter(s.options.Statuses, job.Status) { + continue + } + + log.Debug(). + Str("project", project). + Str("workflowID", workflow.ID). + Int("jobNumber", job.JobNumber). + Str("jobName", job.Name). + Str("jobStatus", job.Status). + Msg("Scanning job") + + s.jobsScanned.Add(1) + if err := s.scanJob(project, pipeline, workflow, job); err != nil { + log.Warn().Err(err).Str("project", project).Int("jobNumber", job.JobNumber).Msg("Job scan failed, continuing") + } + } + + return nil +} + +func (s *circleScanner) scanJob(project string, pipeline pipelineItem, workflow workflowItem, job workflowJobItem) error { + jobDetails, err := s.options.APIClient.GetProjectJob(s.options.Context, project, job.JobNumber) + if err != nil { + return err + } + if len(jobDetails.Steps) == 0 { + legacyDetails, legacyErr := s.options.APIClient.GetProjectJobV1(s.options.Context, project, job.JobNumber) + if legacyErr == nil && len(legacyDetails.Steps) > 0 { + if strings.TrimSpace(jobDetails.Name) == "" { + jobDetails.Name = legacyDetails.Name + } + if strings.TrimSpace(jobDetails.WebURL) == "" { + jobDetails.WebURL = legacyDetails.WebURL + } + jobDetails.Steps = legacyDetails.Steps + } + } + + log.Debug(). + Str("project", project). + Int("pipelineNumber", pipeline.Number). + Str("workflowID", workflow.ID). + Int("jobNumber", job.JobNumber). + Int("steps", len(jobDetails.Steps)). + Msg("Fetched job details") + + locationURL := circleAppWorkflowURL(workflow.ID) + jobURL := circleAppJobURL(project, pipeline.Number, workflow.ID, job.JobNumber, locationURL) + + if err := s.scanJobLogs(project, workflow, jobURL, jobDetails); err != nil { + log.Debug().Err(err).Str("project", project).Int("job", job.JobNumber).Msg("Failed scanning job logs") + } + + if s.options.IncludeTests { + if err := s.scanJobTests(project, workflow, job, jobDetails, jobURL); err != nil { + log.Debug().Err(err).Str("project", project).Int("job", job.JobNumber).Msg("Failed scanning job tests") + } + } + + if s.options.Artifacts { + if err := s.scanJobArtifacts(project, workflow, job, jobDetails, jobURL); err != nil { + log.Debug().Err(err).Str("project", project).Int("job", job.JobNumber).Msg("Failed scanning job artifacts") + } + } + + return nil +} + +func (s *circleScanner) scanJobLogs(project string, workflow workflowItem, jobURL string, details projectJobResponse) error { + log.Debug(). + Str("project", project). + Str("workflowID", workflow.ID). + Str("jobName", details.Name). + Int("steps", len(details.Steps)). + Msg("Scanning job logs") + + for _, step := range details.Steps { + for _, action := range step.Actions { + if action.OutputURL == "" { + continue + } + + logBytes, err := s.options.APIClient.DownloadWithAuth(s.options.Context, action.OutputURL) + if err != nil { + continue + } + if len(logBytes) == 0 { + continue + } + + processed := flattenLogOutput(logBytes) + logResult, err := logline.ProcessLogs(processed, logline.ProcessOptions{ + MaxGoRoutines: s.options.MaxScanGoRoutines, + VerifyCredentials: s.options.TruffleHogVerification, + HitTimeout: s.options.HitTimeout, + }) + if err != nil { + continue + } + + if len(logResult.Findings) > 0 { + log.Debug(). + Str("project", project). + Str("workflowID", workflow.ID). + Str("jobName", details.Name). + Str("stepName", step.Name). + Int("findings", len(logResult.Findings)). + Msg("Detected findings in job log output") + } + + stepLabel := step.Name + if action.Name != "" && action.Name != step.Name { + stepLabel = step.Name + " / " + action.Name + } + result.ReportFindings(logResult.Findings, result.ReportOptions{ + LocationURL: jobURL, + JobName: workflow.Name, + BuildName: details.Name + " / " + stepLabel, + Type: logging.SecretTypeLog, + }) + } + } + + return nil +} + +func (s *circleScanner) scanJobTests(project string, workflow workflowItem, job workflowJobItem, details projectJobResponse, locationURL string) error { + tests, err := s.options.APIClient.ListJobTests(s.options.Context, project, job.JobNumber) + if err != nil { + return err + } + + log.Debug(). + Str("project", project). + Str("workflowID", workflow.ID). + Int("jobNumber", job.JobNumber). + Int("testsReturned", len(tests)). + Msg("Fetched job tests") + + if len(tests) == 0 { + return nil + } + + payload, err := json.Marshal(tests) + if err != nil { + return err + } + + findings, err := pkgscanner.DetectHits(payload, s.options.MaxScanGoRoutines, s.options.TruffleHogVerification, s.options.HitTimeout) + if err != nil { + return err + } + + if len(findings) > 0 { + log.Debug(). + Str("project", project). + Str("workflowID", workflow.ID). + Int("jobNumber", job.JobNumber). + Int("findings", len(findings)). + Msg("Detected findings in job tests") + } + + result.ReportFindings(findings, result.ReportOptions{ + LocationURL: locationURL, + JobName: workflow.Name, + BuildName: fmt.Sprintf("%s tests", details.Name), + Type: logging.SecretTypeLog, + }) + + return nil +} + +func (s *circleScanner) scanJobArtifacts(project string, workflow workflowItem, job workflowJobItem, details projectJobResponse, locationURL string) error { + artifacts, err := s.options.APIClient.ListJobArtifacts(s.options.Context, project, job.JobNumber) + if err != nil { + return err + } + + log.Debug(). + Str("project", project). + Str("workflowID", workflow.ID). + Int("jobNumber", job.JobNumber). + Int("artifactsReturned", len(artifacts)). + Msg("Fetched job artifacts") + + for _, artifact := range artifacts { + if artifact.URL == "" || artifact.Path == "" { + continue + } + + content, err := s.options.APIClient.DownloadWithAuth(s.options.Context, artifact.URL) + if err != nil { + continue + } + + if int64(len(content)) > s.options.MaxArtifactSize { + log.Debug(). + Str("project", project). + Str("workflowID", workflow.ID). + Int("jobNumber", job.JobNumber). + Str("artifact", artifact.Path). + Int("bytes", len(content)). + Int64("maxBytes", s.options.MaxArtifactSize). + Msg("Skipped large artifact") + continue + } + + s.artifactsScanned.Add(1) + if filetype.IsArchive(content) { + pkgscanner.HandleArchiveArtifact(artifact.Path, content, locationURL, details.Name, s.options.TruffleHogVerification, s.options.HitTimeout) + continue + } + + pkgscanner.DetectFileHits(content, locationURL, details.Name, artifact.Path, workflow.Name, s.options.TruffleHogVerification, s.options.HitTimeout) + } + + return nil +} + +func (s *circleScanner) inTimeWindow(value *time.Time) bool { + if value == nil { + return true + } + if s.options.Since != nil && value.Before(*s.options.Since) { + return false + } + if s.options.Until != nil && value.After(*s.options.Until) { + return false + } + return true +} + +func InitializeOptions(input InitializeOptionsInput) (ScanOptions, error) { + orgName := normalizedOrgName(input.Organization) + + if input.CircleURL == "" { + input.CircleURL = "https://circleci.com" + } + if input.VCS == "" { + input.VCS = "github" + } + + maxArtifactBytes, err := format.ParseHumanSize(input.MaxArtifactSize) + if err != nil { + return ScanOptions{}, err + } + + since, err := parseOptionalRFC3339(input.Since) + if err != nil { + return ScanOptions{}, fmt.Errorf("invalid --since value: %w", err) + } + until, err := parseOptionalRFC3339(input.Until) + if err != nil { + return ScanOptions{}, fmt.Errorf("invalid --until value: %w", err) + } + if since != nil && until != nil && since.After(*until) { + return ScanOptions{}, fmt.Errorf("--since must be before --until") + } + + projects := make([]string, 0, len(input.Projects)) + for _, p := range input.Projects { + normalized, err := normalizeProjectSlug(p, input.VCS) + if err != nil { + return ScanOptions{}, err + } + if orgName != "" && !belongsToOrg(normalized, orgName) { + continue + } + projects = append(projects, normalized) + } + + baseURL, err := url.Parse(strings.TrimRight(input.CircleURL, "/") + "/api/v2/") + if err != nil { + return ScanOptions{}, err + } + + httpClient := &http.Client{Timeout: 45 * time.Second} + apiClient := newCircleAPIClient(baseURL, input.Token, httpClient) + + if len(projects) == 0 { + if strings.TrimSpace(input.Organization) != "" { + resolved, err := apiClient.ListOrganizationProjects(context.Background(), input.Organization, input.VCS) + if err != nil { + // v1 fallback only makes sense for GitHub/Bitbucket orgs whose username + // matches the GitHub/Bitbucket username in v1 project records. For native + // circleci/ orgs, the orgName is a UUID-like slug that will never match a + // VCS username, so skip the v1 fallback and surface the original error. + v1Filter := orgName + if strings.HasPrefix(strings.ToLower(input.Organization), "circleci/") { + v1Filter = "" + } + fallbackProjects, fallbackErr := apiClient.ListAccessibleProjectsV1(context.Background(), input.VCS, v1Filter) + if fallbackErr != nil { + hint := orgDiscoveryHint(input.Organization) + if hint != "" { + return ScanOptions{}, fmt.Errorf("ListOrganizationProjects failed: %v; fallback ListAccessibleProjectsV1 failed: %w. Hint: %s", err, fallbackErr, hint) + } + return ScanOptions{}, fmt.Errorf("ListOrganizationProjects failed: %v; fallback ListAccessibleProjectsV1 failed: %w", err, fallbackErr) + } + projects = uniqueStrings(append(projects, fallbackProjects...)) + } else { + projects = resolved + } + } else { + resolved, err := apiClient.ListAccessibleProjectsV1(context.Background(), input.VCS, "") + if err != nil { + return ScanOptions{}, fmt.Errorf("provide --project or --org, or ensure token can list accessible projects: %w", err) + } + projects = resolved + } + } + + if len(projects) == 0 { + return ScanOptions{}, fmt.Errorf("no project remains after applying organization filter") + } + + return ScanOptions{ + Token: input.Token, + CircleURL: input.CircleURL, + Organization: input.Organization, + Projects: projects, + Branch: input.Branch, + Statuses: toFilterSet(input.Statuses), + WorkflowNames: toFilterSet(input.WorkflowNames), + JobNames: toFilterSet(input.JobNames), + Since: since, + Until: until, + MaxPipelines: input.MaxPipelines, + IncludeTests: input.IncludeTests, + IncludeInsights: input.IncludeInsights, + Artifacts: input.Artifacts, + MaxArtifactSize: maxArtifactBytes, + ConfidenceFilter: input.ConfidenceFilter, + MaxScanGoRoutines: input.MaxScanGoRoutines, + TruffleHogVerification: input.TruffleHogVerification, + HitTimeout: input.HitTimeout, + Context: context.Background(), + APIClient: apiClient, + HTTPClient: httpClient, + }, nil +} + +func parseOptionalRFC3339(value string) (*time.Time, error) { + if strings.TrimSpace(value) == "" { + return nil, nil + } + t, err := time.Parse(time.RFC3339, value) + if err != nil { + return nil, err + } + return &t, nil +} + +func parseRFC3339Ptr(value string) *time.Time { + if value == "" { + return nil + } + t, err := time.Parse(time.RFC3339, value) + if err != nil { + return nil + } + return &t +} + +func flattenLogOutput(raw []byte) []byte { + trimmed := strings.TrimSpace(string(raw)) + if trimmed == "" { + return raw + } + + if strings.HasPrefix(trimmed, "[") { + var entries []map[string]interface{} + if err := json.Unmarshal([]byte(trimmed), &entries); err == nil && len(entries) > 0 { + var b strings.Builder + for _, entry := range entries { + if msg, ok := entry["message"].(string); ok && msg != "" { + b.WriteString(msg) + b.WriteByte('\n') + } + } + if b.Len() > 0 { + return []byte(b.String()) + } + } + } + + if strings.HasPrefix(trimmed, "{") { + var entry map[string]interface{} + if err := json.Unmarshal([]byte(trimmed), &entry); err == nil { + if msg, ok := entry["message"].(string); ok && msg != "" { + return []byte(msg) + } + } + } + + return []byte(trimmed) +} + +func circleAppWorkflowURL(workflowID string) string { + if strings.TrimSpace(workflowID) == "" { + return "https://app.circleci.com/pipelines" + } + return fmt.Sprintf("https://app.circleci.com/pipelines/workflows/%s", workflowID) +} + +// circleAppJobURL builds the stable CircleCI app job URL. +// Format: https://app.circleci.com/pipelines/////workflows//jobs/ +func circleAppJobURL(project string, pipelineNumber int, workflowID string, jobNum int, fallback string) string { + parts := strings.Split(project, "/") + if len(parts) != 3 || strings.TrimSpace(workflowID) == "" || pipelineNumber <= 0 || jobNum <= 0 { + return fallback + } + + vcs := normalizeVCSName(parts[0]) + return fmt.Sprintf( + "https://app.circleci.com/pipelines/%s/%s/%s/%d/workflows/%s/jobs/%d", + vcs, + parts[1], + parts[2], + pipelineNumber, + workflowID, + jobNum, + ) +} diff --git a/pkg/circle/scan/scanner_test.go b/pkg/circle/scan/scanner_test.go new file mode 100644 index 00000000..359bab0d --- /dev/null +++ b/pkg/circle/scan/scanner_test.go @@ -0,0 +1,319 @@ +package scan + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestNormalizeProjectSlug(t *testing.T) { + tests := []struct { + name string + in string + vcs string + want string + wantError bool + }{ + {name: "org/repo", in: "org/repo", vcs: "github", want: "github/org/repo"}, + {name: "vcs/org/repo", in: "bitbucket/org/repo", vcs: "github", want: "bitbucket/org/repo"}, + {name: "invalid", in: "org", vcs: "github", wantError: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := normalizeProjectSlug(tt.in, tt.vcs) + if tt.wantError { + if err == nil { + t.Fatalf("expected error, got nil") + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got != tt.want { + t.Fatalf("expected %q, got %q", tt.want, got) + } + }) + } +} + +func TestBelongsToOrg(t *testing.T) { + if !belongsToOrg("github/my-org/my-repo", "my-org") { + t.Fatal("expected project to belong to org") + } + if belongsToOrg("github/other-org/my-repo", "my-org") { + t.Fatal("expected project to not belong to org") + } +} + +func TestNormalizedOrgName(t *testing.T) { + tests := []struct { + in string + want string + }{ + {in: "my-org", want: "my-org"}, + {in: "github/my-org", want: "my-org"}, + {in: "gh/my-org", want: "my-org"}, + {in: "", want: ""}, + } + + for _, tt := range tests { + if got := normalizedOrgName(tt.in); got != tt.want { + t.Fatalf("normalizedOrgName(%q) = %q, want %q", tt.in, got, tt.want) + } + } +} + +func TestOrgSlugCandidates(t *testing.T) { + t.Run("prefixed org adds plain candidate", func(t *testing.T) { + got := orgSlugCandidates("github/storybookjs", "github") + if len(got) == 0 { + t.Fatal("expected non-empty candidates") + } + if got[0] != "github/storybookjs" { + t.Fatalf("expected first candidate to preserve input, got %q", got[0]) + } + seenPlain := false + for _, c := range got { + if c == "storybookjs" { + seenPlain = true + break + } + } + if !seenPlain { + t.Fatalf("expected plain org candidate in %v", got) + } + }) + + t.Run("app pipelines url extracts vcs org", func(t *testing.T) { + got := orgSlugCandidates("https://app.circleci.com/pipelines/github/storybookjs/storybook", "github") + if len(got) == 0 { + t.Fatal("expected non-empty candidates") + } + seenPrefixed := false + seenPlain := false + for _, c := range got { + if c == "github/storybookjs" { + seenPrefixed = true + } + if c == "storybookjs" { + seenPlain = true + } + } + if !seenPrefixed || !seenPlain { + t.Fatalf("expected both github/storybookjs and storybookjs candidates, got %v", got) + } + }) +} + +func TestOrgDiscoveryHint(t *testing.T) { + t.Run("project url input", func(t *testing.T) { + hint := orgDiscoveryHint("https://app.circleci.com/pipelines/github/storybookjs/storybook") + want := "--org appears to be a project URL; use --project github/storybookjs/storybook instead" + if hint != want { + t.Fatalf("unexpected hint: %q", hint) + } + }) + + t.Run("project selector input", func(t *testing.T) { + hint := orgDiscoveryHint("github/storybookjs/storybook") + want := "--org appears to be a project selector; use --project github/storybookjs/storybook instead" + if hint != want { + t.Fatalf("unexpected hint: %q", hint) + } + }) + + t.Run("org input", func(t *testing.T) { + hint := orgDiscoveryHint("github/storybookjs") + if hint == "" { + t.Fatal("expected non-empty generic hint") + } + }) +} + +func TestVCSFromURL(t *testing.T) { + tests := []struct { + in string + want string + }{ + {in: "https://github.com/example/repo", want: "github"}, + {in: "https://bitbucket.org/example/repo", want: "bitbucket"}, + {in: "https://example.com/example/repo", want: ""}, + } + + for _, tt := range tests { + if got := vcsFromURL(tt.in); got != tt.want { + t.Fatalf("vcsFromURL(%q) = %q, want %q", tt.in, got, tt.want) + } + } +} + +func TestNormalizeVCSName(t *testing.T) { + tests := []struct { + in string + want string + }{ + {in: "github", want: "github"}, + {in: "gh", want: "github"}, + {in: "circleci", want: "circleci"}, + {in: "bb", want: "bitbucket"}, + {in: "bitbucket", want: "bitbucket"}, + } + + for _, tt := range tests { + if got := normalizeVCSName(tt.in); got != tt.want { + t.Fatalf("normalizeVCSName(%q) = %q, want %q", tt.in, got, tt.want) + } + } +} + +func TestCircleciUUIDSlug(t *testing.T) { + slug, ok := circleciUUIDSlug("//circleci.com/3901667c-bcfd-4296-8bda-c5c6e35ab886/4856fff8-1113-43d7-a091-4f7950757db9") + if !ok { + t.Fatal("expected slug extraction to succeed") + } + + want := "circleci/3901667c-bcfd-4296-8bda-c5c6e35ab886/4856fff8-1113-43d7-a091-4f7950757db9" + if slug != want { + t.Fatalf("expected %q, got %q", want, slug) + } +} + +func TestProjectSlugFromV1(t *testing.T) { + item := v1ProjectItem{ + Username: "pipeleek", + Reponame: "pipeleek-secrets-demo", + VCSURL: "//circleci.com/3901667c-bcfd-4296-8bda-c5c6e35ab886/4856fff8-1113-43d7-a091-4f7950757db9", + VCSType: "circleci", + } + + slug, ok := projectSlugFromV1(item, "github") + if !ok { + t.Fatal("expected project slug conversion to succeed") + } + + want := "circleci/3901667c-bcfd-4296-8bda-c5c6e35ab886/4856fff8-1113-43d7-a091-4f7950757db9" + if slug != want { + t.Fatalf("expected %q, got %q", want, slug) + } +} + +func TestCircleAppWorkflowURL(t *testing.T) { + if got := circleAppWorkflowURL(""); got != "https://app.circleci.com/pipelines" { + t.Fatalf("unexpected fallback url: %s", got) + } + + if got := circleAppWorkflowURL("wf-123"); got != "https://app.circleci.com/pipelines/workflows/wf-123" { + t.Fatalf("unexpected workflow url: %s", got) + } +} + +func TestCircleAppJobURL(t *testing.T) { + fallback := "https://app.circleci.com/pipelines/workflows/wf-123" + + want := "https://app.circleci.com/pipelines/github/storybookjs/storybook/119097/workflows/4ddee5f3-d2bf-4b90-a3a9-3939595fd3c4/jobs/1339721" + if got := circleAppJobURL("github/storybookjs/storybook", 119097, "4ddee5f3-d2bf-4b90-a3a9-3939595fd3c4", 1339721, fallback); got != want { + t.Fatalf("expected %q, got %q", want, got) + } + + if got := circleAppJobURL("bad-slug", 119097, "wf-123", 42, fallback); got != fallback { + t.Fatalf("expected fallback for invalid slug, got %q", got) + } + + if got := circleAppJobURL("github/storybookjs/storybook", 0, "wf-123", 42, fallback); got != fallback { + t.Fatalf("expected fallback for invalid pipeline number, got %q", got) + } +} + +func TestFlattenLogOutput(t *testing.T) { + t.Run("json array", func(t *testing.T) { + raw := []byte(`[{"message":"line1"},{"message":"line2"}]`) + got := string(flattenLogOutput(raw)) + if got != "line1\nline2\n" { + t.Fatalf("unexpected flattened output: %q", got) + } + }) + + t.Run("json object", func(t *testing.T) { + raw := []byte(`{"message":"single-line"}`) + got := string(flattenLogOutput(raw)) + if got != "single-line" { + t.Fatalf("unexpected flattened output: %q", got) + } + }) + + t.Run("plain text", func(t *testing.T) { + raw := []byte(" hello secrets \n") + got := string(flattenLogOutput(raw)) + if got != "hello secrets" { + t.Fatalf("unexpected flattened output: %q", got) + } + }) +} + +func TestInitializeOptionsOrgDiscoveryHints(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path == "/api/v2/me/collaborations": + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`[]`)) + case strings.HasPrefix(r.URL.Path, "/api/v2/organization/") && strings.HasSuffix(r.URL.Path, "/project"): + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"message":"not found"}`)) + case r.URL.Path == "/api/v1.1/projects": + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode([]v1ProjectItem{{ + Username: "pipeleek", + Reponame: "demo", + VCSType: "github", + VCSURL: "https://github.com/pipeleek/demo", + }}) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + defer ts.Close() + + tests := []struct { + name string + org string + hintMatch string + }{ + { + name: "generic org visibility hint", + org: "github/storybookjs", + hintMatch: "Hint: org-wide discovery requires token visibility", + }, + { + name: "project url hint", + org: "https://app.circleci.com/pipelines/github/storybookjs/storybook", + hintMatch: "Hint: --org appears to be a project URL; use --project github/storybookjs/storybook instead", + }, + { + name: "project selector hint", + org: "github/storybookjs/storybook", + hintMatch: "Hint: --org appears to be a project selector; use --project github/storybookjs/storybook instead", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := InitializeOptions(InitializeOptionsInput{ + Token: "test-token", + CircleURL: ts.URL, + Organization: tt.org, + VCS: "github", + MaxArtifactSize: "1MB", + }) + if err == nil { + t.Fatal("expected InitializeOptions to fail") + } + if !strings.Contains(err.Error(), tt.hintMatch) { + t.Fatalf("expected error to contain %q, got %q", tt.hintMatch, err.Error()) + } + }) + } +} diff --git a/pkg/circle/scan/transport.go b/pkg/circle/scan/transport.go new file mode 100644 index 00000000..7f000fc6 --- /dev/null +++ b/pkg/circle/scan/transport.go @@ -0,0 +1,443 @@ +package scan + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "strings" + + "github.com/CircleCI-Public/circleci-cli/api/rest" + "github.com/rs/zerolog/log" +) + +type CircleClient interface { + ListCollaborations(ctx context.Context) ([]collaborationItem, error) + ListOrganizationProjects(ctx context.Context, orgSlug, defaultVCS string) ([]string, error) + ListAccessibleProjectsV1(ctx context.Context, defaultVCS, orgFilter string) ([]string, error) + ListPipelines(ctx context.Context, projectSlug, branch, pageToken string) ([]pipelineItem, string, error) + ListPipelineWorkflows(ctx context.Context, pipelineID string) ([]workflowItem, error) + ListWorkflowJobs(ctx context.Context, workflowID string) ([]workflowJobItem, error) + GetProjectJob(ctx context.Context, projectSlug string, jobNumber int) (projectJobResponse, error) + GetProjectJobV1(ctx context.Context, projectSlug string, jobNumber int) (projectJobResponse, error) + ListJobArtifacts(ctx context.Context, projectSlug string, jobNumber int) ([]jobArtifactItem, error) + ListJobTests(ctx context.Context, projectSlug string, jobNumber int) ([]jobTestItem, error) + ListProjectInsightsWorkflows(ctx context.Context, projectSlug, branch string) ([]string, error) + GetProjectInsightsWorkflow(ctx context.Context, projectSlug, workflowName, branch string) (map[string]interface{}, error) + DownloadWithAuth(ctx context.Context, rawURL string) ([]byte, error) +} + +type circleAPIClient struct { + restClient *rest.Client + httpClient *http.Client + token string +} + +func newCircleAPIClient(baseURL *url.URL, token string, httpClient *http.Client) *circleAPIClient { + return &circleAPIClient{ + restClient: rest.New(baseURL, token, httpClient), + httpClient: httpClient, + token: token, + } +} + +type collaborationItem struct { + ID string `json:"id"` + Slug string `json:"slug"` + Name string `json:"name"` + VCSType string `json:"vcs-type"` +} + +type pipelineListResponse struct { + Items []pipelineItem `json:"items"` + NextPageToken string `json:"next_page_token"` +} + +type orgProjectListResponse struct { + Items []struct { + Slug string `json:"slug"` + } `json:"items"` + NextPageToken string `json:"next_page_token"` +} + +type v1ProjectItem struct { + Username string `json:"username"` + Reponame string `json:"reponame"` + VCSURL string `json:"vcs_url"` + VCSType string `json:"vcs_type"` +} + +type pipelineItem struct { + ID string `json:"id"` + Number int `json:"number"` + State string `json:"state"` + CreatedAt string `json:"created_at"` +} + +type workflowListResponse struct { + Items []workflowItem `json:"items"` +} + +type workflowItem struct { + ID string `json:"id"` + Name string `json:"name"` + Status string `json:"status"` + CreatedAt string `json:"created_at"` +} + +type workflowJobListResponse struct { + Items []workflowJobItem `json:"items"` +} + +type workflowJobItem struct { + JobNumber int `json:"job_number"` + Name string `json:"name"` + Status string `json:"status"` +} + +type projectJobResponse struct { + Name string `json:"name"` + WebURL string `json:"web_url"` + Steps []struct { + Name string `json:"name"` + Actions []struct { + Step int `json:"step"` + Index int `json:"index"` + Name string `json:"name"` + OutputURL string `json:"output_url"` + } `json:"actions"` + } `json:"steps"` +} + +type jobArtifactsResponse struct { + Items []jobArtifactItem `json:"items"` +} + +type jobArtifactItem struct { + Path string `json:"path"` + URL string `json:"url"` +} + +type jobTestsResponse struct { + Items []jobTestItem `json:"items"` +} + +type jobTestItem struct { + Name string `json:"name"` + Result string `json:"result"` + Message string `json:"message"` + File string `json:"file"` +} + +func (c *circleAPIClient) ListPipelines(ctx context.Context, projectSlug, branch, pageToken string) ([]pipelineItem, string, error) { + q := url.Values{} + if branch != "" { + q.Set("branch", branch) + } + if pageToken != "" { + q.Set("page-token", pageToken) + } + + var out pipelineListResponse + if err := c.getJSON(ctx, fmt.Sprintf("project/%s/pipeline", projectSlug), q, &out); err != nil { + return nil, "", err + } + return out.Items, out.NextPageToken, nil +} + +func (c *circleAPIClient) ListCollaborations(ctx context.Context) ([]collaborationItem, error) { + var items []collaborationItem + if err := c.getJSON(ctx, "me/collaborations", nil, &items); err != nil { + return nil, err + } + return items, nil +} + +func (c *circleAPIClient) ListOrganizationProjects(ctx context.Context, orgSlug, defaultVCS string) ([]string, error) { + candidates := orgSlugCandidates(orgSlug, defaultVCS) + // For circleci-native orgs the v2 API requires the org UUID, not the slug. + // Attempt to resolve it via the collaborations endpoint. + if collabs, err := c.ListCollaborations(ctx); err == nil { + for _, collab := range collabs { + matched := false + for _, candidate := range candidates { + if strings.EqualFold(collab.Slug, candidate) || strings.EqualFold(collab.Name, candidate) { + matched = true + break + } + } + if matched { + if collab.ID != "" { + candidates = append(candidates, collab.ID) + } + break + } + } + } + candidates = uniqueStrings(candidates) + + var lastErr error + for _, candidate := range candidates { + var out []string + var pageToken string + + for { + q := url.Values{} + if pageToken != "" { + q.Set("page-token", pageToken) + } + + var resp orgProjectListResponse + if err := c.getJSON(ctx, fmt.Sprintf("organization/%s/project", candidate), q, &resp); err != nil { + lastErr = err + out = nil + break + } + + for _, item := range resp.Items { + slug := strings.TrimSpace(item.Slug) + if slug == "" { + continue + } + if !strings.Contains(slug, "/") { + continue + } + if len(strings.Split(slug, "/")) == 2 { + slug = fmt.Sprintf("%s/%s", defaultVCS, slug) + } + out = append(out, slug) + } + + if resp.NextPageToken == "" { + break + } + pageToken = resp.NextPageToken + } + + if len(out) > 0 { + return out, nil + } + } + + if lastErr != nil { + return nil, lastErr + } + return nil, fmt.Errorf("organization %q has no accessible projects", orgSlug) +} + +func (c *circleAPIClient) ListAccessibleProjectsV1(ctx context.Context, defaultVCS, orgFilter string) ([]string, error) { + requestURL, err := c.restClient.BaseURL.Parse("../v1.1/projects") + if err != nil { + return nil, err + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, requestURL.String(), nil) + if err != nil { + return nil, err + } + req.Header.Set("Circle-Token", c.token) + req.Header.Set("Accept", "application/json") + + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode >= 300 { + return nil, fmt.Errorf("v1 project discovery failed: %s", resp.Status) + } + + var items []v1ProjectItem + if err := json.NewDecoder(resp.Body).Decode(&items); err != nil { + return nil, err + } + + projects := make([]string, 0, len(items)) + normalizedFilter := strings.ToLower(strings.TrimSpace(orgFilter)) + for _, item := range items { + log.Debug(). + Str("username", strings.TrimSpace(item.Username)). + Str("reponame", strings.TrimSpace(item.Reponame)). + Str("vcsType", strings.TrimSpace(item.VCSType)). + Str("vcsURL", strings.TrimSpace(item.VCSURL)). + Msg("Discovered project from CircleCI v1 API") + + if normalizedFilter != "" && strings.ToLower(strings.TrimSpace(item.Username)) != normalizedFilter { + log.Debug(). + Str("username", strings.TrimSpace(item.Username)). + Str("orgFilter", orgFilter). + Msg("Skipped discovered project due to org filter mismatch") + continue + } + + slug, ok := projectSlugFromV1(item, defaultVCS) + if !ok { + log.Debug(). + Str("username", strings.TrimSpace(item.Username)). + Str("reponame", strings.TrimSpace(item.Reponame)). + Str("vcsType", strings.TrimSpace(item.VCSType)). + Msg("Skipped discovered project because slug normalization failed") + continue + } + + log.Debug(). + Str("slug", slug). + Msg("Normalized discovered project to scan slug") + projects = append(projects, slug) + } + + projects = uniqueStrings(projects) + if len(projects) == 0 { + return nil, fmt.Errorf("no accessible projects returned by v1 discovery") + } + + return projects, nil +} + +func (c *circleAPIClient) ListPipelineWorkflows(ctx context.Context, pipelineID string) ([]workflowItem, error) { + var out workflowListResponse + if err := c.getJSON(ctx, fmt.Sprintf("pipeline/%s/workflow", pipelineID), nil, &out); err != nil { + return nil, err + } + return out.Items, nil +} + +func (c *circleAPIClient) ListWorkflowJobs(ctx context.Context, workflowID string) ([]workflowJobItem, error) { + var out workflowJobListResponse + if err := c.getJSON(ctx, fmt.Sprintf("workflow/%s/job", workflowID), nil, &out); err != nil { + return nil, err + } + return out.Items, nil +} + +func (c *circleAPIClient) GetProjectJob(ctx context.Context, projectSlug string, jobNumber int) (projectJobResponse, error) { + var out projectJobResponse + err := c.getJSON(ctx, fmt.Sprintf("project/%s/job/%s", projectSlug, strconv.Itoa(jobNumber)), nil, &out) + return out, err +} + +func (c *circleAPIClient) GetProjectJobV1(ctx context.Context, projectSlug string, jobNumber int) (projectJobResponse, error) { + requestURL, err := c.restClient.BaseURL.Parse(fmt.Sprintf("../v1.1/project/%s/%d", projectSlug, jobNumber)) + if err != nil { + return projectJobResponse{}, err + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, requestURL.String(), nil) + if err != nil { + return projectJobResponse{}, err + } + req.Header.Set("Circle-Token", c.token) + req.Header.Set("Accept", "application/json") + + resp, err := c.httpClient.Do(req) + if err != nil { + return projectJobResponse{}, err + } + defer resp.Body.Close() + + if resp.StatusCode >= 300 { + return projectJobResponse{}, fmt.Errorf("v1 job details failed: %s", resp.Status) + } + + var out projectJobResponse + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + return projectJobResponse{}, err + } + + return out, nil +} + +func (c *circleAPIClient) ListJobArtifacts(ctx context.Context, projectSlug string, jobNumber int) ([]jobArtifactItem, error) { + var out jobArtifactsResponse + if err := c.getJSON(ctx, fmt.Sprintf("project/%s/%s/artifacts", projectSlug, strconv.Itoa(jobNumber)), nil, &out); err != nil { + return nil, err + } + return out.Items, nil +} + +func (c *circleAPIClient) ListJobTests(ctx context.Context, projectSlug string, jobNumber int) ([]jobTestItem, error) { + var out jobTestsResponse + if err := c.getJSON(ctx, fmt.Sprintf("project/%s/%s/tests", projectSlug, strconv.Itoa(jobNumber)), nil, &out); err != nil { + return nil, err + } + return out.Items, nil +} + +func (c *circleAPIClient) ListProjectInsightsWorkflows(ctx context.Context, projectSlug, branch string) ([]string, error) { + q := url.Values{} + if branch != "" { + q.Set("branch", branch) + } + + var resp struct { + Items []struct { + Name string `json:"name"` + } `json:"items"` + } + if err := c.getJSON(ctx, fmt.Sprintf("insights/%s/workflows", projectSlug), q, &resp); err != nil { + return nil, err + } + + out := make([]string, 0, len(resp.Items)) + for _, item := range resp.Items { + if strings.TrimSpace(item.Name) != "" { + out = append(out, item.Name) + } + } + + return out, nil +} + +func (c *circleAPIClient) GetProjectInsightsWorkflow(ctx context.Context, projectSlug, workflowName, branch string) (map[string]interface{}, error) { + q := url.Values{} + if branch != "" { + q.Set("branch", branch) + } + + var resp map[string]interface{} + if err := c.getJSON(ctx, fmt.Sprintf("insights/%s/workflows/%s", projectSlug, url.PathEscape(workflowName)), q, &resp); err != nil { + return nil, err + } + + return resp, nil +} + +func (c *circleAPIClient) DownloadWithAuth(ctx context.Context, rawURL string) ([]byte, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) + if err != nil { + return nil, err + } + req.Header.Set("Circle-Token", c.token) + + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode >= 300 { + return nil, fmt.Errorf("download failed: %s", resp.Status) + } + + return io.ReadAll(resp.Body) +} + +func (c *circleAPIClient) getJSON(ctx context.Context, path string, query url.Values, out interface{}) error { + u := &url.URL{Path: path} + if query != nil { + u.RawQuery = query.Encode() + } + + req, err := c.restClient.NewRequest(http.MethodGet, u, nil) + if err != nil { + return err + } + req = req.WithContext(ctx) + + _, err = c.restClient.DoRequest(req, out) + return err +} diff --git a/pkg/circle/scan/transport_test.go b/pkg/circle/scan/transport_test.go new file mode 100644 index 00000000..f8b7acee --- /dev/null +++ b/pkg/circle/scan/transport_test.go @@ -0,0 +1,255 @@ +package scan + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "net/url" + "sync" + "testing" +) + +func TestListOrganizationProjectsCandidateFallback(t *testing.T) { + var ( + mu sync.Mutex + requests []string + ) + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + requests = append(requests, r.URL.Path) + mu.Unlock() + + switch r.URL.Path { + case "/api/v2/me/collaborations": + // Return empty — no UUID resolution available in this test scenario. + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`[]`)) + case "/api/v2/organization/my-org/project": + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"message":"not found"}`)) + case "/api/v2/organization/github/my-org/project": + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"items":[{"slug":"my-org/repo-a"},{"slug":"bitbucket/other/repo-b"}],"next_page_token":""}`)) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + defer ts.Close() + + baseURL, err := url.Parse(ts.URL + "/api/v2/") + if err != nil { + t.Fatalf("failed to parse base url: %v", err) + } + + client := newCircleAPIClient(baseURL, "token", ts.Client()) + projects, err := client.ListOrganizationProjects(context.Background(), "my-org", "github") + if err != nil { + t.Fatalf("expected fallback candidate lookup to succeed, got error: %v", err) + } + + if len(projects) != 2 { + t.Fatalf("expected 2 projects, got %d: %#v", len(projects), projects) + } + if projects[0] != "github/my-org/repo-a" { + t.Fatalf("unexpected first project: %q", projects[0]) + } + if projects[1] != "bitbucket/other/repo-b" { + t.Fatalf("unexpected second project: %q", projects[1]) + } + + mu.Lock() + defer mu.Unlock() + // requests[0] = me/collaborations (UUID resolution attempt) + // requests[1] = organization/my-org/project (first slug candidate, 404) + // requests[2] = organization/github/my-org/project (VCS-prefixed candidate, succeeds) + if len(requests) < 3 { + t.Fatalf("expected at least 3 requests, got %d: %v", len(requests), requests) + } + if requests[0] != "/api/v2/me/collaborations" { + t.Fatalf("expected collaborations request first, got %q", requests[0]) + } + if requests[1] != "/api/v2/organization/my-org/project" { + t.Fatalf("expected first candidate request path, got %q", requests[1]) + } + if requests[2] != "/api/v2/organization/github/my-org/project" { + t.Fatalf("expected second candidate request path, got %q", requests[2]) + } +} + +func TestListOrganizationProjectsCollaborationUUIDResolution(t *testing.T) { + const orgUUID = "96df906d-3617-46fd-96d0-8f80a8c4d00a" + const orgSlug = "circleci/KdZvpc432VpdV8UBajzc9f" + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/api/v2/me/collaborations": + payload := []collaborationItem{{ + ID: orgUUID, + Slug: orgSlug, + Name: "My Org", + VCSType: "circleci", + }} + _ = json.NewEncoder(w).Encode(payload) + case "/api/v2/organization/" + orgSlug + "/project": + // slug-based call fails — UUID not accepted at this path + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"message":"not found"}`)) + case "/api/v2/organization/" + orgUUID + "/project": + // UUID-based call succeeds + _, _ = w.Write([]byte(`{"items":[{"slug":"github/my-org/repo-a"}],"next_page_token":""}`)) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + defer ts.Close() + + baseURL, err := url.Parse(ts.URL + "/api/v2/") + if err != nil { + t.Fatalf("failed to parse base url: %v", err) + } + + client := newCircleAPIClient(baseURL, "token", ts.Client()) + projects, err := client.ListOrganizationProjects(context.Background(), orgSlug, "github") + if err != nil { + t.Fatalf("expected UUID-resolution to succeed, got error: %v", err) + } + if len(projects) != 1 || projects[0] != "github/my-org/repo-a" { + t.Fatalf("unexpected projects: %#v", projects) + } +} + +func TestListOrganizationProjectsPrefixedOrgFallback(t *testing.T) { + var requests []string + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests = append(requests, r.URL.Path) + switch r.URL.Path { + case "/api/v2/me/collaborations": + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`[]`)) + case "/api/v2/organization/github/storybookjs/project": + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"message":"not found"}`)) + case "/api/v2/organization/storybookjs/project": + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"items":[{"slug":"storybookjs/repo-a"}],"next_page_token":""}`)) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + defer ts.Close() + + baseURL, err := url.Parse(ts.URL + "/api/v2/") + if err != nil { + t.Fatalf("failed to parse base url: %v", err) + } + + client := newCircleAPIClient(baseURL, "token", ts.Client()) + projects, err := client.ListOrganizationProjects(context.Background(), "github/storybookjs", "github") + if err != nil { + t.Fatalf("expected prefixed fallback to succeed, got error: %v", err) + } + if len(projects) != 1 || projects[0] != "github/storybookjs/repo-a" { + t.Fatalf("unexpected projects: %#v", projects) + } + + if len(requests) < 3 { + t.Fatalf("expected at least 3 requests, got %d (%v)", len(requests), requests) + } + if requests[1] != "/api/v2/organization/github/storybookjs/project" { + t.Fatalf("unexpected first org candidate request: %q", requests[1]) + } + if requests[2] != "/api/v2/organization/storybookjs/project" { + t.Fatalf("unexpected fallback org candidate request: %q", requests[2]) + } +} + +func TestListOrganizationProjectsCollaborationNameCaseInsensitive(t *testing.T) { + const orgUUID = "96df906d-3617-46fd-96d0-8f80a8c4d00a" + var uuidRequests int + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/api/v2/me/collaborations": + // Name matches the normalized org candidate with mixed casing. + payload := []collaborationItem{{ + ID: orgUUID, + Slug: "circleci/some-other-org", + Name: "StOrYbOoKjS", + }} + _ = json.NewEncoder(w).Encode(payload) + case "/api/v2/organization/github/storybookjs/project": + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"message":"not found"}`)) + case "/api/v2/organization/storybookjs/project": + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"message":"not found"}`)) + case "/api/v2/organization/" + orgUUID + "/project": + uuidRequests++ + _, _ = w.Write([]byte(`{"items":[{"slug":"github/storybookjs/repo-a"}],"next_page_token":""}`)) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + defer ts.Close() + + baseURL, err := url.Parse(ts.URL + "/api/v2/") + if err != nil { + t.Fatalf("failed to parse base url: %v", err) + } + + client := newCircleAPIClient(baseURL, "token", ts.Client()) + projects, err := client.ListOrganizationProjects(context.Background(), "github/storybookjs", "github") + if err != nil { + t.Fatalf("expected name-based UUID fallback to succeed, got error: %v", err) + } + if len(projects) != 1 || projects[0] != "github/storybookjs/repo-a" { + t.Fatalf("unexpected projects: %#v", projects) + } + if uuidRequests != 1 { + t.Fatalf("expected exactly one UUID project request, got %d", uuidRequests) + } +} + +func TestListAccessibleProjectsV1FiltersAndNormalizes(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/api/v1.1/projects" { + w.WriteHeader(http.StatusNotFound) + return + } + + w.Header().Set("Content-Type", "application/json") + payload := []v1ProjectItem{ + {Username: "team", Reponame: "repo-a", VCSType: "github", VCSURL: "https://github.com/team/repo-a"}, + {Username: "other", Reponame: "repo-z", VCSType: "github", VCSURL: "https://github.com/other/repo-z"}, + {Username: "team", Reponame: "ignored", VCSType: "circleci", VCSURL: "//circleci.com/org-uuid/proj-uuid"}, + } + _ = json.NewEncoder(w).Encode(payload) + })) + defer ts.Close() + + baseURL, err := url.Parse(ts.URL + "/api/v2/") + if err != nil { + t.Fatalf("failed to parse base url: %v", err) + } + + client := newCircleAPIClient(baseURL, "token", ts.Client()) + projects, err := client.ListAccessibleProjectsV1(context.Background(), "github", "team") + if err != nil { + t.Fatalf("expected v1 discovery to succeed, got error: %v", err) + } + + if len(projects) != 2 { + t.Fatalf("expected 2 filtered projects, got %d: %#v", len(projects), projects) + } + if projects[0] != "github/team/repo-a" { + t.Fatalf("unexpected first project: %q", projects[0]) + } + if projects[1] != "circleci/org-uuid/proj-uuid" { + t.Fatalf("unexpected second project: %q", projects[1]) + } +} diff --git a/tests/e2e/circle/scan/scan_test.go b/tests/e2e/circle/scan/scan_test.go new file mode 100644 index 00000000..211d8431 --- /dev/null +++ b/tests/e2e/circle/scan/scan_test.go @@ -0,0 +1,136 @@ +//go:build e2e + +package e2e + +import ( + "encoding/json" + "fmt" + "net/http" + "strings" + "testing" + "time" + + "github.com/CompassSecurity/pipeleek/tests/e2e/internal/testutil" + "github.com/stretchr/testify/assert" +) + +func TestCircleScan_ProjectHappyPath(t *testing.T) { + server, getRequests, cleanup := testutil.StartMockServerWithRecording(t, func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + baseURL := "http://" + r.Host + + switch { + case r.URL.Path == "/api/v2/project/github/example-org/example-repo/pipeline": + _ = json.NewEncoder(w).Encode(map[string]interface{}{ + "items": []map[string]interface{}{{ + "id": "pipeline-1", + "state": "created", + "created_at": "2026-01-10T10:00:00Z", + }}, + "next_page_token": "", + }) + case r.URL.Path == "/api/v2/pipeline/pipeline-1/workflow": + _ = json.NewEncoder(w).Encode(map[string]interface{}{ + "items": []map[string]interface{}{{ + "id": "wf-1", + "name": "build", + "status": "success", + "created_at": "2026-01-10T10:05:00Z", + }}, + }) + case r.URL.Path == "/api/v2/workflow/wf-1/job": + _ = json.NewEncoder(w).Encode(map[string]interface{}{ + "items": []map[string]interface{}{{ + "job_number": 101, + "name": "unit-tests", + "status": "success", + }}, + }) + case r.URL.Path == "/api/v2/project/github/example-org/example-repo/job/101": + _ = json.NewEncoder(w).Encode(map[string]interface{}{ + "name": "unit-tests", + "web_url": fmt.Sprintf("%s/job/101", baseURL), + "steps": []map[string]interface{}{{ + "actions": []map[string]interface{}{{ + "output_url": fmt.Sprintf("%s/log/101", baseURL), + }}, + }}, + }) + case r.URL.Path == "/log/101": + w.Header().Set("Content-Type", "text/plain") + _, _ = w.Write([]byte("build started\nall good\n")) + case r.URL.Path == "/api/v2/project/github/example-org/example-repo/101/tests": + _ = json.NewEncoder(w).Encode(map[string]interface{}{"items": []interface{}{}}) + case r.URL.Path == "/api/v2/insights/github/example-org/example-repo/workflows": + _ = json.NewEncoder(w).Encode(map[string]interface{}{"items": []map[string]interface{}{{"name": "build"}}}) + case r.URL.Path == "/api/v2/insights/github/example-org/example-repo/workflows/build": + _ = json.NewEncoder(w).Encode(map[string]interface{}{"success_rate": 1.0}) + default: + w.WriteHeader(http.StatusOK) + _ = json.NewEncoder(w).Encode(map[string]interface{}{}) + } + }) + defer cleanup() + + stdout, stderr, exitErr := testutil.RunCLI(t, []string{ + "circle", "scan", + "--circle", server.URL, + "--token", "circle-token", + "--project", "example-org/example-repo", + "--max-pipelines", "1", + "--tests", "false", + "--insights", "false", + }, nil, 20*time.Second) + + assert.Nil(t, exitErr, "circle scan should succeed") + requests := getRequests() + assert.True(t, len(requests) >= 5, "expected multiple CircleCI API requests") + + joined := stdout + stderr + t.Logf("Output:\n%s", joined) +} + +func TestCircleScan_OrgDiscovery(t *testing.T) { + server, getRequests, cleanup := testutil.StartMockServerWithRecording(t, func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + + switch { + case r.URL.Path == "/api/v2/organization/example-org/project": + _ = json.NewEncoder(w).Encode(map[string]interface{}{ + "items": []map[string]interface{}{{ + "slug": "github/example-org/example-repo", + }}, + "next_page_token": "", + }) + case r.URL.Path == "/api/v2/project/github/example-org/example-repo/pipeline": + _ = json.NewEncoder(w).Encode(map[string]interface{}{"items": []interface{}{}, "next_page_token": ""}) + case strings.HasPrefix(r.URL.Path, "/api/v2/insights/github/example-org/example-repo/workflows"): + _ = json.NewEncoder(w).Encode(map[string]interface{}{"items": []interface{}{}}) + default: + _ = json.NewEncoder(w).Encode(map[string]interface{}{}) + } + }) + defer cleanup() + + _, _, exitErr := testutil.RunCLI(t, []string{ + "circle", "scan", + "--circle", server.URL, + "--token", "circle-token", + "--org", "example-org", + "--max-pipelines", "1", + "--tests", "false", + "--insights", "false", + }, nil, 15*time.Second) + + assert.Nil(t, exitErr, "circle scan should support org discovery without --project") + + requests := getRequests() + sawOrgProjects := false + for _, req := range requests { + if req.Path == "/api/v2/organization/example-org/project" { + sawOrgProjects = true + break + } + } + assert.True(t, sawOrgProjects, "expected org project discovery request") +}