diff --git a/README.md b/README.md index e67e4cf7..1d2bf07f 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,7 @@ - NTLM/SMB/FTP(S)/RESPONDER Listener **(self-hosted)** - Wildcard / Protected Interactions **(self-hosted)** - Customizable Index / File hosting **(self-hosted)** +- Client file hosting for second-stage OOB payloads **(self-hosted)** - Customizable Payload Length **(self-hosted)** - Custom SSL Certificate **(self-hosted)** @@ -58,34 +59,39 @@ Usage: Flags: INPUT: - -s, -server string interactsh server(s) to use (default "oast.pro,oast.live,oast.site,oast.online,oast.fun,oast.me") + -s, -server string interactsh server(s) to use (default "oast.pro,oast.live,oast.site,oast.online,oast.fun,oast.me") + -fl, -file string[] local file(s) to upload and host on the interactsh server CONFIG: -config string flag configuration file (default "$HOME/.config/interactsh-client/config.yaml") + -auth configure projectdiscovery cloud (pdcp) api key (default true) -n, -number int number of interactsh payload to generate (default 1) -t, -token string authentication token to connect protected interactsh server -pi, -poll-interval int poll interval in seconds to pull interaction data (default 5) -nf, -no-http-fallback disable http fallback registration - -cidl, -correlation-id-length int length of the correlation id preamble (min 3, default 20) - -cidn, -correlation-id-nonce-length int length of the correlation id nonce (min 3, default 13) + -cidl, -correlation-id-length int length of the correlation id preamble (min 3, default 20) (default 20) + -cidn, -correlation-id-nonce-length int length of the correlation id nonce (min 3, default 13) (default 13) -sf, -session-file string store/read from session file + -kai, -keep-alive-interval value keep alive interval (default 1m0s) FILTER: -m, -match string[] match interaction based on the specified pattern -f, -filter string[] filter interaction based on the specified pattern -dns-only display only dns interaction in CLI output - -http-only display only http/https interactions in CLI output + -http-only display only http interaction in CLI output -smtp-only display only smtp interactions in CLI output + -asn include asn information of remote ip in json output UPDATE: -up, -update update interactsh-client to latest version -duc, -disable-update-check disable automatic interactsh-client update check - + OUTPUT: -o string output file to write interaction data -json write output in JSON Lines format - -ps, -payload-store enable storing generated interactsh payload to file + -ps, -payload-store write generated interactsh payload to file -psf, -payload-store-file string store generated interactsh payloads to given file (default "interactsh_payload.txt") + -fsf, -file-store-file string store hosted file URLs to given file (requires -file) -v display verbose interaction DEBUG: @@ -352,7 +358,7 @@ Usage: Flags: INPUT: -d, -domain string[] single/multiple configured domain to use for server - -ip string[] public ip address(es) to use for interactsh server (comma-separated,supports both IPv4 & IPv6) + -i, -ip string[] public IP address(es) to use for interactsh server (comma-separated, supports both IPv4 & IPv6) -lip, -listen-ip string public ip address to listen on (default "0.0.0.0") -e, -eviction int number of days to persist interaction data in memory (default 30) -ne, -no-eviction disable periodic data eviction from memory @@ -362,29 +368,31 @@ INPUT: -acao-url string origin url to send in acao header to use web-client) (default "*") -sa, -skip-acme skip acme registration (certificate checks/handshake + TLS protocols will be disabled) -se, -scan-everywhere scan canary token everywhere - -cidl, -correlation-id-length int length of the correlation id preamble (min 3, default 20) - -cidn, -correlation-id-nonce-length int length of the correlation id nonce (min 3, default 13) + -cidl, -correlation-id-length int length of the correlation id preamble (min 3, default 20) (default 20) + -cidn, -correlation-id-nonce-length int length of the correlation id nonce (min 3, default 13) (default 13) -cert string custom certificate path -privkey string custom private key path -oih, -origin-ip-header string HTTP header containing origin ip (interactsh behind a reverse proxy) CONFIG: - -r, -resolvers string[] list of resolvers to use (file or comma separated) - -config string flag configuration file (default "$HOME/.config/interactsh-server/config.yaml") - -dr, -dynamic-resp enable setting up arbitrary response data - -cr, -custom-records string custom dns records YAML file for DNS server - -hi, -http-index string custom index file for http server + -r, -resolvers string[] list of resolvers to use (file or comma separated) + -config string flag configuration file (default "$HOME/.config/interactsh-server/config.yaml") + -dr, -dynamic-resp enable setting up arbitrary response data + -cr, -custom-records string custom dns records YAML file for DNS server + -hi, -http-index string custom index file for http server + -hd, -http-directory string directory with files to serve with http server -dhr, -default-http-response string file to serve for all http requests (takes priority over other options) - -hd, -http-directory string directory with files to serve with http server - -ds, -disk disk based storage - -dsp, -disk-path string disk storage path - -csh, -server-header string custom value of Server header in response - -dv, -disable-version disable publishing interactsh version in response header + -ds, -disk disk based storage + -dsp, -disk-path string disk storage path + -ru, -redis-url string redis connection URL (enables shared state for multi-instance deployments) + -rp, -redis-prefix string redis key prefix (default "interactsh:") + -csh, -server-header string custom value of Server header in response + -dv, -disable-version disable publishing interactsh version in response header UPDATE: -up, -update update interactsh-server to latest version -duc, -disable-update-check disable automatic interactsh-server update check - + SERVICES: -dns-port int port to use for dns service (default 53) -http-port int port to use for http service (default 80) @@ -403,6 +411,14 @@ SERVICES: -ftps-port int port to use for ftps service (default 990) -ftp-dir string ftp directory - temporary if not specified +UPLOAD: + -upload enable client file upload and hosting - self-hosted servers only (authenticated) + -ud, -upload-directory string directory to host uploaded files from - temporary if not specified; interactsh creates and prunes .interactsh-user-uploads inside it + -umfs, -upload-max-file-size value maximum size of a single uploaded file (default 1mb) + -umf, -upload-max-files int maximum number of uploaded files per session (default 5) + -umts, -upload-max-total-size value maximum total size of all uploaded files on the server (default 1gb) + -ut, -upload-ttl value maximum lifetime of uploaded files (default 24h0m0s) + DEBUG: -version show version of the project -debug start interactsh server in debug mode @@ -654,6 +670,152 @@ interactsh-server -d hackwithautomation.com -http-directory ./paylods ![image](https://user-images.githubusercontent.com/8293321/179396480-d5ff8399-8b91-48aa-b21f-c67e40e80945.png) +## Client File Hosting + +Where `-http-directory` hosts operator-supplied files globally, `-upload` lets a **client** host files +against its own correlation ID. This is aimed at second-stage out-of-band vulnerabilities — XXE with an +external DTD, XSLT includes, JNDI staging — where the target must fetch a payload file before the +callback fires. Each fetch is recorded as an interaction, so the second stage is visible in the client +output. + +> [!WARNING] +> `-upload` is intended for **self-hosted servers only**. Enabling it on a public instance turns it into +> anonymous file hosting on a domain with a valid wildcard certificate, which is a magnet for malware +> staging, and blocklists act on the registrable domain — one abusive sample affects every user of that +> domain. It is off by default and implies authentication when enabled. + +Start a server with uploads enabled: + +```bash +interactsh-server -d hackwithautomation.com -upload -ftp +``` + +Then point a client at it with one or more files: + +```bash +interactsh-client -s https://hackwithautomation.com -t -file evil.dtd +``` + +```console +[INF] Listing 1 payload for OOB Testing +[INF] c6rj61aciaeutn2ae680cndmnioyyyyyn.hackwithautomation.com +[INF] Hosting 1 file(s) for OOB Testing +[INF] https://c6rj61aciaeutn2ae680xk4tqy8pqhwmi.hackwithautomation.com/f/evil.dtd +[INF] ftp://c6rj61aciaeutn2ae680xk4tqy8pqhwmi.hackwithautomation.com/.interactsh-user-uploads/c6rj61aciaeutn2ae680/evil.dtd +``` + +Files are served over HTTP(S), and over FTP(S) as well when `-ftp` is enabled. Responses are always +`Content-Type: application/octet-stream` with `Content-Disposition: attachment`, so the server never +renders client-supplied HTML or SVG on its own domain; DTD, XSLT and JNDI consumers ignore content type, +so this costs nothing for the intended use. + +When the target fetches the file, the fetch arrives in the client like any other interaction — which is +the point: it is the evidence that the first stage of the payload actually executed. The response body +is replaced by a digest so a large payload is not copied back into the interaction stream on every +fetch: + +```console +[c6rj61aciaeutn2ae680xk4tqy8pqhwmi] Received HTTP interaction from 203.0.113.7 at 2026-08-05 15:47:19 +------------ +HTTP Request +------------ + +GET /f/evil.dtd HTTP/1.1 +Host: c6rj61aciaeutn2ae680xk4tqy8pqhwmi.hackwithautomation.com +Accept: */* +User-Agent: curl/8.18.0 + + + +------------- +HTTP Response +------------- + +HTTP/1.1 200 OK +Content-Type: application/octet-stream +Content-Disposition: attachment; filename="evil.dtd" +Content-Length: 144 + +[body elided: 144 of 144 bytes of uploaded file "evil.dtd", sha256 0c1b960b076cdff8666f1f302dddd8f3ff0e6ed4b6c09002fbe6d1cdbb5d68f8] +``` + +The two counts are "delivered of hosted": a conditional fetch answered `304` records `0 of 144`, and a +ranged one records the bytes the range actually carried, so the record cannot claim a delivery that did +not happen. + +Whatever second-stage callback the payload then triggers arrives as a further interaction on the same +correlation ID, so both stages land in one client. + +A client asked to host files against a server that was not started with `-upload` says so and stops, +rather than silently continuing without the payload: + +```console +$ interactsh-client -s https://hackwithautomation.com -t -file evil.dtd +[FTL] Server https://hackwithautomation.com does not accept file uploads; it must be started with -upload +``` + +The failing server is named because the client registers with only one of the servers in `-s`. When +several are listed, it also says how the choice was made, since the outcome can differ between runs: + +```console +$ interactsh-client -s https://a.example,https://b.example -t -file evil.dtd +[FTL] Server https://a.example does not accept file uploads; it must be started with -upload (chosen at random from the 2 servers in -s, so this may differ between runs; pass a single server with -file) +``` + +Server-side options: + +| Flag | Default | Description | +| --- | --- | --- | +| `-upload` | off | enable client file upload and hosting | +| `-ud, -upload-directory` | temporary dir | directory to host uploaded files from; interactsh owns `.interactsh-user-uploads` inside it | +| `-umfs, -upload-max-file-size` | `1mb` | maximum size of a single file | +| `-umf, -upload-max-files` | `5` | maximum files per session | +| `-umts, -upload-max-total-size` | `1gb` | maximum total bytes across all sessions | +| `-ut, -upload-ttl` | `24h` | maximum lifetime of uploaded files | + +Files are removed when the client deregisters, when its session leaves the cache, and in any case once +`-upload-ttl` has elapsed since the last upload for that session. + +Things worth knowing before enabling it: + +- **Hosted files are readable by anyone who learns the correlation ID.** That ID is deliberately leaked + to the target — it appears in every DNS query the target's resolver makes, and so in its DNS logs, its + WAF, and passive-DNS aggregators. A target can fetch your payload to fingerprint your tooling, and + that fetch will appear in your interaction stream. Do not upload anything you would mind a target + reading. +- Uploads are authenticated with the session's correlation ID and secret key, so only the client that + owns a session can attach files to it. +- The client uploads to the **one server it registered with**. If `-s` lists several, files are hosted + only on the elected one; its payload URLs are the ones printed. **Pass a single server with `-file`:** + the client picks one at random from `-s` and cannot take upload support into account, since it only + learns that after registering — so a list mixing upload and non-upload servers fails at random. +- `-upload` cannot be combined with `-redis-url`. Hosted bytes are written to a single instance's local + filesystem, so with a storage backend shared between instances the other instances would advertise + files they do not have. The server refuses to start on that combination: + + ```console + $ interactsh-server -d hackwithautomation.com -upload -redis-url redis://127.0.0.1:6379/0 + [FTL] -upload cannot be used with -redis-url: hosted files are stored on a single instance's local filesystem + ``` +- Uploads refuse to travel over plaintext HTTP to a remote server, since the request carries both the + file and the session secret key. Use an `https://` server URL. +- The default upload directory is a temporary directory, which on many Linux distributions is + memory-backed. Set `-upload-directory` explicitly on a real deployment. +- **Interactsh creates and prunes one directory inside the upload root.** Hosted files are laid out as + `/.interactsh-user-uploads//`, and everything under + `.interactsh-user-uploads` is deleted when its session ends, when `-upload-ttl` expires it, and at + startup — upload metadata lives only in memory, so nothing there survives a restart. The rest of the + root is never touched, which is what makes it safe to point `-upload-directory` at a directory you + already use, or to share it with `-ftp-dir`. +- With `-ftp` and no `-ftp-dir`, the FTP root is set to the upload root so that hosted files are + reachable over FTP with no extra configuration. If you set both flags they must name the same + directory, otherwise FTP cannot see the uploads: the server reports the mismatch at startup and stops + offering `ftp://` URLs to clients, so hosting degrades to HTTP only rather than handing out FTP URLs + that resolve to nothing. The uploads directory is hidden from FTP listings — `LIST /` shows your own content but + not `.interactsh-user-uploads`, and that directory refuses to list its own contents, so an anonymous + client cannot enumerate the correlation IDs that currently have hosted files. `RETR` of a known path + works, which is what the payload URL relies on. + ## Dynamic HTTP Response Interactsh http server optionally enables responding with dynamic HTTP response by using query parameters. This feature can be enabled by using `-dr` or `-dynamic-resp` flag. @@ -870,7 +1032,15 @@ sudo interactsh-server -responder -d localhost ### Use as library -The [examples](examples/) uses interactsh client library to get external interactions for a generated URL by making a http request to the URL. +The [examples](examples/) use the interactsh client library to get external interactions for a generated +URL by making an http request to the URL, and to host a file against the same session for second-stage +verification. + +File hosting is an optional server capability, so a library consumer negotiates rather than assumes: +`Capabilities()` reports what the server advertised at registration, `UploadFiles` returns +`ErrUploadUnsupported` when a server cannot host files and `ErrUploadNotAdvertised` when it predates the +feature, and `FileURL`/`FTPFileURL` compose the URLs a target should fetch. The public `oast.*` servers +do not offer hosting, so the example skips it rather than failing. ### Nuclei - OAST diff --git a/cmd/interactsh-client/main.go b/cmd/interactsh-client/main.go index d0556728..e4f07ec2 100644 --- a/cmd/interactsh-client/main.go +++ b/cmd/interactsh-client/main.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "os" "os/signal" @@ -44,6 +45,11 @@ func main() { flagSet.CreateGroup("input", "Input", flagSet.StringVarP(&cliOptions.ServerURL, "server", "s", defaultOpts.ServerURL, "interactsh server(s) to use"), + // StringSliceOptions, not the FileCommaSeparated variant used by + // -match/-filter: that one reads the file and splits its contents, + // which for -file would turn a DTD into a list of names. + flagSet.StringSliceVarP(&cliOptions.Files, "file", "fl", nil, + "local file(s) to upload and host on the interactsh server", goflags.StringSliceOptions), ) flagSet.CreateGroup("config", "config", @@ -78,6 +84,7 @@ func main() { flagSet.BoolVar(&cliOptions.JSON, "json", false, "write output in JSON Lines format"), flagSet.BoolVarP(&cliOptions.StorePayload, "payload-store", "ps", false, "write generated interactsh payload to file"), flagSet.StringVarP(&cliOptions.StorePayloadFile, "payload-store-file", "psf", settings.StorePayloadFileDefault, "store generated interactsh payloads to given file"), + flagSet.StringVarP(&cliOptions.FileStoreFile, "file-store-file", "fsf", "", "store hosted file URLs to given file (requires -file)"), flagSet.BoolVar(&cliOptions.Verbose, "v", false, "display verbose interaction"), ) @@ -171,6 +178,10 @@ func main() { gologger.Fatal().Msgf("Could not create client: %s\n", err) } + // Uploads must follow registration, since the server verifies the session, + // and precede the payload listing so every URL is shown together. + fileURLs := uploadFiles(client, cliOptions) + interactshURLs := generatePayloadURL(cliOptions.NumberOfPayloads, client) gologger.Info().Msgf("Listing %d payload for OOB Testing\n", cliOptions.NumberOfPayloads) @@ -180,11 +191,31 @@ func main() { warnIfServerLacksIPv6(client) + if len(fileURLs) > 0 { + gologger.Info().Msgf("Hosting %d file(s) for OOB Testing\n", len(cliOptions.Files)) + for _, fileURL := range fileURLs { + gologger.Info().Msgf("%s\n", fileURL) + } + } + + // One record type per file. -psf is a machine-readable list of payload + // hostnames, one per line and exactly -n of them, which is what a wrapper + // script substituting into a payload template relies on; mixing hosted-file + // URLs into it turns "$line" into "https://host/f/x" and silently produces + // nonsense like http://https://host/f/x/. Hosted-file URLs get their own file. if cliOptions.StorePayload && cliOptions.StorePayloadFile != "" { - if err := os.WriteFile(cliOptions.StorePayloadFile, []byte(strings.Join(interactshURLs, "\n")), 0644); err != nil { + if err := writeLines(cliOptions.StorePayloadFile, interactshURLs); err != nil { gologger.Fatal().Msgf("Could not write to payload output file: %s\n", err) } } + if cliOptions.FileStoreFile != "" { + if len(fileURLs) == 0 { + gologger.Warning().Msgf("-file-store-file was given without -file, so no hosted file URLs were written\n") + } + if err := writeLines(cliOptions.FileStoreFile, fileURLs); err != nil { + gologger.Fatal().Msgf("Could not write to file URL output file: %s\n", err) + } + } // show all interactions noFilter := !cliOptions.DNSOnly && !cliOptions.HTTPOnly && !cliOptions.SmtpOnly @@ -320,6 +351,83 @@ func generatePayloadURL(numberOfPayloads int, client *client.Client) []string { return interactshURLs } +// electionHint explains which server the complaint is about when -s named more +// than one. The client registers with a single server chosen at random, and +// upload support cannot influence that choice because it is only advertised in +// the registration response. Without this, a mixed list reads as "none of my +// servers support uploads" on the runs that happen to elect one that does not. +func electionHint(serverList string) string { + var listed int + for _, s := range strings.Split(serverList, ",") { + if strings.TrimSpace(s) != "" { + listed++ + } + } + if listed < 2 { + return "" + } + return fmt.Sprintf(" (chosen at random from the %d servers in -s, so this may differ between runs;"+ + " pass a single server with -file)", listed) +} + +// uploadFiles hosts local files on the interactsh server and returns the URLs a +// target should fetch. It returns nil when no files were requested. +func uploadFiles(c *client.Client, cliOptions *options.CLIClientOptions) []string { + if len(cliOptions.Files) == 0 { + return nil + } + + uploaded, err := c.UploadFiles(cliOptions.Files) + if err != nil { + // Registration already happened -- it has to, since upload support is + // only advertised in the register response -- so a session exists on the + // server. Wind it down the same way the signal handler does, rather than + // leaving it to sit until the eviction TTL: persist it if the user asked + // for a resumable session, otherwise deregister it. Doing neither would + // overstate the server's live session count for every client that trips + // this path, and would strand a session the user cannot resume. + if cliOptions.SessionFile != "" { + _ = c.SaveSessionTo(cliOptions.SessionFile) + } else { + _ = c.Close() + } + // Name the server in both failures. The client registers with one server + // out of -s, so without it the reader cannot tell which of their servers + // the complaint is about. + if errors.Is(err, client.ErrUploadNotAdvertised) { + gologger.Fatal().Msgf("Server %s did not advertise file hosting, so it predates -file; upgrade the server%s\n", + c.ServerURL(), electionHint(cliOptions.ServerURL)) + } + if errors.Is(err, client.ErrUploadUnsupported) { + gologger.Fatal().Msgf("Server %s does not accept file uploads; it must be started with -upload%s\n", + c.ServerURL(), electionHint(cliOptions.ServerURL)) + } + // Fatal rather than a warning: the user asked to host a payload, and + // carrying on without it produces a confusing "no interaction" result. + gologger.Fatal().Msgf("Could not upload files to %s: %s\n", c.ServerURL(), err) + } + + // One payload host for every file, so the target performs a single DNS + // lookup and the output is consistent. Any nonce works, since the server + // only reads the correlation id prefix. + host := c.URL() + withFTP := false + if caps := c.Capabilities(); caps != nil { + withFTP = caps.FTP + } + + var urls []string + for _, file := range uploaded { + urls = append(urls, c.FileURL(host, file)) + // Only when the server actually runs an FTP listener, otherwise the + // URL would never connect. + if withFTP { + urls = append(urls, c.FTPFileURL(host, file)) + } + } + return urls +} + func writeOutput(outputFile *os.File, builder *bytes.Buffer) { if outputFile != nil { _, _ = outputFile.Write(builder.Bytes()) @@ -352,3 +460,17 @@ func (m *regexMatcher) match(item string) bool { } return false } + +// writeLines writes one record per line, newline-terminated. +// +// The terminator matters: without it the last record has no newline, so a plain +// "while read line" loop -- the most likely consumer of these files -- drops it, +// and wc -l reports one fewer record than the file holds. +func writeLines(path string, lines []string) error { + var b strings.Builder + for _, line := range lines { + b.WriteString(line) + b.WriteString("\n") + } + return os.WriteFile(path, []byte(b.String()), 0644) +} diff --git a/cmd/interactsh-client/main_test.go b/cmd/interactsh-client/main_test.go new file mode 100644 index 00000000..1e314451 --- /dev/null +++ b/cmd/interactsh-client/main_test.go @@ -0,0 +1,91 @@ +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestElectionHint(t *testing.T) { + t.Run("stays silent for a single server", func(t *testing.T) { + // Nothing to disambiguate: the message already names the only server. + require.Empty(t, electionHint("https://oast.pro")) + require.Empty(t, electionHint("")) + // A trailing comma still describes one server. + require.Empty(t, electionHint("https://oast.pro,")) + }) + + t.Run("names the count when several were listed", func(t *testing.T) { + hint := electionHint("oast.pro,oast.live,oast.site") + require.Contains(t, hint, "3 servers") + require.Contains(t, hint, "pass a single server with -file", + "the hint must say what to do, not just what happened") + }) + + t.Run("ignores blank entries and whitespace", func(t *testing.T) { + require.Empty(t, electionHint(" , ")) + require.Contains(t, electionHint("oast.pro, oast.live"), "2 servers") + }) + + t.Run("reads as a suffix to the failure sentence", func(t *testing.T) { + hint := electionHint("a.example,b.example") + require.True(t, strings.HasPrefix(hint, " ("), "must append cleanly after the message") + require.True(t, strings.HasSuffix(hint, ")")) + }) +} + +// -psf is a machine interface: one payload hostname per line, exactly -n of them. +// A consumer substitutes each line into a payload template, so a line carrying a +// full URL produces nonsense, and a missing trailing newline costs it the last +// record. +func TestWriteLines(t *testing.T) { + t.Run("one record per line, newline terminated", func(t *testing.T) { + path := filepath.Join(t.TempDir(), "payloads.txt") + payloads := []string{ + "c6rj61aciaeutn2ae680ti6cc3rxeenc3.oast.pro", + "c6rj61aciaeutn2ae680xk4tqy8pqhwmi.oast.pro", + } + require.NoError(t, writeLines(path, payloads)) + + raw, err := os.ReadFile(path) + require.NoError(t, err) + require.Equal(t, strings.Join(payloads, "\n")+"\n", string(raw)) + + // What "while read" and "wc -l" see, which is the point of the terminator. + require.Equal(t, len(payloads), strings.Count(string(raw), "\n")) + require.Equal(t, payloads, strings.Split(strings.TrimSuffix(string(raw), "\n"), "\n")) + }) + + t.Run("no record is a valid empty file", func(t *testing.T) { + path := filepath.Join(t.TempDir(), "empty.txt") + require.NoError(t, writeLines(path, nil)) + raw, err := os.ReadFile(path) + require.NoError(t, err) + require.Empty(t, raw, "an empty list must not leave a stray newline") + }) + + t.Run("hosted file URLs stay in their own file", func(t *testing.T) { + dir := t.TempDir() + payloads := []string{"c6rj61aciaeutn2ae680ti6cc3rxeenc3.oast.pro"} + fileURLs := []string{ + "https://c6rj61aciaeutn2ae680xk4tqy8pqhwmi.oast.pro/f/evil.dtd", + "ftp://c6rj61aciaeutn2ae680xk4tqy8pqhwmi.oast.pro/.interactsh-user-uploads/c6rj61aciaeutn2ae680/evil.dtd", + } + payloadFile := filepath.Join(dir, "payloads.txt") + urlFile := filepath.Join(dir, "files.txt") + require.NoError(t, writeLines(payloadFile, payloads)) + require.NoError(t, writeLines(urlFile, fileURLs)) + + gotPayloads, err := os.ReadFile(payloadFile) + require.NoError(t, err) + require.NotContains(t, string(gotPayloads), "://", + "a payload file line must be a hostname, never a URL") + + gotURLs, err := os.ReadFile(urlFile) + require.NoError(t, err) + require.Equal(t, strings.Join(fileURLs, "\n")+"\n", string(gotURLs)) + }) +} diff --git a/cmd/interactsh-server/main.go b/cmd/interactsh-server/main.go index 9e566732..eb644d7b 100644 --- a/cmd/interactsh-server/main.go +++ b/cmd/interactsh-server/main.go @@ -19,6 +19,7 @@ import ( _ "net/http/pprof" + units "github.com/docker/go-units" "github.com/projectdiscovery/goflags" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/gologger/levels" @@ -104,6 +105,15 @@ func main() { flagSet.StringVar(&cliOptions.FTPDirectory, "ftp-dir", "", "ftp directory - temporary if not specified"), ) + flagSet.CreateGroup("upload", "Upload", + flagSet.BoolVar(&cliOptions.Upload, "upload", false, "enable client file upload and hosting - self-hosted servers only (authenticated)"), + flagSet.StringVarP(&cliOptions.UploadDirectory, "upload-directory", "ud", "", "directory to host uploaded files from - temporary if not specified; interactsh creates and prunes .interactsh-user-uploads inside it"), + flagSet.SizeVarP(&cliOptions.UploadMaxFileSize, "upload-max-file-size", "umfs", "1mb", "maximum size of a single uploaded file"), + flagSet.IntVarP(&cliOptions.UploadMaxFiles, "upload-max-files", "umf", 5, "maximum number of uploaded files per session"), + flagSet.SizeVarP(&cliOptions.UploadMaxTotalSize, "upload-max-total-size", "umts", "1gb", "maximum total size of all uploaded files on the server"), + flagSet.DurationVarP(&cliOptions.UploadTTL, "upload-ttl", "ut", 24*time.Hour, "maximum lifetime of uploaded files"), + ) + flagSet.CreateGroup("debug", "Debug", flagSet.BoolVar(&cliOptions.Version, "version", false, "show version of the project"), flagSet.BoolVar(&cliOptions.Debug, "debug", false, "start interactsh server in debug mode"), @@ -209,7 +219,7 @@ func main() { } // Requires auth if token is specified or enables it automatically for responder and smb options - if serverOptions.Token != "" || cliOptions.Responder || cliOptions.Smb || cliOptions.Ftp || cliOptions.LdapWithFullLogger { + if serverOptions.Token != "" || cliOptions.Responder || cliOptions.Smb || cliOptions.Ftp || cliOptions.LdapWithFullLogger || cliOptions.Upload { serverOptions.Auth = true } @@ -269,6 +279,54 @@ func main() { atomic.AddInt64(&serverOptions.Stats.Sessions, -1) } + // The upload store must exist before the HTTP and FTP servers are built, + // since both serve from its root. + var uploadStore *server.UploadStore + if cliOptions.Upload { + // Hosted bytes live on this instance's local filesystem and the + // capacity quota is an in-process counter, so file hosting cannot be + // combined with a storage backend shared between instances: peers + // would advertise files they do not have. + if cliOptions.RedisURL != "" { + gologger.Fatal().Msgf("-upload cannot be used with -redis-url: hosted files are stored on a single instance's local filesystem\n") + } + var err error + if uploadStore, err = server.NewUploadStore(serverOptions); err != nil { + gologger.Fatal().Msgf("could not create upload store: %s\n", err) + } + serverOptions.UploadStore = uploadStore + + // Sharing the root is what lets the existing FTP file driver serve + // uploads. If the operator pinned both to different places, say so and + // stop advertising FTP, rather than printing ftp:// payload URLs that + // resolve to nothing. + switch { + case serverOptions.FTPDirectory == "": + serverOptions.FTPDirectory = uploadStore.Root() + serverOptions.FTPServesUploads = true + default: + // Compared as resolved paths, not as the operator typed them: + // "./uploads", "/abs/uploads/" and a symlink to the same place are + // one directory, and warning about a working configuration teaches + // the operator to ignore the warning that matters. + shared, err := sameDirectory(serverOptions.FTPDirectory, uploadStore.Root()) + if err != nil { + gologger.Fatal().Msgf("could not compare ftp and upload directories: %s\n", err) + } + serverOptions.FTPServesUploads = shared + if !shared && cliOptions.Ftp { + gologger.Error().Msgf("ftp directory %s is not the upload directory %s, so uploaded files will not be served over FTP; ftp:// URLs will not be offered to clients\n", + serverOptions.FTPDirectory, uploadStore.Root()) + } + } + + // Deleting a session's files is driven by the correlation-id leaving + // the cache, whatever the reason. + storeOptions.OnEviction = func(correlationID string, _ *storage.CorrelationData) { + uploadStore.RemoveSession(correlationID) + } + } + var err error switch { case cliOptions.RedisURL != "": @@ -294,6 +352,12 @@ func main() { serverOptions.Storage = store + if uploadStore != nil { + uploadStore.Start() + gologger.Info().Msgf("Uploads enabled, hosting from %s (max %d files of %s each)\n", + uploadStore.Root(), cliOptions.UploadMaxFiles, units.BytesSize(float64(cliOptions.UploadMaxFileSize))) + } + if serverOptions.Auth { _ = serverOptions.Storage.SetID(serverOptions.Token) } @@ -503,9 +567,16 @@ func main() { c := make(chan os.Signal, 1) signal.Notify(c, os.Interrupt) for range c { + // Closed first: cache.Close blocks until every removal callback has run, + // so all session deletions are queued by the time we drain them below. if err := store.Close(); err != nil { gologger.Warning().Msgf("Couldn't close the storage: %s\n", err) } + if uploadStore != nil { + if err := uploadStore.Close(); err != nil { + gologger.Warning().Msgf("Couldn't close the upload store: %s\n", err) + } + } if pprofServer != nil { if err := pprofServer.Close(); err != nil { gologger.Warning().Msgf("Couldn't close the pprof server: %s\n", err) @@ -535,3 +606,36 @@ func getPublicIP() (string, error) { return externalIP, errors.New("couldn't find an interface configured with external ip") } + +// sameDirectory reports whether two paths name the same directory. Both are made +// absolute and symlink-resolved first, so that the FTP root and the upload root +// are compared as directories rather than as the strings the operator typed. +// +// A path that does not exist yet is compared in its cleaned absolute form: +// EvalSymlinks fails on a missing path, and that is not an error worth refusing +// to start over. +func sameDirectory(a, b string) (bool, error) { + resolve := func(p string) (string, error) { + abs, err := filepath.Abs(p) + if err != nil { + return "", err + } + resolved, err := filepath.EvalSymlinks(abs) + if err != nil { + if os.IsNotExist(err) { + return filepath.Clean(abs), nil + } + return "", err + } + return resolved, nil + } + ra, err := resolve(a) + if err != nil { + return false, err + } + rb, err := resolve(b) + if err != nil { + return false, err + } + return ra == rb, nil +} diff --git a/cmd/interactsh-server/main_test.go b/cmd/interactsh-server/main_test.go new file mode 100644 index 00000000..3746405d --- /dev/null +++ b/cmd/interactsh-server/main_test.go @@ -0,0 +1,66 @@ +package main + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +// The FTP and upload roots decide whether hosted files are reachable over ftp://, +// so they have to be compared as directories rather than as the strings the +// operator typed: warning about a working configuration teaches them to ignore +// the warning that matters. +func TestSameDirectory(t *testing.T) { + base := t.TempDir() + shared := filepath.Join(base, "shared") + require.NoError(t, os.Mkdir(shared, 0o700)) + other := filepath.Join(base, "other") + require.NoError(t, os.Mkdir(other, 0o700)) + + link := filepath.Join(base, "link") + require.NoError(t, os.Symlink(shared, link)) + + wd, err := os.Getwd() + require.NoError(t, err) + t.Cleanup(func() { _ = os.Chdir(wd) }) + require.NoError(t, os.Chdir(base)) + + t.Run("same directory in different spellings", func(t *testing.T) { + for _, pair := range [][2]string{ + {shared, shared}, + {shared, shared + string(filepath.Separator)}, + {shared, filepath.Join(base, ".", "shared")}, + {shared, filepath.Join(base, "other", "..", "shared")}, + {"./shared", "shared"}, + {"./shared", shared}, + {link, shared}, + } { + got, err := sameDirectory(pair[0], pair[1]) + require.NoError(t, err) + require.True(t, got, "%q and %q are the same directory", pair[0], pair[1]) + } + }) + + t.Run("genuinely different directories", func(t *testing.T) { + for _, pair := range [][2]string{ + {shared, other}, + {"./shared", "./other"}, + {shared, filepath.Join(base, "absent")}, + } { + got, err := sameDirectory(pair[0], pair[1]) + require.NoError(t, err) + require.False(t, got, "%q and %q are different directories", pair[0], pair[1]) + } + }) + + // EvalSymlinks fails on a path that does not exist, which is not a reason to + // refuse to start: -ftp-dir may legitimately not exist yet. + t.Run("missing paths compare by cleaned absolute form", func(t *testing.T) { + missing := filepath.Join(base, "not-created-yet") + got, err := sameDirectory(missing, missing+string(filepath.Separator)) + require.NoError(t, err) + require.True(t, got) + }) +} diff --git a/examples/client.go b/examples/client.go index 6f261e9f..e64be5d9 100644 --- a/examples/client.go +++ b/examples/client.go @@ -1,8 +1,10 @@ package main import ( + "errors" "fmt" "net/http" + "os" "time" "github.com/projectdiscovery/interactsh/pkg/client" @@ -10,28 +12,30 @@ import ( ) func main() { - client, err := client.New(client.DefaultOptions) + // Named c rather than client so the package stays reachable for its + // exported errors below. + c, err := client.New(client.DefaultOptions) if err != nil { panic(err) } defer func() { - if err := client.Close(); err != nil { + if err := c.Close(); err != nil { panic(err) } }() - if err := client.StartPolling(time.Duration(1*time.Second), func(interaction *server.Interaction) { + if err := c.StartPolling(time.Duration(1*time.Second), func(interaction *server.Interaction) { fmt.Printf("Got Interaction: %v => %v\n", interaction.Protocol, interaction.FullId) }); err != nil { panic(err) } defer func() { - if err := client.StopPolling(); err != nil { + if err := c.StopPolling(); err != nil { panic(err) } }() - URL := client.URL() + URL := c.URL() resp, err := http.Get("https://" + URL) if err != nil { @@ -42,5 +46,87 @@ func main() { } fmt.Printf("Got URL: %v => %v\n", URL, resp) + + // Second stage: host a file against this session, if the server offers it. + hostFile(c, URL) + time.Sleep(1 * time.Second) } + +// hostFile uploads a file and fetches it back, which is how a second-stage +// payload is verified: the target retrieves the hosted file, and that retrieval +// arrives as an interaction of its own. +// +// Hosting is optional, so a caller has to handle its absence. The server +// advertises it at registration, and the public oast.* servers deliberately do +// not offer it, so this returns quietly rather than failing. +func hostFile(c *client.Client, payloadHost string) { + caps := c.Capabilities() + switch { + case caps == nil && c.CapabilitiesKnown(): + // The server answered and said nothing about capabilities, so it + // predates file hosting. + fmt.Println("Hosting: server predates file hosting, skipping") + return + case caps != nil && !caps.Upload: + fmt.Println("Hosting: server does not offer file hosting, skipping") + return + } + // caps == nil with nothing known -- a resumed session, say -- falls through: + // the only way to find out is to ask, and UploadFiles reports what it learns. + + path, err := writeTempFile(``) + if err != nil { + fmt.Printf("Hosting: %v\n", err) + return + } + defer func() { _ = os.Remove(path) }() + + files, err := c.UploadFiles([]string{path}) + if err != nil { + // Distinguishable so a caller can tell "cannot host" from "the request + // failed", and act on the right one. + switch { + case errors.Is(err, client.ErrUploadNotAdvertised): + fmt.Println("Hosting: server predates file hosting, skipping") + case errors.Is(err, client.ErrUploadUnsupported): + fmt.Println("Hosting: server does not offer file hosting, skipping") + default: + fmt.Printf("Hosting: upload failed: %v\n", err) + } + return + } + + for _, file := range files { + // Any host from URL() works: the server reads only its correlation ID. + fileURL := c.FileURL(payloadHost, file) + fmt.Printf("Hosting %s (%d bytes, sha256 %s) => %s\n", file.Name, file.Size, file.SHA256, fileURL) + if caps != nil && caps.FTP { + fmt.Printf("Hosting %s over FTP => %s\n", file.Name, c.FTPFileURL(payloadHost, file)) + } + + // Stand in for the target fetching it; the fetch is recorded as an + // interaction against this session and arrives in the poll callback. + resp, err := http.Get(fileURL) + if err != nil { + fmt.Printf("Hosting: could not fetch %s: %v\n", fileURL, err) + continue + } + if err := resp.Body.Close(); err != nil { + panic(err) + } + fmt.Printf("Fetched %s => %v\n", fileURL, resp.Status) + } +} + +func writeTempFile(content string) (string, error) { + f, err := os.CreateTemp("", "interactsh-example-*.dtd") + if err != nil { + return "", err + } + defer func() { _ = f.Close() }() + if _, err := f.WriteString(content); err != nil { + return "", err + } + return f.Name(), nil +} diff --git a/pkg/client/client.go b/pkg/client/client.go index bea9caa5..07084abe 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -67,6 +67,16 @@ type Client struct { token string correlationIdLength int CorrelationIdNonceLength int + // capabilitiesKnown records that a registration response was received and + // parsed, which is what makes the absence of a capabilities block meaningful. + // A resumed session whose re-registration was refused because the session is + // still alive never learns what the server offers, and "unknown" must not be + // read as "the server offers nothing". + capabilitiesKnown atomic.Bool + // capabilities holds the *server.Capabilities advertised at registration. + // Written from performRegistration, which the keep-alive goroutine also + // calls, hence atomic.Value rather than a bare field. + capabilities atomic.Value } // Options contains configuration options for interactsh client @@ -632,16 +642,23 @@ func (c *Client) performRegistration(serverURL string, payload []byte) error { data, _ := io.ReadAll(resp.Body) return fmt.Errorf("could not register to server: %s", string(data)) } - response := make(map[string]interface{}) - if err := json.NewDecoder(resp.Body).Decode(&response); err != nil { + response := &server.RegisterResponse{} + if err := json.NewDecoder(resp.Body).Decode(response); err != nil { return errkit.Wrap(err, "could not register to server") } - message, ok := response["message"] - if !ok { + if response.Message == "" { return errors.New("could not get register response") } - if message.(string) != "registration successful" { - return fmt.Errorf("could not get register response: %s", message.(string)) + if response.Message != "registration successful" { + return fmt.Errorf("could not get register response: %s", response.Message) + } + + // A successful registration is authoritative about what the server offers, + // including the absence of a capabilities block, which means the server + // predates them. + c.capabilitiesKnown.Store(true) + if response.Capabilities != nil { + c.capabilities.Store(response.Capabilities) } c.State.Store(Idle) @@ -649,6 +666,18 @@ func (c *Client) performRegistration(serverURL string, payload []byte) error { return nil } +// CapabilitiesKnown reports whether a registration response has been received, +// which is what distinguishes a server that advertised no capabilities from a +// resumed session that never got to ask. +func (c *Client) CapabilitiesKnown() bool { return c.capabilitiesKnown.Load() } + +// Capabilities returns the optional features advertised by the server at +// registration, or nil if the server did not advertise any. +func (c *Client) Capabilities() *server.Capabilities { + caps, _ := c.capabilities.Load().(*server.Capabilities) + return caps +} + // URL returns a new URL that can be used for external interaction requests. func (c *Client) URL() string { if c.State.Load() == Closed { diff --git a/pkg/client/upload.go b/pkg/client/upload.go new file mode 100644 index 00000000..39687543 --- /dev/null +++ b/pkg/client/upload.go @@ -0,0 +1,263 @@ +package client + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "os" + "path/filepath" + "regexp" + "strings" + + "github.com/projectdiscovery/interactsh/pkg/server" + "github.com/projectdiscovery/retryablehttp-go" + "github.com/projectdiscovery/utils/errkit" +) + +// ErrUploadUnsupported is returned when the interactsh server does not offer +// file hosting, either because it was started without -upload or because it +// predates the feature. +// +// Declared with errors.New rather than errkit.New so that errors.Is stays exact: +// errkit compares errors by message, which makes a sentinel and anything whose +// message contains it match in both directions. +var ErrUploadUnsupported = errors.New("interactsh server does not support file upload") + +// ErrUploadNotAdvertised is returned when the server advertised no capabilities +// at all, which means it predates file hosting. The remedy is to upgrade the +// server rather than to pass it a flag, so it is distinguishable from a server +// that advertised uploads as switched off. +// +// It wraps ErrUploadUnsupported, so a caller that only asks "can this server +// host files?" needs no change. +var ErrUploadNotAdvertised = fmt.Errorf("%w: server did not advertise file hosting capabilities", ErrUploadUnsupported) + +// defaultMaxUploadFileSize bounds a local file when the server has not told us +// its limit, so a mistyped -file cannot try to push a huge file over the wire. +const defaultMaxUploadFileSize = 1 << 20 + +// uploadNameRe mirrors the server's allowlist, so an unusable name is rejected +// locally with a clear message instead of as a 400 from the server. +var uploadNameRe = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$`) + +// UploadedFile describes a file hosted by the interactsh server. +type UploadedFile struct { + Name string `json:"name"` + Size int64 `json:"size"` + SHA256 string `json:"sha256"` + HTTPPath string `json:"http-path"` + FTPPath string `json:"ftp-path"` +} + +// UploadFiles uploads local files to the interactsh server this client +// registered with, to be hosted against its correlation ID. +// +// Only the registered server is targeted. A Client holds a single correlation +// ID, registered with whichever server answered first, so the other servers in +// -s have never seen it and would reject the upload. Posting blindly to them +// would also be actively harmful: a server without -upload has no route for the +// request, so it falls through to the catch-all handler that records whole +// requests as interactions, and the file would be stored there anyway. +func (c *Client) UploadFiles(paths []string) ([]UploadedFile, error) { + c.busy.RLock() + defer c.busy.RUnlock() + + if c.State.Load() == Closed { + return nil, errkit.New("client is closed") + } + if c.serverURL == nil { + return nil, errkit.New("client is not registered with any server") + } + + // Fail closed rather than send a request that cannot succeed, but only where + // the answer is known. A registration that produced no capabilities block is + // authoritative -- that server predates the feature, and its catch-all + // answers 200 with HTML, which would otherwise surface as an opaque JSON + // decode error. A resumed session whose re-registration was refused because + // the session is still alive knows nothing either way, so it must attempt the + // upload and let the response speak. + switch caps := c.Capabilities(); { + case caps != nil && !caps.Upload: + return nil, ErrUploadUnsupported + case caps == nil && c.CapabilitiesKnown(): + return nil, ErrUploadNotAdvertised + } + + // Uploads carry the file and the secret key, so they must never traverse + // the plaintext fallback that registration is permitted to use. + if c.serverURL.Scheme != "https" && !isLoopbackURL(c.serverURL.Host) { + return nil, errkit.New("refusing to upload over plaintext http to " + c.serverURL.Host + + "; use an https server url") + } + + request, err := c.buildUploadRequest(paths) + if err != nil { + return nil, err + } + + payload, err := json.Marshal(request) + if err != nil { + return nil, errkit.Wrap(err, "could not encode upload request") + } + + ctx := context.WithValue(context.Background(), retryablehttp.RETRY_MAX, 0) + req, err := retryablehttp.NewRequestWithContext(ctx, http.MethodPost, c.serverURL.String()+"/upload", bytes.NewReader(payload)) + if err != nil { + return nil, errkit.Wrap(err, "could not create upload request") + } + req.ContentLength = int64(len(payload)) + req.Header.Set("Content-Type", "application/json") + if c.token != "" { + req.Header.Add("Authorization", c.token) + } + + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, errkit.Wrap(err, "could not make upload request") + } + defer func() { + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + }() + + switch resp.StatusCode { + case http.StatusOK: + case http.StatusNotImplemented: + // Advertised uploads and then refused them, which a mismatched + // deployment behind a load balancer can produce. The capability check + // above catches the ordinary case before a request is made. + return nil, ErrUploadUnsupported + case http.StatusUnauthorized: + return nil, errkit.New("invalid token provided for interactsh server") + default: + // 404 and 405 are deliberately not read as "uploads unsupported". The + // server answers 404 for an unknown correlation id, which is a session + // problem needing a re-register rather than a server flag, and a server + // predating /upload answers 200 from its catch-all rather than either + // status -- so mapping them here misdiagnosed the one case it caught and + // never caught the one it was aimed at. Reporting the status and body + // says something true whatever the cause. + body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + return nil, fmt.Errorf("could not upload files (%s): %s", resp.Status, strings.TrimSpace(string(body))) + } + + response := &server.UploadResponse{} + if err := json.NewDecoder(resp.Body).Decode(response); err != nil { + return nil, errkit.Wrap(err, "could not decode upload response") + } + + files := make([]UploadedFile, 0, len(response.Files)) + for _, f := range response.Files { + files = append(files, UploadedFile{ + Name: f.Name, + Size: f.Size, + SHA256: f.SHA256, + HTTPPath: f.HTTPPath, + FTPPath: f.FTPPath, + }) + } + return files, nil +} + +// buildUploadRequest reads and validates the local files before anything is +// sent, so a bad path or an oversize file fails immediately and clearly. +func (c *Client) buildUploadRequest(paths []string) (*server.UploadRequest, error) { + maxFileSize, maxFiles := int64(defaultMaxUploadFileSize), 0 + if caps := c.Capabilities(); caps != nil { + if caps.UploadMaxFileSize > 0 { + maxFileSize = caps.UploadMaxFileSize + } + maxFiles = caps.UploadMaxFiles + } + if maxFiles > 0 && len(paths) > maxFiles { + return nil, fmt.Errorf("%d files requested but the server accepts at most %d", len(paths), maxFiles) + } + + request := &server.UploadRequest{CorrelationID: c.correlationID, SecretKey: c.secretKey} + seen := make(map[string]string, len(paths)) + + for _, p := range paths { + info, err := os.Stat(p) + if err != nil { + return nil, errkit.Wrap(err, "could not read file "+p) + } + if !info.Mode().IsRegular() { + return nil, fmt.Errorf("%s is not a regular file", p) + } + if info.Size() == 0 { + return nil, fmt.Errorf("%s is empty", p) + } + if info.Size() > maxFileSize { + return nil, fmt.Errorf("%s is %d bytes, the server accepts at most %d", p, info.Size(), maxFileSize) + } + + name := filepath.Base(p) + if !uploadNameRe.MatchString(name) || name == "." || name == ".." { + return nil, fmt.Errorf("%s has a name the server will not accept; "+ + "use only letters, digits, dot, dash and underscore", p) + } + if previous, dup := seen[name]; dup { + return nil, fmt.Errorf("%s and %s would both be hosted as %q", previous, p, name) + } + seen[name] = p + + data, err := os.ReadFile(p) + if err != nil { + return nil, errkit.Wrap(err, "could not read file "+p) + } + request.Files = append(request.Files, server.UploadFileRequest{ + Name: name, + Data: base64.StdEncoding.EncodeToString(data), + }) + } + + if len(request.Files) == 0 { + return nil, errkit.New("no files to upload") + } + return request, nil +} + +// FileURL returns the URL a target should fetch to retrieve a hosted file. +// payloadHost is any host produced by URL(); only its correlation ID prefix is +// significant to the server, so a single call to URL() serves every file. +func (c *Client) FileURL(payloadHost string, file UploadedFile) string { + scheme := "https" + if c.serverURL != nil && c.serverURL.Scheme != "" { + scheme = c.serverURL.Scheme + } + return scheme + "://" + payloadHost + file.HTTPPath +} + +// FTPFileURL returns the ftp:// URL for a hosted file. FTP has no host-based +// routing, so the correlation ID travels in the path instead. +// +// Any port on payloadHost is dropped: it belongs to the server's HTTP listener, +// which says nothing about where the FTP listener is bound, so carrying it over +// would produce a URL that cannot connect. +func (c *Client) FTPFileURL(payloadHost string, file UploadedFile) string { + host := payloadHost + if h, _, err := net.SplitHostPort(host); err == nil { + host = h + } + return "ftp://" + host + file.FTPPath +} + +// isLoopbackURL reports whether a host refers to the local machine, where a +// plaintext upload is not exposed to the network. +func isLoopbackURL(host string) bool { + name := host + if h, _, err := net.SplitHostPort(host); err == nil { + name = h + } + switch strings.ToLower(name) { + case "localhost", "127.0.0.1", "::1", "[::1]": + return true + } + return false +} diff --git a/pkg/client/upload_test.go b/pkg/client/upload_test.go new file mode 100644 index 00000000..982bf84b --- /dev/null +++ b/pkg/client/upload_test.go @@ -0,0 +1,335 @@ +package client + +import ( + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "sync/atomic" + "testing" + + "github.com/projectdiscovery/interactsh/pkg/server" + "github.com/projectdiscovery/retryablehttp-go" + "github.com/stretchr/testify/require" +) + +// newUploadClient returns a client pointed at handler, already "registered". +func newUploadClient(t *testing.T, handler http.HandlerFunc, caps *server.Capabilities) *Client { + t.Helper() + + ts := httptest.NewServer(handler) + t.Cleanup(ts.Close) + + parsed, err := url.Parse(ts.URL) + require.NoError(t, err) + + c := &Client{ + correlationID: "c6rj61aciaeutn2ae680", + secretKey: "6a1b0e5c-3f2d-4a7b-8c9d-0e1f2a3b4c5d", + serverURL: parsed, + httpClient: retryablehttp.NewClient(retryablehttp.DefaultOptionsSingle), + correlationIdLength: 20, + CorrelationIdNonceLength: 13, + } + c.State.Store(Idle) + if caps != nil { + c.capabilities.Store(caps) + } + // A registration happened, so whatever it said -- including nothing -- is + // authoritative. newResumedUploadClient covers the other case. + c.capabilitiesKnown.Store(true) + return c +} + +// newResumedUploadClient models a session resumed from -sf whose re-registration +// was refused because the session is still alive: no capabilities were ever +// received, so nothing is known about what the server offers. +func newResumedUploadClient(t *testing.T, handler http.HandlerFunc) *Client { + t.Helper() + c := newUploadClient(t, handler, nil) + c.capabilitiesKnown.Store(false) + return c +} + +func writeTempFile(t *testing.T, name string, content []byte) string { + t.Helper() + p := filepath.Join(t.TempDir(), name) + require.NoError(t, os.WriteFile(p, content, 0o600)) + return p +} + +// errkit compares errors by message, so building ErrUploadNotAdvertised on an +// errkit sentinel made errors.Is match in both directions -- and the CLI checks the +// specific error first, so a server with -upload merely switched off was told to +// upgrade. The direction has to stay one-way. +func TestUploadSentinelsAreDirectional(t *testing.T) { + require.True(t, errors.Is(ErrUploadNotAdvertised, ErrUploadUnsupported), + "a server that advertised nothing cannot host files either") + require.False(t, errors.Is(ErrUploadUnsupported, ErrUploadNotAdvertised), + "a server that answered 501 is not a server that failed to advertise") +} + +func TestUploadFiles(t *testing.T) { + caps := &server.Capabilities{Upload: true, UploadMaxFileSize: 1024, UploadMaxFiles: 5, FTP: true} + + t.Run("posts the expected request and decodes the response", func(t *testing.T) { + var ( + gotAuth string + gotPath string + gotMethod string + gotBody server.UploadRequest + ) + c := newUploadClient(t, func(w http.ResponseWriter, r *http.Request) { + gotAuth, gotPath, gotMethod = r.Header.Get("Authorization"), r.URL.Path, r.Method + require.NoError(t, json.NewDecoder(r.Body).Decode(&gotBody)) + + _ = json.NewEncoder(w).Encode(&server.UploadResponse{ + Message: "upload successful", + Files: []server.UploadedFileResponse{{ + Name: "evil.dtd", Size: 7, SHA256: "abc", + HTTPPath: "/f/evil.dtd", FTPPath: "/.interactsh-user-uploads/c6rj61aciaeutn2ae680/evil.dtd", + }}, + }) + }, caps) + c.token = "sekrit" + + files, err := c.UploadFiles([]string{writeTempFile(t, "evil.dtd", []byte("payload"))}) + require.NoError(t, err) + + require.Equal(t, http.MethodPost, gotMethod) + require.Equal(t, "/upload", gotPath) + require.Equal(t, "sekrit", gotAuth) + require.Equal(t, "c6rj61aciaeutn2ae680", gotBody.CorrelationID) + require.Equal(t, c.secretKey, gotBody.SecretKey) + require.Len(t, gotBody.Files, 1) + require.Equal(t, "evil.dtd", gotBody.Files[0].Name) + require.Equal(t, "cGF5bG9hZA==", gotBody.Files[0].Data, "content should be base64 encoded") + + require.Len(t, files, 1) + require.Equal(t, "/f/evil.dtd", files[0].HTTPPath) + }) + + // Capabilities are advertised here, so the request is actually made: a server + // that says it can host files and then refuses is the case 501 is left for. + t.Run("501 reports unsupported", func(t *testing.T) { + var called atomic.Bool + c := newUploadClient(t, func(w http.ResponseWriter, r *http.Request) { + called.Store(true) + w.WriteHeader(http.StatusNotImplemented) + }, caps) + + _, err := c.UploadFiles([]string{writeTempFile(t, "a.dtd", []byte("x"))}) + require.ErrorIs(t, err, ErrUploadUnsupported) + require.True(t, called.Load(), "the 501 path is only reachable by sending the request") + }) + + // 404 is what the server answers for an unknown correlation id -- a session + // problem needing a re-register, not a server missing a flag. Reporting it as + // "does not accept file uploads" sent the operator after the wrong remedy. + t.Run("404 surfaces the server's reason rather than claiming unsupported", func(t *testing.T) { + c := newUploadClient(t, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"error":"unknown correlation-id"}`)) + }, caps) + + _, err := c.UploadFiles([]string{writeTempFile(t, "a.dtd", []byte("x"))}) + require.Error(t, err) + require.Contains(t, err.Error(), "unknown correlation-id") + require.Contains(t, err.Error(), "404") + require.False(t, errors.Is(err, ErrUploadUnsupported), + "an unknown session must not be reported as a server without -upload") + }) + + t.Run("405 is surfaced rather than claiming unsupported", func(t *testing.T) { + c := newUploadClient(t, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusMethodNotAllowed) + }, caps) + + _, err := c.UploadFiles([]string{writeTempFile(t, "a.dtd", []byte("x"))}) + require.Error(t, err) + require.False(t, errors.Is(err, ErrUploadUnsupported)) + }) + + // A server advertising no capabilities predates file hosting. Its catch-all + // answers 200 with HTML, so without this the failure was an opaque + // "invalid character '<'" decode error. + t.Run("a server that advertised nothing is refused before any request", func(t *testing.T) { + var called atomic.Bool + c := newUploadClient(t, func(w http.ResponseWriter, r *http.Request) { + called.Store(true) + w.Header().Set("Content-Type", "text/html") + _, _ = w.Write([]byte("")) + }, nil) + + _, err := c.UploadFiles([]string{writeTempFile(t, "a.dtd", []byte("x"))}) + require.ErrorIs(t, err, ErrUploadNotAdvertised) + require.ErrorIs(t, err, ErrUploadUnsupported, + "callers asking only whether hosting is possible need no change") + require.False(t, called.Load(), "no point asking a server that cannot answer") + require.NotContains(t, err.Error(), "invalid character") + }) + + // Regression: a resumed session knows nothing about the server's capabilities, + // and treating that as "the server offers nothing" made every -sf resume + // refuse to upload, blaming a server that hosts files perfectly well. + t.Run("a resumed session attempts the upload rather than assuming", func(t *testing.T) { + var called atomic.Bool + c := newResumedUploadClient(t, func(w http.ResponseWriter, r *http.Request) { + called.Store(true) + _ = json.NewEncoder(w).Encode(&server.UploadResponse{ + Message: "upload successful", + Files: []server.UploadedFileResponse{{ + Name: "a.dtd", Size: 1, SHA256: "abc", HTTPPath: "/f/a.dtd", + }}, + }) + }) + + files, err := c.UploadFiles([]string{writeTempFile(t, "a.dtd", []byte("x"))}) + require.NoError(t, err, "an unknown capability set must not be read as unsupported") + require.True(t, called.Load(), "the request has to be made to find out") + require.Len(t, files, 1) + }) + + t.Run("a resumed session still reports a genuine 501", func(t *testing.T) { + c := newResumedUploadClient(t, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotImplemented) + }) + _, err := c.UploadFiles([]string{writeTempFile(t, "a.dtd", []byte("x"))}) + require.ErrorIs(t, err, ErrUploadUnsupported) + require.False(t, errors.Is(err, ErrUploadNotAdvertised), + "the server answered, so this is not version skew") + }) + + t.Run("server error message is surfaced", func(t *testing.T) { + c := newUploadClient(t, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusRequestEntityTooLarge) + _, _ = w.Write([]byte(`{"error":"file is too big"}`)) + }, caps) + + _, err := c.UploadFiles([]string{writeTempFile(t, "a.dtd", []byte("x"))}) + require.Error(t, err) + require.Contains(t, err.Error(), "file is too big") + }) + + t.Run("advertised absence of upload fails without a request", func(t *testing.T) { + var called atomic.Bool + c := newUploadClient(t, func(w http.ResponseWriter, r *http.Request) { + called.Store(true) + }, &server.Capabilities{Upload: false}) + + _, err := c.UploadFiles([]string{writeTempFile(t, "a.dtd", []byte("x"))}) + require.ErrorIs(t, err, ErrUploadUnsupported) + require.False(t, called.Load(), "must not contact a server known not to support uploads") + }) + + t.Run("local validation happens before any request", func(t *testing.T) { + cases := []struct { + name string + paths func(t *testing.T) []string + want string + }{ + {"missing file", func(t *testing.T) []string { + return []string{filepath.Join(t.TempDir(), "absent.dtd")} + }, "could not read file"}, + {"directory", func(t *testing.T) []string { + return []string{t.TempDir()} + }, "not a regular file"}, + {"empty file", func(t *testing.T) []string { + return []string{writeTempFile(t, "empty.dtd", nil)} + }, "is empty"}, + {"oversize", func(t *testing.T) []string { + return []string{writeTempFile(t, "big.dtd", make([]byte, 4096))} + }, "accepts at most"}, + {"unusable name", func(t *testing.T) []string { + return []string{writeTempFile(t, ".hidden", []byte("x"))} + }, "will not accept"}, + {"too many files", func(t *testing.T) []string { + var paths []string + for i := 0; i < 6; i++ { + paths = append(paths, writeTempFile(t, "f.dtd", []byte("x"))) + } + return paths + }, "accepts at most"}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var called atomic.Bool + c := newUploadClient(t, func(w http.ResponseWriter, r *http.Request) { + called.Store(true) + }, caps) + + _, err := c.UploadFiles(tc.paths(t)) + require.Error(t, err) + require.Contains(t, err.Error(), tc.want) + require.False(t, called.Load(), "must fail before contacting the server") + }) + } + }) + + t.Run("duplicate basenames are rejected", func(t *testing.T) { + c := newUploadClient(t, func(w http.ResponseWriter, r *http.Request) {}, caps) + + a := writeTempFile(t, "evil.dtd", []byte("one")) + b := writeTempFile(t, "evil.dtd", []byte("two")) + _, err := c.UploadFiles([]string{a, b}) + require.Error(t, err) + require.Contains(t, err.Error(), "would both be hosted as") + }) + + t.Run("refuses plaintext http to a remote server", func(t *testing.T) { + c := newUploadClient(t, func(w http.ResponseWriter, r *http.Request) {}, caps) + c.serverURL = &url.URL{Scheme: "http", Host: "oast.example.com"} + + _, err := c.UploadFiles([]string{writeTempFile(t, "a.dtd", []byte("x"))}) + require.Error(t, err) + require.Contains(t, err.Error(), "refusing to upload over plaintext http", + "the file and the secret key must not go over the wire in clear") + }) + + t.Run("closed client", func(t *testing.T) { + c := newUploadClient(t, func(w http.ResponseWriter, r *http.Request) {}, caps) + c.State.Store(Closed) + + _, err := c.UploadFiles([]string{writeTempFile(t, "a.dtd", []byte("x"))}) + require.Error(t, err) + require.False(t, errors.Is(err, ErrUploadUnsupported)) + }) +} + +func TestFileURLComposition(t *testing.T) { + file := UploadedFile{Name: "evil.dtd", HTTPPath: "/f/evil.dtd", FTPPath: "/.interactsh-user-uploads/c6rj61aciaeutn2ae680/evil.dtd"} + // 20-char correlation id plus a 13-char nonce, as URL() composes it. + host := "c6rj61aciaeutn2ae680xk4tqy8pqhwmi.oast.test" + + t.Run("https server", func(t *testing.T) { + c := &Client{serverURL: &url.URL{Scheme: "https", Host: "oast.test"}} + require.Equal(t, "https://"+host+"/f/evil.dtd", c.FileURL(host, file)) + require.Equal(t, "ftp://"+host+"/.interactsh-user-uploads/c6rj61aciaeutn2ae680/evil.dtd", c.FTPFileURL(host, file)) + }) + + t.Run("http server", func(t *testing.T) { + c := &Client{serverURL: &url.URL{Scheme: "http", Host: "127.0.0.1:8080"}} + require.Equal(t, "http://"+host+"/f/evil.dtd", c.FileURL(host, file)) + }) + + // The payload host carries the HTTP listener's port, which tells us nothing + // about where FTP is bound, so it must not leak into the ftp:// URL. + t.Run("ftp url drops the http port", func(t *testing.T) { + c := &Client{serverURL: &url.URL{Scheme: "http", Host: "127.0.0.1:8080"}} + require.Equal(t, "ftp://"+host+"/.interactsh-user-uploads/c6rj61aciaeutn2ae680/evil.dtd", + c.FTPFileURL(host+":8080", file)) + }) +} + +func TestIsLoopbackURL(t *testing.T) { + for _, host := range []string{"localhost", "127.0.0.1", "127.0.0.1:8080", "localhost:8080", "[::1]:8080"} { + require.True(t, isLoopbackURL(host), "expected %q to be loopback", host) + } + for _, host := range []string{"oast.test", "example.com:8080", "10.0.0.1"} { + require.False(t, isLoopbackURL(host), "expected %q not to be loopback", host) + } +} diff --git a/pkg/options/client_options.go b/pkg/options/client_options.go index 835380ec..f8c7333b 100644 --- a/pkg/options/client_options.go +++ b/pkg/options/client_options.go @@ -9,6 +9,7 @@ import ( type CLIClientOptions struct { Match goflags.StringSlice Filter goflags.StringSlice + Files goflags.StringSlice Config string Version bool ServerURL string @@ -17,6 +18,7 @@ type CLIClientOptions struct { JSON bool StorePayload bool StorePayloadFile string + FileStoreFile string Verbose bool PollInterval int DNSOnly bool diff --git a/pkg/options/server_options.go b/pkg/options/server_options.go index 96352cb9..21b1fdad 100644 --- a/pkg/options/server_options.go +++ b/pkg/options/server_options.go @@ -2,6 +2,7 @@ package options import ( "net" + "time" "github.com/projectdiscovery/goflags" "github.com/projectdiscovery/gologger" @@ -55,15 +56,21 @@ type CLIServerOptions struct { // RedisURL, when set, switches the server to a Redis-backed storage // backend so multiple instances can share state behind a load balancer. // Disk storage flags are ignored when RedisURL is set. - RedisURL string - RedisKeyPrefix string - EnablePprof bool - EnableMetrics bool - Verbose bool - DisableUpdateCheck bool - NoVersionHeader bool - HeaderServer string - DefaultHTTPResponseFile string + RedisURL string + RedisKeyPrefix string + EnablePprof bool + EnableMetrics bool + Verbose bool + DisableUpdateCheck bool + NoVersionHeader bool + HeaderServer string + DefaultHTTPResponseFile string + Upload bool + UploadDirectory string + UploadMaxFileSize goflags.Size + UploadMaxFiles int + UploadMaxTotalSize goflags.Size + UploadTTL time.Duration } func (cliServerOptions *CLIServerOptions) AsServerOptions() *server.Options { @@ -119,6 +126,13 @@ func (cliServerOptions *CLIServerOptions) AsServerOptions() *server.Options { NoVersionHeader: cliServerOptions.NoVersionHeader, HeaderServer: cliServerOptions.HeaderServer, DefaultHTTPResponseFile: cliServerOptions.DefaultHTTPResponseFile, + Ftp: cliServerOptions.Ftp, + Upload: cliServerOptions.Upload, + UploadDirectory: cliServerOptions.UploadDirectory, + UploadMaxFileSize: int64(cliServerOptions.UploadMaxFileSize), + UploadMaxFiles: cliServerOptions.UploadMaxFiles, + UploadMaxTotalSize: int64(cliServerOptions.UploadMaxTotalSize), + UploadTTL: cliServerOptions.UploadTTL, } } diff --git a/pkg/server/ftp_server.go b/pkg/server/ftp_server.go index 79a68f59..860c24a0 100644 --- a/pkg/server/ftp_server.go +++ b/pkg/server/ftp_server.go @@ -5,11 +5,14 @@ import ( "fmt" "io" "os" + "path" "strings" "sync/atomic" "time" "encoding/json" + + "github.com/asaskevich/govalidator" "github.com/projectdiscovery/gologger" ftpserver "goftp.io/server/v2" "goftp.io/server/v2/driver/file" @@ -119,6 +122,18 @@ func (h *FTPServer) Close() { } func (h *FTPServer) recordInteraction(remoteAddress, data string) { + h.recordInteractionForPath(remoteAddress, data, "") +} + +// recordInteractionForPath records an FTP interaction, attributing it to the +// session that owns dstPath when the path points into a hosted-files directory. +// +// Interactions that cannot be attributed go, as before, to the shared token +// bucket, which pollHandler fans out to every authenticated client. That is +// fine for connection noise, but a fetch of a specific session's hosted file +// belongs to that session: otherwise it is reported to everyone and attributed +// to no one. +func (h *FTPServer) recordInteractionForPath(remoteAddress, data, dstPath string) { atomic.AddUint64(&h.options.Stats.Ftp, 1) if data == "" { @@ -130,15 +145,54 @@ func (h *FTPServer) recordInteraction(remoteAddress, data string) { RawRequest: data, Timestamp: time.Now(), } + + correlationID := h.correlationIDFromPath(dstPath) + if correlationID != "" { + interaction.UniqueID = correlationID + interaction.FullId = correlationID + } + dataBytes, err := json.Marshal(interaction) if err != nil { gologger.Warning().Msgf("Could not encode ftp interaction: %s\n", err) - } else { - gologger.Debug().Msgf("FTP Interaction: \n%s\n", string(dataBytes)) - if err := h.options.Storage.AddInteractionWithId(h.options.Token, dataBytes); err != nil { + return + } + gologger.Debug().Msgf("FTP Interaction: \n%s\n", string(dataBytes)) + + if correlationID != "" { + if err := h.options.Storage.AddInteraction(correlationID, dataBytes); err != nil { gologger.Warning().Msgf("Could not store ftp interaction: %s\n", err) } + return + } + if err := h.options.Storage.AddInteractionWithId(h.options.Token, dataBytes); err != nil { + gologger.Warning().Msgf("Could not store ftp interaction: %s\n", err) + } +} + +// correlationIDFromPath returns the correlation id owning an FTP path, or "" if +// the path does not name a live session with hosted files. +func (h *FTPServer) correlationIDFromPath(dstPath string) string { + uploadStorage := h.options.UploadStorage() + if dstPath == "" || h.options.UploadStore == nil || uploadStorage == nil { + return "" } + // Hosted files live at ///, so the + // correlation id is the second segment. Cleaning first means a traversal can + // only ever resolve to the session it actually points at. + rest, ok := strings.CutPrefix(ftpCleanPath(dstPath), "/"+uploadsDirName+"/") + if !ok { + return "" + } + segment, _, _ := strings.Cut(rest, "/") + segment = strings.ToLower(segment) + if len(segment) != h.options.CorrelationIdLength || !govalidator.IsAlphanumeric(segment) { + return "" + } + if _, ok := uploadStorage.ListUploads(segment); !ok { + return "" + } + return segment } func (h *FTPServer) Print(sessionID string, message interface{}) {} @@ -211,7 +265,7 @@ func (h *FTPServer) BeforeDownloadFile(ctx *ftpserver.Context, dstPath string) { b.WriteString(ctx.Param) b.WriteString("\n") b.WriteString("downloading file " + dstPath) - h.recordInteraction(ctx.Sess.RemoteAddr().String(), b.String()) + h.recordInteractionForPath(ctx.Sess.RemoteAddr().String(), b.String(), dstPath) } func (h *FTPServer) AfterUserLogin(ctx *ftpserver.Context, userName, password string, passMatched bool, err error) { var b strings.Builder @@ -247,7 +301,7 @@ func (h *FTPServer) AfterFileDownloaded(ctx *ftpserver.Context, dstPath string, b.WriteString(ctx.Param) b.WriteString("\n") b.WriteString("downloaded file " + dstPath) - h.recordInteraction(ctx.Sess.RemoteAddr().String(), b.String()) + h.recordInteractionForPath(ctx.Sess.RemoteAddr().String(), b.String(), dstPath) } func (h *FTPServer) AfterCurDirChanged(ctx *ftpserver.Context, oldCurDir, newCurDir string, err error) { var b strings.Builder @@ -295,10 +349,55 @@ func (n *NopDriver) Stat(c *ftpserver.Context, s string) (os.FileInfo, error) { return n.driver.Stat(c, s) } +// ListDir hides interactsh's own upload storage, and nothing else. +// +// NopAuth accepts any credentials, so an anonymous client must not be able to +// read off every correlation id that currently has uploaded files and then walk +// into each one. Two rules close that off: the uploads directory never lists its +// own contents, and it is filtered out of the root listing so that it cannot be +// discovered in the first place. A client that already knows its own correlation +// id can still list and RETR inside it. +// +// Only those two rules, deliberately: the root itself lists normally, because +// -ftp-dir is documented as serving the operator's own directory and a blanket +// refusal there would silently break that. func (n *NopDriver) ListDir(c *ftpserver.Context, s string, f func(os.FileInfo) error) error { + if isUploadsDir(s) { + return nil + } + if isFTPRoot(s) { + return n.driver.ListDir(c, s, func(info os.FileInfo) error { + if info.Name() == uploadsDirName { + return nil + } + return f(info) + }) + } return n.driver.ListDir(c, s, f) } +// isUploadsDir reports whether an FTP path refers to the uploads directory +// itself, which sits directly under the root. +func isUploadsDir(p string) bool { + return ftpCleanPath(p) == "/"+uploadsDirName +} + +// isFTPRoot reports whether an FTP path refers to the server root. +func isFTPRoot(p string) bool { + switch ftpCleanPath(p) { + case "/", ".", "": + return true + } + return false +} + +// ftpCleanPath resolves an FTP path to an absolute, traversal-free form. Every +// path decision in this file goes through it, so "/a/../b", "//b" and "/./b" can +// never be treated differently from "/b". +func ftpCleanPath(p string) string { + return path.Clean("/" + strings.TrimPrefix(p, "/")) +} + func (n *NopDriver) DeleteDir(c *ftpserver.Context, s string) error { return nil } diff --git a/pkg/server/ftp_upload_test.go b/pkg/server/ftp_upload_test.go new file mode 100644 index 00000000..59c563c3 --- /dev/null +++ b/pkg/server/ftp_upload_test.go @@ -0,0 +1,201 @@ +package server + +import ( + "encoding/json" + "net/http" + "os" + "testing" + "time" + + "github.com/stretchr/testify/require" + ftpserver "goftp.io/server/v2" +) + +func TestIsFTPRoot(t *testing.T) { + for _, p := range []string{"/", "", ".", "//", "/.", "/../.."} { + require.True(t, isFTPRoot(p), "expected %q to be treated as root", p) + } + for _, p := range []string{"/c6rj61aciaeutn2ae680", "/a/b", "/evil.dtd", "/dir/"} { + require.False(t, isFTPRoot(p), "expected %q not to be treated as root", p) + } +} + +// countingDriver records which paths reached the wrapped driver, and reports the +// entries it was configured with for each of them. +type countingDriver struct { + ftpserver.Driver + listed []string + entries []string +} + +func (d *countingDriver) ListDir(c *ftpserver.Context, s string, f func(os.FileInfo) error) error { + d.listed = append(d.listed, s) + for _, name := range d.entries { + if err := f(dirEntryInfo(name)); err != nil { + return err + } + } + return nil +} + +// dirEntryInfo is a minimal os.FileInfo standing in for a directory entry. +type dirEntryInfo string + +func (d dirEntryInfo) Name() string { return string(d) } +func (d dirEntryInfo) Size() int64 { return 0 } +func (d dirEntryInfo) Mode() os.FileMode { return os.ModeDir | 0o700 } +func (d dirEntryInfo) ModTime() time.Time { return time.Time{} } +func (d dirEntryInfo) IsDir() bool { return true } +func (d dirEntryInfo) Sys() interface{} { return nil } + +func TestIsUploadsDir(t *testing.T) { + for _, p := range []string{ + "/" + uploadsDirName, + "/" + uploadsDirName + "/", + "//" + uploadsDirName, + "/./" + uploadsDirName, + "/other/../" + uploadsDirName, + "/" + uploadsDirName + "/.", + } { + require.True(t, isUploadsDir(p), "expected %q to resolve to the uploads directory", p) + } + for _, p := range []string{ + "/", + "/" + uploadsDirName + "/c6rj61aciaeutn2ae680", + "/nested/" + uploadsDirName, + "/" + uploadsDirName + "-other", + } { + require.False(t, isUploadsDir(p), "expected %q not to resolve to the uploads directory", p) + } +} + +// NopAuth accepts any credentials, so an unauthenticated client must not be able +// to enumerate the correlation ids that currently have hosted files. +func TestNopDriverHidesUploadsDir(t *testing.T) { + t.Run("the uploads directory never enumerates", func(t *testing.T) { + for _, p := range []string{ + "/" + uploadsDirName, + "/" + uploadsDirName + "/", + "//" + uploadsDirName, + "/./" + uploadsDirName, + "/pub/../" + uploadsDirName, + } { + inner := &countingDriver{entries: []string{"c6rj61aciaeutn2ae680", "c6rj61aciaeutn2ae681"}} + driver := NewNopDriver(inner) + + var seen []string + require.NoError(t, driver.ListDir(nil, p, func(fi os.FileInfo) error { + seen = append(seen, fi.Name()) + return nil + })) + require.Empty(t, seen, "%q must not enumerate correlation ids", p) + require.Empty(t, inner.listed, "%q must not even reach the filesystem", p) + } + }) + + t.Run("the root lists operator content without the uploads directory", func(t *testing.T) { + inner := &countingDriver{entries: []string{"index.html", uploadsDirName, "assets"}} + driver := NewNopDriver(inner) + + var seen []string + require.NoError(t, driver.ListDir(nil, "/", func(fi os.FileInfo) error { + seen = append(seen, fi.Name()) + return nil + })) + require.Equal(t, []string{"index.html", "assets"}, seen, + "-ftp-dir is documented as serving the operator's directory, minus our own") + }) + + t.Run("a session directory still lists for a client that knows its id", func(t *testing.T) { + inner := &countingDriver{entries: []string{"evil.dtd"}} + driver := NewNopDriver(inner) + p := "/" + uploadsDirName + "/c6rj61aciaeutn2ae680" + + var seen []string + require.NoError(t, driver.ListDir(nil, p, func(fi os.FileInfo) error { + seen = append(seen, fi.Name()) + return nil + })) + require.Equal(t, []string{"evil.dtd"}, seen) + require.Equal(t, []string{p}, inner.listed) + }) +} + +func TestFTPDownloadCorrelation(t *testing.T) { + newFTP := func(t *testing.T) (*FTPServer, *HTTPServer, string, string) { + t.Helper() + h, id, secret := uploadTestServer(t, true) + h.options.Token = "shared-token" + require.NoError(t, h.options.Storage.SetID(h.options.Token)) + + resp := doUpload(t, h, uploadBody(t, id, secret, map[string][]byte{"evil.dtd": []byte("payload")})) + require.Equal(t, http.StatusOK, resp.StatusCode) + + return &FTPServer{options: h.options}, h, id, secret + } + + t.Run("download attributed to the owning session", func(t *testing.T) { + ftp, h, id, _ := newFTP(t) + + hosted := "/" + uploadsDirName + "/" + id + "/evil.dtd" + ftp.recordInteractionForPath("198.51.100.4:3333", "RETR "+hosted+"\ndownloaded file", hosted) + + item, err := h.options.Storage.GetCacheItem(id) + require.NoError(t, err) + require.Len(t, item.Data, 1, "the owning session should see its own file being fetched") + + record := &Interaction{} + require.NoError(t, json.Unmarshal([]byte(item.Data[0]), record)) + require.Equal(t, "ftp", record.Protocol) + require.Equal(t, id, record.UniqueID) + + // And it must not also land in the shared bucket, or every client sees it. + shared, err := h.options.Storage.GetCacheItem(h.options.Token) + require.NoError(t, err) + require.Empty(t, shared.Data, "an attributed download must not be duplicated to the token bucket") + }) + + t.Run("unattributable interactions still use the token bucket", func(t *testing.T) { + ftp, h, _, _ := newFTP(t) + + // A login has no path, and a path outside any session cannot be attributed. + ftp.recordInteraction("198.51.100.4:3333", "USER anonymous\nlogging in") + ftp.recordInteractionForPath("198.51.100.4:3333", "RETR /nope\ndownloaded file", "/nope") + + shared, err := h.options.Storage.GetCacheItem(h.options.Token) + require.NoError(t, err) + require.Len(t, shared.Data, 2, "unattributable FTP noise keeps its existing behaviour") + }) + + t.Run("path for an unknown session is not attributed", func(t *testing.T) { + ftp, h, _, _ := newFTP(t) + unknown := "c6rj61aciaeutn2ae681" + + require.Equal(t, "", ftp.correlationIDFromPath("/"+uploadsDirName+"/"+unknown+"/evil.dtd"), + "only live sessions should be attributed") + + shared, err := h.options.Storage.GetCacheItem(h.options.Token) + require.NoError(t, err) + require.Empty(t, shared.Data) + }) + + // Paths are cleaned before the leading segment is taken, so traversal can + // only ever resolve to the session it actually points at -- never to a + // different one, and never outside the hosted-files root. + t.Run("traversal resolves before attribution", func(t *testing.T) { + ftp, _, id, _ := newFTP(t) + + // Leading ".." is dropped at the root, so this still names id. + require.Equal(t, id, ftp.correlationIDFromPath("/../"+uploadsDirName+"/"+id+"/evil.dtd")) + require.Equal(t, id, ftp.correlationIDFromPath("//"+uploadsDirName+"/"+id+"/evil.dtd")) + + // Climbing out of a session directory stops attributing to it. + require.Equal(t, "", ftp.correlationIDFromPath("/"+uploadsDirName+"/"+id+"/../other/evil.dtd")) + require.Equal(t, "", ftp.correlationIDFromPath("/"+uploadsDirName+"/"+id+"/..")) + + // A path outside the uploads directory is not ours, however much it looks + // like a session: an operator directory at the FTP root shares that shape. + require.Equal(t, "", ftp.correlationIDFromPath("/"+id+"/evil.dtd"), + "only paths under the uploads directory may be attributed") + }) +} diff --git a/pkg/server/http_server.go b/pkg/server/http_server.go index 7770e6b7..626fd80e 100644 --- a/pkg/server/http_server.go +++ b/pkg/server/http_server.go @@ -84,6 +84,25 @@ func NewHTTPServer(options *Options) (*HTTPServer, error) { router.Handle("/register", server.corsMiddleware(server.authMiddleware(http.HandlerFunc(server.registerHandler)))) router.Handle("/deregister", server.corsMiddleware(server.authMiddleware(http.HandlerFunc(server.deregisterHandler)))) router.Handle("/poll", server.corsMiddleware(server.authMiddleware(http.HandlerFunc(server.pollHandler)))) + // Registered even when uploads are disabled, so that an upload request to a + // server without -upload gets a clean 501 rather than falling through to + // "/", where the logger middleware would persist the whole file body as an + // interaction record. + // + // Unlike the other authenticated routes this one is not wrapped in + // authMiddleware: it checks the token itself, because a request that fails + // the check has to be recorded as an interaction before the 401 and the + // middleware returns too early to allow that. -upload forces -auth with a + // random token, and the client refuses to send an upload to a server whose + // advertised capabilities say uploads are off, so anything unauthenticated + // arriving here is a target probing the endpoint -- exactly what we exist to + // record. + router.Handle("/upload", server.corsMiddleware(http.HandlerFunc(server.uploadHandler))) + // Hosted files are served outside the logger middleware, which would + // otherwise copy each file body into an interaction record; the handler + // records a body-elided interaction itself. No CORS: these are fetched by + // the target under test, not cross-origin by a browser. + router.Handle("/f/", http.HandlerFunc(server.serveUploadedFile)) if server.options.EnableMetrics { router.Handle("/metrics", server.corsMiddleware(server.authMiddleware(http.HandlerFunc(server.metricsHandler)))) } @@ -399,7 +418,17 @@ func (h *HTTPServer) registerHandler(w http.ResponseWriter, req *http.Request) { } atomic.AddInt64(&h.options.Stats.Sessions, 1) atomic.AddInt64(&h.options.Stats.SessionsTotal, 1) - jsonMsg(w, "registration successful", http.StatusOK) + + // Capabilities ride along on the registration response so the client knows + // whether uploads are available without a second round trip. Older clients + // read only "message" and ignore the extra key. + w.Header().Set("Content-Type", "application/json; charset=utf-8") + w.Header().Set("X-Content-Type-Options", "nosniff") + w.WriteHeader(http.StatusOK) + _ = json.NewEncoder(w).Encode(&RegisterResponse{ + Message: "registration successful", + Capabilities: h.capabilities(), + }) gologger.Debug().Msgf("Registered correlationID %s for key\n", r.CorrelationID) } @@ -425,6 +454,17 @@ func (h *HTTPServer) deregisterHandler(w http.ResponseWriter, req *http.Request) jsonError(w, fmt.Sprintf("could not remove id: %s", err), http.StatusBadRequest) return } + + // Deleted synchronously rather than queued: the cache eviction hook fired + // by RemoveID above only enqueues the directory, leaving a window in which + // a client that has just deregistered could still fetch its own hosted + // files. Blocking here is safe -- unlike the cache's event goroutine, this + // handler can afford the filesystem call -- and the deletion is idempotent, + // so the queued removal that follows is a no-op. + if h.options.UploadStore != nil { + h.options.UploadStore.removeSessionNow(r.CorrelationID) + } + if h.options.RootTLD { for _, domain := range h.options.Domains { _ = h.options.Storage.RemoveConsumer(domain, r.CorrelationID) @@ -549,7 +589,20 @@ func (h *HTTPServer) checkToken(req *http.Request) bool { // metricsHandler is a handler for /metrics endpoint func (h *HTTPServer) metricsHandler(w http.ResponseWriter, req *http.Request) { - interactMetrics := h.options.Stats + // h.options.Stats is a *Metrics shared by every protocol server, whose + // counters are updated concurrently with atomic adds. Snapshot it into a + // local value with atomic loads rather than mutating the shared struct, + // which would race with those writers and with concurrent /metrics calls. + interactMetrics := Metrics{ + Dns: atomic.LoadUint64(&h.options.Stats.Dns), + Ftp: atomic.LoadUint64(&h.options.Stats.Ftp), + Http: atomic.LoadUint64(&h.options.Stats.Http), + Ldap: atomic.LoadUint64(&h.options.Stats.Ldap), + Smb: atomic.LoadUint64(&h.options.Stats.Smb), + Smtp: atomic.LoadUint64(&h.options.Stats.Smtp), + Sessions: atomic.LoadInt64(&h.options.Stats.Sessions), + SessionsTotal: atomic.LoadInt64(&h.options.Stats.SessionsTotal), + } interactMetrics.Cache = GetCacheMetrics(h.options) interactMetrics.Cpu = GetCpuMetrics() interactMetrics.Memory = GetMemoryMetrics() @@ -557,5 +610,5 @@ func (h *HTTPServer) metricsHandler(w http.ResponseWriter, req *http.Request) { w.Header().Set("Content-Type", "application/json; charset=utf-8") w.Header().Set("X-Content-Type-Options", "nosniff") - _ = json.NewEncoder(w).Encode(interactMetrics) + _ = json.NewEncoder(w).Encode(&interactMetrics) } diff --git a/pkg/server/metrics_race_test.go b/pkg/server/metrics_race_test.go new file mode 100644 index 00000000..114b0a00 --- /dev/null +++ b/pkg/server/metrics_race_test.go @@ -0,0 +1,88 @@ +package server + +import ( + "net/http" + "net/http/httptest" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/projectdiscovery/interactsh/pkg/storage" + "github.com/stretchr/testify/require" +) + +// newMetricsTestServer returns an HTTPServer backed by in-memory storage and a +// zeroed metrics struct, suitable for driving metricsHandler directly. +func newMetricsTestServer(t *testing.T) *HTTPServer { + t.Helper() + + store, err := storage.New(&storage.Options{EvictionTTL: 1 * time.Hour}) + require.NoError(t, err, "could not create storage") + t.Cleanup(func() { _ = store.Close() }) + + return &HTTPServer{options: &Options{Storage: store, Stats: &Metrics{}, EnableMetrics: true}} +} + +// TestMetricsHandlerDoesNotMutateSharedStats is a regression test for the +// /metrics handler aliasing options.Stats. It used to assign the *Metrics to a +// local variable, which copied the pointer rather than the struct, so every +// request wrote Cache/Cpu/Memory/Network into the one struct shared by all the +// protocol servers. The handler must leave that struct untouched. +func TestMetricsHandlerDoesNotMutateSharedStats(t *testing.T) { + h := newMetricsTestServer(t) + + w := httptest.NewRecorder() + h.metricsHandler(w, httptest.NewRequest("GET", "http://example.com/metrics", nil)) + require.Equal(t, http.StatusOK, w.Result().StatusCode) + + require.Nil(t, h.options.Stats.Cache, "handler must not write Cache into the shared stats") + require.Nil(t, h.options.Stats.Cpu, "handler must not write Cpu into the shared stats") + require.Nil(t, h.options.Stats.Memory, "handler must not write Memory into the shared stats") + require.Nil(t, h.options.Stats.Network, "handler must not write Network into the shared stats") +} + +// TestMetricsHandlerConcurrent exercises the /metrics snapshot against +// concurrent counter writers. Run with -race to catch regressions. +func TestMetricsHandlerConcurrent(t *testing.T) { + h := newMetricsTestServer(t) + + var wg sync.WaitGroup + stop := make(chan struct{}) + + // Writers mimic the protocol servers updating shared counters. + for i := 0; i < 4; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for { + select { + case <-stop: + return + default: + atomic.AddUint64(&h.options.Stats.Http, 1) + atomic.AddUint64(&h.options.Stats.Dns, 1) + atomic.AddInt64(&h.options.Stats.Sessions, 1) + atomic.AddInt64(&h.options.Stats.SessionsTotal, 1) + } + } + }() + } + + // Concurrent readers hitting the metrics endpoint. + for i := 0; i < 4; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for j := 0; j < 50; j++ { + w := httptest.NewRecorder() + h.metricsHandler(w, httptest.NewRequest("GET", "http://example.com/metrics", nil)) + require.Equal(t, http.StatusOK, w.Result().StatusCode) + } + }() + } + + time.Sleep(100 * time.Millisecond) + close(stop) + wg.Wait() +} diff --git a/pkg/server/server.go b/pkg/server/server.go index c480e37b..773ad9d8 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -111,6 +111,30 @@ type Options struct { // DefaultHTTPResponseFile is a file to serve for all HTTP requests (takes priority over other options) DefaultHTTPResponseFile string + // Ftp indicates the FTP server is enabled, so uploaded files are also + // reachable over ftp:// + Ftp bool + // Upload enables the client file upload endpoint and file hosting + Upload bool + // UploadDirectory is the root directory uploaded files are stored under + UploadDirectory string + // UploadMaxFileSize is the maximum size in bytes of a single uploaded file + UploadMaxFileSize int64 + // UploadMaxFiles is the maximum number of files a single session may upload + UploadMaxFiles int + // UploadMaxTotalSize is the maximum total size in bytes of all uploaded files + UploadMaxTotalSize int64 + // UploadTTL is the maximum lifetime of uploaded files on disk + UploadTTL time.Duration + // UploadStore serves and stores uploaded files. Nil when uploads are disabled. + UploadStore *UploadStore + // FTPServesUploads reports whether the FTP root and the upload root resolve + // to the same directory, which is what makes hosted files reachable over + // ftp://. Derived at startup from the resolved paths, never from the flags + // as written, so that two spellings of one directory are not mistaken for + // two directories. + FTPServesUploads bool + ACMEStore *acme.Provider Stats *Metrics OnResult OnResultCallback @@ -120,6 +144,15 @@ type Options struct { } type OnResultCallback func(out interface{}) +// UploadStorage returns the configured storage backend's upload-tracking +// capability, or nil when the backend does not implement it. Only +// instance-local backends do, since hosted bytes live on the local filesystem; +// see storage.UploadStorage. +func (options *Options) UploadStorage() storage.UploadStorage { + uploadStorage, _ := options.Storage.(storage.UploadStorage) + return uploadStorage +} + func (options *Options) GetIdLength() int { return options.CorrelationIdLength + options.CorrelationIdNonceLength } diff --git a/pkg/server/upload.go b/pkg/server/upload.go new file mode 100644 index 00000000..b7b04977 --- /dev/null +++ b/pkg/server/upload.go @@ -0,0 +1,580 @@ +package server + +import ( + "crypto/sha256" + "encoding/hex" + "os" + "path/filepath" + "regexp" + "sync" + "sync/atomic" + "time" + + "github.com/asaskevich/govalidator" + "github.com/pkg/errors" + "github.com/projectdiscovery/gologger" + "github.com/rs/xid" +) + +const ( + uploadSessionDirPerm = 0o700 + uploadFilePerm = 0o600 + + // uploadsDirName is the single directory, directly under the upload root, + // that holds every session's uploaded files. + // + // Sessions live one level down rather than at the root so that the root can + // be a shared -ftp-dir full of the operator's own files: the janitor and the + // startup purge only ever read, and delete, inside this directory, so no + // amount of coincidence between an operator's directory name and a + // correlation id can put their content at risk. The leading dot keeps it out + // of the way of an operator listing their own FTP root; it is not a security + // measure, since NopDriver is what actually hides it (see ftp_server.go). + uploadsDirName = ".interactsh-user-uploads" + + // deleteQueueSize bounds the backlog of session directories awaiting + // removal. Sends are non-blocking, so a full queue drops the request and + // leaves the directory for the janitor. + deleteQueueSize = 1024 + + // maxSweepInterval caps how long the janitor sleeps regardless of TTL. + maxSweepInterval = 10 * time.Minute + minSweepInterval = 1 * time.Minute +) + +// uploadNameRe is a strict allowlist for uploaded file names. It is applied +// identically on upload and on serve. +// +// This is deliberately an allowlist rather than a sanitiser: the user needs the +// exact byte-for-byte name to reference the file from a DTD or XSLT payload, so +// silently mangling a name is worse than rejecting it. It also means the name +// never needs escaping when it is placed in a Content-Disposition header. +// +// Rejects: path separators, "..", NUL and control characters, leading dots, +// absolute paths, and over-long names. +var uploadNameRe = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$`) + +// isSafeUploadName reports whether name is acceptable as an uploaded file name. +func isSafeUploadName(name string) bool { + if !uploadNameRe.MatchString(name) { + return false + } + // Belt and braces: the regexp already excludes "/" and "\", so no traversal + // sequence can survive, but a bare ".." must never slip through either. + return name != "." && name != ".." +} + +// UploadErrorKind classifies upload failures so the HTTP layer can map them to +// status codes without matching on error strings. +type UploadErrorKind int + +const ( + UploadErrOther UploadErrorKind = iota + UploadErrBadName + UploadErrTooLarge + UploadErrTooManyFiles + UploadErrOutOfSpace +) + +// UploadError is a failure attributable to the uploaded content or to server +// capacity, as opposed to an internal error. +type UploadError struct { + Kind UploadErrorKind + Err error +} + +func (e *UploadError) Error() string { return e.Err.Error() } +func (e *UploadError) Unwrap() error { return e.Err } + +func uploadErr(kind UploadErrorKind, format string, args ...interface{}) *UploadError { + return &UploadError{Kind: kind, Err: errors.Errorf(format, args...)} +} + +// UploadStore owns the on-disk lifecycle of client-uploaded files. +// +// Layout is ///. The session +// directory is named by correlation ID because FTP has no Host header, so the +// identifier has to be carried in the path for the FTP view to work. The +// uploadsDirName level exists so that the root may be shared with -ftp-dir +// without this store ever touching an operator's files. +type UploadStore struct { + root string + // sessionsRoot is /: everything this store creates, + // enumerates and deletes lives under it, and nothing above it is ours. + sessionsRoot string + // rootFS is an openat-rooted handle used for the serving path. Names there + // come from a URL, and os.Root refuses any resolution that escapes the root + // or traverses a symlink out of it, on every platform we build for. + rootFS *os.Root + maxFileSize int64 + maxFiles int + maxTotal int64 + ttl time.Duration + + // correlationIDLength is used to recognise session directories inside + // sessionsRoot, and to reject implausible identifiers before they reach a + // filepath.Join. + correlationIDLength int + + totalBytes atomic.Int64 + + deleteCh chan string + closeCh chan struct{} + closeWG sync.WaitGroup + closeAt sync.Once +} + +// NewUploadStore prepares the upload root and returns a store. It does not +// start the background goroutines; call Start for that. +func NewUploadStore(options *Options) (*UploadStore, error) { + root := options.UploadDirectory + switch { + case root != "": + case options.FTPDirectory != "": + // Share the FTP root so that -ftp serves uploads with no extra config. + root = options.FTPDirectory + default: + var err error + if root, err = os.MkdirTemp("", "interactsh-uploads-"); err != nil { + return nil, errors.Wrap(err, "could not create temporary upload directory") + } + } + + abs, err := filepath.Abs(root) + if err != nil { + return nil, errors.Wrap(err, "could not resolve upload directory") + } + + // Creating the sessions directory creates the root along with it, so every + // branch above ends up with both present before os.OpenRoot needs them -- + // including a -ftp-dir that does not exist yet, which the FTP server itself + // never required. + sessionsRoot := filepath.Join(abs, uploadsDirName) + if err := os.MkdirAll(sessionsRoot, uploadSessionDirPerm); err != nil { + return nil, errors.Wrap(err, "could not create upload directory") + } + + // MkdirAll succeeds on a directory that already exists but cannot be written + // to -- a read-only mount, or one owned by another user -- and the failure + // would otherwise surface only when a client's first upload 500s, long after + // startup. Probe it now so the server refuses to start with a clear reason. + if err := checkUploadDirWritable(sessionsRoot); err != nil { + return nil, err + } + + rootFS, err := os.OpenRoot(abs) + if err != nil { + return nil, errors.Wrap(err, "could not open upload directory") + } + + s := &UploadStore{ + root: abs, + sessionsRoot: sessionsRoot, + rootFS: rootFS, + maxFileSize: options.UploadMaxFileSize, + maxFiles: options.UploadMaxFiles, + maxTotal: options.UploadMaxTotalSize, + ttl: options.UploadTTL, + correlationIDLength: options.CorrelationIdLength, + deleteCh: make(chan string, deleteQueueSize), + closeCh: make(chan struct{}), + } + + // Upload metadata lives only in the cache, so no session survives a + // restart; anything already here is by definition an orphan. + s.purge() + return s, nil +} + +// checkUploadDirWritable verifies that files can actually be created in dir, by +// doing it. Permission bits are not consulted directly: they answer the question +// for the wrong subject on a setuid binary, and they do not answer it at all for +// a read-only mount, an exhausted filesystem or a restrictive ACL. +func checkUploadDirWritable(dir string) error { + f, err := os.CreateTemp(dir, ".writable-*") + if err != nil { + return errors.Wrapf(err, "upload directory %s is not writable", dir) + } + name := f.Name() + defer func() { _ = os.Remove(name) }() + + // Written to as well as created, so that a filesystem which allows the + // create but refuses the write is caught here rather than mid-upload. + if _, err := f.Write([]byte("interactsh")); err != nil { + _ = f.Close() + return errors.Wrapf(err, "upload directory %s is not writable", dir) + } + if err := f.Close(); err != nil { + return errors.Wrapf(err, "upload directory %s is not writable", dir) + } + return nil +} + +// Root returns the directory uploaded files are stored under. +func (s *UploadStore) Root() string { return s.root } + +// MaxFileSize returns the per-file byte limit. +func (s *UploadStore) MaxFileSize() int64 { return s.maxFileSize } + +// MaxFiles returns the per-session file count limit. +func (s *UploadStore) MaxFiles() int { return s.maxFiles } + +// Start launches the deleter and janitor goroutines. +func (s *UploadStore) Start() { + s.closeWG.Add(2) + go s.runDeleter() + go s.runJanitor() +} + +// Close stops the background goroutines and releases the rooted handle. +func (s *UploadStore) Close() error { + s.closeAt.Do(func() { close(s.closeCh) }) + s.closeWG.Wait() + return s.rootFS.Close() +} + +// looksLikeSessionDir reports whether a directory entry name could be one of +// our session directories. +// +// Operator files are kept safe structurally, by everything of ours living under +// uploadsDirName; this check is the second line of defence, so that anything +// unexpected inside that directory is left alone rather than deleted. +func (s *UploadStore) looksLikeSessionDir(name string) bool { + return len(name) == s.correlationIDLength && govalidator.IsAlphanumeric(name) +} + +// sessionDir returns the directory for a correlation ID, or "" if the ID is not +// a plausible correlation ID. Validating before any filepath.Join is what keeps +// a hostile identifier from escaping the root. +func (s *UploadStore) sessionDir(correlationID string) string { + if !s.looksLikeSessionDir(correlationID) { + return "" + } + return filepath.Join(s.sessionsRoot, correlationID) +} + +// stagedUpload is a file written into its session directory under a temporary +// name, with its quota already reserved, waiting to be renamed into place. +// +// Staging exists so that a multi-file upload is all-or-nothing. Writing straight +// to the final name would commit each file as it went, and a failure part-way +// through a batch -- the per-session file cap, the global quota, a full disk -- +// left the earlier files on disk holding quota while the caller discarded the +// metadata that made them reachable. +type stagedUpload struct { + correlationID string + // name is the final name; tmp is where the bytes are until Commit. + name string + tmp string + size int64 + // delta is the quota reserved for this file: the net change, since + // overwriting a name replaces the bytes it already occupied. + delta int64 + sha256 string + // overwrites records that the final name already held a file, which is what + // makes discarding a committed file unsafe: the previous content is gone. + overwrites bool + committed bool +} + +// Stage validates one file, reserves its quota and writes it under a temporary +// name in its session directory. Nothing is reachable until Commit. +// +// It is called from inside Storage.UpdateUploads, i.e. under the correlation +// ID's lock, so the caller's quota check and this write are atomic with respect +// to other uploads for the same session. +func (s *UploadStore) Stage(correlationID, name string, data []byte, existingSize int64) (*stagedUpload, error) { + if !isSafeUploadName(name) { + return nil, uploadErr(UploadErrBadName, "invalid file name %q", name) + } + size := int64(len(data)) + if size == 0 { + return nil, uploadErr(UploadErrBadName, "file %q is empty", name) + } + if size > s.maxFileSize { + return nil, uploadErr(UploadErrTooLarge, "file %q is %d bytes, limit is %d", name, size, s.maxFileSize) + } + + dir := s.sessionDir(correlationID) + if dir == "" { + return nil, uploadErr(UploadErrOther, "invalid correlation-id") + } + + // Reserve space before writing. existingSize is what this name already + // occupies, since overwriting replaces rather than adds. + delta := size - existingSize + if delta > 0 { + if s.totalBytes.Add(delta) > s.maxTotal { + s.totalBytes.Add(-delta) + return nil, uploadErr(UploadErrOutOfSpace, "server upload capacity exhausted") + } + } else { + s.totalBytes.Add(delta) + } + + release := func() { s.totalBytes.Add(-delta) } + + if err := os.MkdirAll(dir, uploadSessionDirPerm); err != nil { + release() + return nil, errors.Wrap(err, "could not create session directory") + } + + // Plain os rather than the rooted handle: os.Root has no Rename before Go + // 1.25, and both components are already constrained -- the correlation ID + // is alphanumeric and length-checked, the name passed the allowlist, so + // neither can contain a separator or traversal sequence. + tmp := filepath.Join(dir, ".upload-"+xid.New().String()) + if err := os.WriteFile(tmp, data, uploadFilePerm); err != nil { + release() + return nil, errors.Wrap(err, "could not write uploaded file") + } + + sum := sha256.Sum256(data) + return &stagedUpload{ + correlationID: correlationID, + name: name, + tmp: tmp, + size: size, + delta: delta, + sha256: hex.EncodeToString(sum[:]), + overwrites: existingSize > 0, + }, nil +} + +// Commit renames a staged file into place, making it reachable. Writing to a +// temporary name and renaming is also what stops a reader -- the HTTP handler or +// the FTP file driver -- from ever observing a partially written file. +func (s *UploadStore) Commit(st *stagedUpload) error { + dir := s.sessionDir(st.correlationID) + if dir == "" { + return uploadErr(UploadErrOther, "invalid correlation-id") + } + if err := os.Rename(st.tmp, filepath.Join(dir, st.name)); err != nil { + return errors.Wrap(err, "could not commit uploaded file") + } + st.committed = true + return nil +} + +// Abort discards a staged file and releases its reservation. +// +// A file that was never committed is only a temp file, so it goes without +// question. A committed one is a narrower case -- a later rename in the same +// batch failed -- and is only removed when it did not overwrite anything: the +// previous content of an overwritten name is already gone, so deleting it would +// turn a leaked file into lost data. Everything Abort touches resolves to +// //, so it can never reach another +// session's files. +func (s *UploadStore) Abort(st *stagedUpload) { + if !st.committed { + if err := os.Remove(st.tmp); err != nil && !os.IsNotExist(err) { + gologger.Debug().Msgf("Could not remove staged upload %s: %s\n", st.tmp, err) + } + s.totalBytes.Add(-st.delta) + return + } + if st.overwrites { + gologger.Warning().Msgf("Uploaded file %s/%s replaced an existing file and cannot be rolled back\n", + st.correlationID, st.name) + return + } + dir := s.sessionDir(st.correlationID) + if dir == "" { + return + } + if err := os.Remove(filepath.Join(dir, st.name)); err != nil && !os.IsNotExist(err) { + gologger.Debug().Msgf("Could not roll back uploaded file %s/%s: %s\n", st.correlationID, st.name, err) + return + } + s.totalBytes.Add(-st.delta) +} + +// Save stages and immediately commits one file, for callers handling a single +// file with nothing to unwind. +func (s *UploadStore) Save(correlationID, name string, data []byte, existingSize int64) (int64, string, error) { + st, err := s.Stage(correlationID, name, data, existingSize) + if err != nil { + return 0, "", err + } + if err := s.Commit(st); err != nil { + s.Abort(st) + return 0, "", err + } + return st.size, st.sha256, nil +} + +// Open returns a readable handle to an uploaded file. The name is re-validated +// here rather than trusted from the caller. +func (s *UploadStore) Open(correlationID, name string) (*os.File, os.FileInfo, error) { + if !isSafeUploadName(name) { + return nil, nil, errors.New("invalid file name") + } + if !s.looksLikeSessionDir(correlationID) { + return nil, nil, errors.New("invalid correlation-id") + } + + // Resolved through the rooted handle, so neither component can escape the + // upload directory even if the root is operator-supplied and contains a + // planted symlink. + f, err := s.rootFS.Open(filepath.Join(uploadsDirName, correlationID, name)) + if err != nil { + return nil, nil, err + } + fi, err := f.Stat() + if err != nil { + _ = f.Close() + return nil, nil, err + } + if !fi.Mode().IsRegular() { + _ = f.Close() + return nil, nil, errors.New("not a regular file") + } + return f, fi, nil +} + +// RemoveSession queues a session's directory for deletion. It is safe to call +// from the storage cache's event goroutine: the send is non-blocking, so +// filesystem latency can never back-pressure cache maintenance. A dropped +// request is collected by the janitor instead. +func (s *UploadStore) RemoveSession(correlationID string) { + if s.sessionDir(correlationID) == "" { + return + } + select { + case s.deleteCh <- correlationID: + default: + gologger.Debug().Msgf("Upload delete queue full, leaving %s to the janitor\n", correlationID) + } +} + +// removeSessionNow deletes a session directory synchronously. +func (s *UploadStore) removeSessionNow(correlationID string) { + dir := s.sessionDir(correlationID) + if dir == "" { + return + } + freed := dirSize(dir) + if err := os.RemoveAll(dir); err != nil { + gologger.Warning().Msgf("Could not remove upload directory for %s: %s\n", correlationID, err) + return + } + if freed > 0 { + s.totalBytes.Add(-freed) + } +} + +func (s *UploadStore) runDeleter() { + defer s.closeWG.Done() + for { + select { + case id := <-s.deleteCh: + s.removeSessionNow(id) + case <-s.closeCh: + // Drain whatever is already queued, then stop. + for { + select { + case id := <-s.deleteCh: + s.removeSessionNow(id) + default: + return + } + } + } + } +} + +// runJanitor is the authoritative garbage collector for uploaded files. +// +// It cannot be left to cache eviction: goburrow/cache has no background +// janitor, so expiry is only processed on cache activity. An idle server would +// never evict, and would therefore leak every uploaded file indefinitely. +func (s *UploadStore) runJanitor() { + defer s.closeWG.Done() + + interval := s.ttl / 10 + if interval > maxSweepInterval { + interval = maxSweepInterval + } + if interval < minSweepInterval { + interval = minSweepInterval + } + + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ticker.C: + s.sweep() + case <-s.closeCh: + return + } + } +} + +// sweep removes session directories older than the TTL and recomputes the +// global byte total. +// +// Liveness is judged by directory mtime, never by asking the cache. A cache +// lookup would refresh the entry's access time, so under the default sliding +// eviction strategy probing every swept session would make exactly those +// sessions immortal. +func (s *UploadStore) sweep() { + entries, err := os.ReadDir(s.sessionsRoot) + if err != nil { + gologger.Warning().Msgf("Could not read upload directory: %s\n", err) + return + } + + var total int64 + for _, entry := range entries { + if !entry.IsDir() || !s.looksLikeSessionDir(entry.Name()) { + continue + } + dir := filepath.Join(s.sessionsRoot, entry.Name()) + info, err := entry.Info() + if err != nil { + continue + } + // Directory mtime advances when a file is added or removed, so this is + // effectively "time since the last upload". Reads do not touch it. + if time.Since(info.ModTime()) > s.ttl { + if err := os.RemoveAll(dir); err != nil { + gologger.Warning().Msgf("Could not sweep upload directory %s: %s\n", dir, err) + total += dirSize(dir) + } + continue + } + total += dirSize(dir) + } + s.totalBytes.Store(total) +} + +// purge removes every session directory in the root, leaving anything that does +// not look like one untouched. +func (s *UploadStore) purge() { + entries, err := os.ReadDir(s.sessionsRoot) + if err != nil { + return + } + for _, entry := range entries { + if entry.IsDir() && s.looksLikeSessionDir(entry.Name()) { + _ = os.RemoveAll(filepath.Join(s.sessionsRoot, entry.Name())) + } + } + s.totalBytes.Store(0) +} + +// dirSize sums the regular files directly inside dir. +func dirSize(dir string) int64 { + entries, err := os.ReadDir(dir) + if err != nil { + return 0 + } + var total int64 + for _, entry := range entries { + if info, err := entry.Info(); err == nil && info.Mode().IsRegular() { + total += info.Size() + } + } + return total +} diff --git a/pkg/server/upload_batch_test.go b/pkg/server/upload_batch_test.go new file mode 100644 index 00000000..dd817843 --- /dev/null +++ b/pkg/server/upload_batch_test.go @@ -0,0 +1,191 @@ +package server + +import ( + "encoding/base64" + "encoding/json" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/google/uuid" + "github.com/rs/xid" + "github.com/stretchr/testify/require" +) + +// orderedUploadBody is uploadBody with a defined file order, which the map-based +// helper cannot give: these tests need to know which file fails. +func orderedUploadBody(t *testing.T, correlationID, secret string, names []string, data [][]byte) string { + t.Helper() + require.Len(t, data, len(names)) + + req := UploadRequest{CorrelationID: correlationID, SecretKey: secret} + for i, name := range names { + req.Files = append(req.Files, UploadFileRequest{ + Name: name, + Data: base64.StdEncoding.EncodeToString(data[i]), + }) + } + encoded, err := json.Marshal(req) + require.NoError(t, err) + return string(encoded) +} + +// sessionFiles lists the file names on disk for a session, temp files included, +// so that a leaked staging file is visible to these assertions. +func sessionFiles(t *testing.T, store *UploadStore, correlationID string) []string { + t.Helper() + entries, err := os.ReadDir(store.sessionDir(correlationID)) + if os.IsNotExist(err) { + return nil + } + require.NoError(t, err) + out := make([]string, 0, len(entries)) + for _, e := range entries { + out = append(out, e.Name()) + } + return out +} + +// newSession registers an additional session on an existing server, for the +// cross-session assertions. +func newSession(t *testing.T, h *HTTPServer) (correlationID, secret string) { + t.Helper() + correlationID = xid.New().String() + secret = uuid.New().String() + require.NoError(t, h.options.Storage.SetIDPublicKey(correlationID, secret, testPublicKey(t))) + return correlationID, secret +} + +// A multi-file upload is all-or-nothing. Committing each file as it is written +// left the earlier ones on disk holding quota while the metadata that made them +// reachable was discarded: unusable, uncollectable until the session ended, and +// counted against every other session's uploads. +func TestUploadBatchIsAtomic(t *testing.T) { + t.Run("quota exhausted part-way leaves nothing behind", func(t *testing.T) { + h, id, secret := uploadTestServer(t, true) + store := h.options.UploadStore + store.maxTotal = 1500 // fits one 1000-byte file, not two + + body := orderedUploadBody(t, id, secret, + []string{"a.bin", "b.bin"}, + [][]byte{make([]byte, 1000), make([]byte, 1000)}) + resp := doUpload(t, h, body) + require.Equal(t, http.StatusInsufficientStorage, resp.StatusCode) + + files, _ := h.options.UploadStorage().ListUploads(id) + require.Empty(t, files, "metadata must be unchanged") + require.Empty(t, sessionFiles(t, store, id), "no file, and no staging temp file, may survive") + require.Zero(t, store.totalBytes.Load(), "the reservation must be released") + }) + + t.Run("and the same upload then succeeds on retry", func(t *testing.T) { + h, id, secret := uploadTestServer(t, true) + store := h.options.UploadStore + store.maxTotal = 1500 + + require.Equal(t, http.StatusInsufficientStorage, doUpload(t, h, orderedUploadBody(t, id, secret, + []string{"a.bin", "b.bin"}, + [][]byte{make([]byte, 1000), make([]byte, 1000)})).StatusCode) + + // Previously wedged: the orphan still held 1000 of the 1500 bytes while + // metadata reported nothing, so this retry was refused indefinitely. + resp := doUpload(t, h, orderedUploadBody(t, id, secret, + []string{"a.bin"}, [][]byte{make([]byte, 1000)})) + require.Equal(t, http.StatusOK, resp.StatusCode, "a retry of the file that fitted must succeed") + + files, _ := h.options.UploadStorage().ListUploads(id) + require.Len(t, files, 1) + require.EqualValues(t, 1000, store.totalBytes.Load(), "charged once, not twice") + }) + + t.Run("per-session file cap reached part-way leaves nothing behind", func(t *testing.T) { + h, id, secret := uploadTestServer(t, true) // UploadMaxFiles is 3 + store := h.options.UploadStore + + for _, name := range []string{"f0.dtd", "f1.dtd"} { + require.Equal(t, http.StatusOK, + doUpload(t, h, orderedUploadBody(t, id, secret, []string{name}, [][]byte{[]byte("x")})).StatusCode) + } + before := store.totalBytes.Load() + + // Two more against a limit of three: the first fits, the second does not. + resp := doUpload(t, h, orderedUploadBody(t, id, secret, + []string{"f2.dtd", "f3.dtd"}, [][]byte{[]byte("yy"), []byte("zz")})) + require.Equal(t, http.StatusRequestEntityTooLarge, resp.StatusCode) + + files, _ := h.options.UploadStorage().ListUploads(id) + require.Len(t, files, 2, "metadata must be unchanged") + require.ElementsMatch(t, []string{"f0.dtd", "f1.dtd"}, sessionFiles(t, store, id), + "the file that fitted must not be left on disk") + require.Equal(t, before, store.totalBytes.Load()) + }) + + // The reason staging beats deleting what was already written: for a name that + // already existed, the previous content is gone the moment it is overwritten, + // so a compensating delete would turn a leaked file into lost data. + t.Run("a failed batch does not disturb the file it would have replaced", func(t *testing.T) { + h, id, secret := uploadTestServer(t, true) + store := h.options.UploadStore + + original := []byte("original payload") + require.Equal(t, http.StatusOK, + doUpload(t, h, orderedUploadBody(t, id, secret, []string{"a.dtd"}, [][]byte{original})).StatusCode) + before, _ := h.options.UploadStorage().ListUploads(id) + require.Len(t, before, 1) + charged := store.totalBytes.Load() + + // Replace a.dtd and add a file that cannot fit: the batch must fail whole. + store.maxTotal = int64(len(original)) + 10 + resp := doUpload(t, h, orderedUploadBody(t, id, secret, + []string{"a.dtd", "b.dtd"}, + [][]byte{[]byte("replacement payload"), make([]byte, 1000)})) + require.NotEqual(t, http.StatusOK, resp.StatusCode) + + onDisk, err := os.ReadFile(filepath.Join(store.sessionDir(id), "a.dtd")) + require.NoError(t, err) + require.Equal(t, original, onDisk, "the previously hosted file must be untouched") + + after, _ := h.options.UploadStorage().ListUploads(id) + require.Equal(t, before, after, "metadata must still describe the file that is on disk") + require.Equal(t, charged, store.totalBytes.Load()) + require.ElementsMatch(t, []string{"a.dtd"}, sessionFiles(t, store, id)) + }) + + // Unwinding resolves every path through sessionDir plus a validated name, so + // it cannot reach outside the session it belongs to. + t.Run("unwinding one session does not touch another", func(t *testing.T) { + h, mine, mySecret := uploadTestServer(t, true) + store := h.options.UploadStore + + theirs, theirSecret := newSession(t, h) + require.Equal(t, http.StatusOK, doUpload(t, h, + orderedUploadBody(t, theirs, theirSecret, []string{"a.bin"}, [][]byte{[]byte("their payload")})).StatusCode) + + store.maxTotal = store.totalBytes.Load() + 1200 + resp := doUpload(t, h, orderedUploadBody(t, mine, mySecret, + []string{"a.bin", "b.bin"}, + [][]byte{make([]byte, 1000), make([]byte, 1000)})) + require.Equal(t, http.StatusInsufficientStorage, resp.StatusCode) + + theirFile, err := os.ReadFile(filepath.Join(store.sessionDir(theirs), "a.bin")) + require.NoError(t, err) + require.Equal(t, []byte("their payload"), theirFile, + "another session's identically named file must be untouched") + theirMeta, ok := h.options.UploadStorage().ListUploads(theirs) + require.True(t, ok) + require.Len(t, theirMeta, 1) + require.Empty(t, sessionFiles(t, store, mine)) + }) + + t.Run("no staging temp files survive a successful batch either", func(t *testing.T) { + h, id, secret := uploadTestServer(t, true) + require.Equal(t, http.StatusOK, doUpload(t, h, orderedUploadBody(t, id, secret, + []string{"a.dtd", "b.dtd"}, [][]byte{[]byte("one"), []byte("two")})).StatusCode) + + for _, name := range sessionFiles(t, h.options.UploadStore, id) { + require.False(t, strings.HasPrefix(name, ".upload-"), "temp file %q left behind", name) + } + }) +} diff --git a/pkg/server/upload_handler.go b/pkg/server/upload_handler.go new file mode 100644 index 00000000..c4d2d4ae --- /dev/null +++ b/pkg/server/upload_handler.go @@ -0,0 +1,522 @@ +package server + +import ( + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "mime" + "net" + "net/http" + "net/http/httputil" + "path" + "strings" + "sync/atomic" + "time" + + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/interactsh/pkg/storage" +) + +// Capabilities advertises optional server features to clients at registration. +// Clients use it to decide whether to attempt an upload at all, and which URLs +// are worth printing. +type Capabilities struct { + Upload bool `json:"upload"` + UploadMaxFileSize int64 `json:"upload-max-file-size,omitempty"` + UploadMaxFiles int `json:"upload-max-files,omitempty"` + FTP bool `json:"ftp"` +} + +// RegisterResponse is the response to a client registration. The bare +// {"message": ...} shape older clients expect is preserved, so an old client +// against a new server simply ignores the extra key, and a new client against +// an old server sees a nil Capabilities. +type RegisterResponse struct { + Message string `json:"message"` + Capabilities *Capabilities `json:"capabilities,omitempty"` +} + +// UploadFileRequest is a single file within an upload request. +type UploadFileRequest struct { + Name string `json:"name"` + Data string `json:"data"` // base64 +} + +// UploadRequest is a request to host files against a correlation ID. +type UploadRequest struct { + CorrelationID string `json:"correlation-id"` + SecretKey string `json:"secret-key"` + Files []UploadFileRequest `json:"files"` +} + +// UploadedFileResponse describes one hosted file. Paths rather than URLs: the +// server does not know which of its domains, or which scheme, the client will +// use to reference the file. +type UploadedFileResponse struct { + Name string `json:"name"` + Size int64 `json:"size"` + SHA256 string `json:"sha256"` + HTTPPath string `json:"http-path"` + FTPPath string `json:"ftp-path"` +} + +// UploadResponse is the response to a successful upload. +type UploadResponse struct { + Message string `json:"message"` + Files []UploadedFileResponse `json:"files"` +} + +// capabilities describes what this server offers. +func (h *HTTPServer) capabilities() *Capabilities { + // FTP is advertised only when it can actually reach the hosted files. The + // client uses this to decide whether an ftp:// URL is worth printing, and a + // URL pointing into a directory the FTP server does not serve is worse than + // no URL at all: the target follows it, gets a 550, and the operator reads + // the silence as "not vulnerable". + c := &Capabilities{FTP: h.options.Ftp && h.options.FTPServesUploads} + // Advertised only when the storage backend can track uploads too, so the + // server never announces a capability uploadHandler would refuse with 501. + if store := h.options.UploadStore; store != nil && h.options.UploadStorage() != nil { + c.Upload = true + c.UploadMaxFileSize = store.MaxFileSize() + c.UploadMaxFiles = store.MaxFiles() + } + return c +} + +// maxUploadRequestBytes bounds the request body: every file at its size limit, +// base64 expanded, plus room for JSON framing. +func maxUploadRequestBytes(store *UploadStore) int64 { + return int64(store.MaxFiles())*store.MaxFileSize()*4/3 + 8192 +} + +// uploadHandler stores files against a correlation ID for later hosting. +// +// Registered as its own route rather than under "/" so that it never passes +// through the logger middleware, which would otherwise dump the entire request +// body -- i.e. every uploaded file -- into an interaction record. It is +// registered even when uploads are disabled, precisely so that a request to a +// non-upload server gets a clean 501 instead of falling through to that path. +func (h *HTTPServer) uploadHandler(w http.ResponseWriter, req *http.Request) { + // The token check lives here rather than in authMiddleware so that a request + // failing it can be recorded first; see the route registration for why that + // matters. Recording is deliberately limited to requests no legitimate client + // could have sent: an authenticated upload is the operator's own traffic, and + // filing it as an interaction would attribute their action to the target. + if !h.checkToken(req) { + h.recordUploadProbe(req, http.StatusUnauthorized, "") + w.WriteHeader(http.StatusUnauthorized) + return + } + + store := h.options.UploadStore + uploadStorage := h.options.UploadStorage() + // uploadStorage is nil when the configured storage backend cannot track + // uploads (a shared backend such as Redis). Treated the same as uploads + // being switched off, so the client sees the usual capability signal. + if store == nil || uploadStorage == nil { + // A client that has seen this server's capabilities does not send an + // upload here at all, so this is a probe too. + const message = "file upload is not enabled on this server" + h.recordUploadProbe(req, http.StatusNotImplemented, message) + jsonError(w, message, http.StatusNotImplemented) + return + } + if req.Method != http.MethodPost { + jsonError(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + + req.Body = http.MaxBytesReader(w, req.Body, maxUploadRequestBytes(store)) + + r := &UploadRequest{} + if err := json.NewDecoder(req.Body).Decode(r); err != nil { + var maxErr *http.MaxBytesError + if errors.As(err, &maxErr) { + jsonError(w, "upload request too large", http.StatusRequestEntityTooLarge) + return + } + jsonError(w, fmt.Sprintf("could not decode json body: %s", err), http.StatusBadRequest) + return + } + if len(r.Files) == 0 { + jsonError(w, "no files provided", http.StatusBadRequest) + return + } + if len(r.Files) > store.MaxFiles() { + jsonError(w, fmt.Sprintf("too many files, limit is %d", store.MaxFiles()), http.StatusRequestEntityTooLarge) + return + } + + // Decode and validate everything before touching storage, so a malformed + // request cannot leave a session half-populated. + decoded := make([][]byte, len(r.Files)) + seen := make(map[string]struct{}, len(r.Files)) + for i, f := range r.Files { + if !isSafeUploadName(f.Name) { + jsonError(w, fmt.Sprintf("invalid file name %q", f.Name), http.StatusBadRequest) + return + } + if _, dup := seen[f.Name]; dup { + jsonError(w, fmt.Sprintf("duplicate file name %q", f.Name), http.StatusBadRequest) + return + } + seen[f.Name] = struct{}{} + + data, err := base64.StdEncoding.DecodeString(f.Data) + if err != nil { + jsonError(w, fmt.Sprintf("could not decode data for %q: %s", f.Name, err), http.StatusBadRequest) + return + } + if len(data) == 0 { + jsonError(w, fmt.Sprintf("file %q is empty", f.Name), http.StatusBadRequest) + return + } + if int64(len(data)) > store.MaxFileSize() { + jsonError(w, fmt.Sprintf("file %q exceeds the %d byte limit", f.Name, store.MaxFileSize()), http.StatusRequestEntityTooLarge) + return + } + decoded[i] = data + } + + var ( + response []UploadedFileResponse + ftpPrefix = path.Join("/", uploadsDirName, r.CorrelationID) + ) + + // Files are staged first and committed only once every one of them has been + // written, so a failure part-way through a batch leaves nothing reachable and + // nothing charged. Committing as we went would leave the earlier files on + // disk holding quota while UpdateUploads discarded the metadata that made + // them reachable -- unusable, uncollectable until the session ended, and + // still counted against every other session's uploads. + // + // Anything still staged when this returns is unwound, including on a panic. + var staged []*stagedUpload + defer func() { + for _, st := range staged { + store.Abort(st) + } + }() + + // The disk writes happen inside UpdateUploads, i.e. under the correlation + // ID's lock, so the per-session quota check and the commit are atomic with + // respect to a concurrent upload for the same session. + err := uploadStorage.UpdateUploads(r.CorrelationID, r.SecretKey, func(existing []storage.UploadedFile) ([]storage.UploadedFile, error) { + updated := append([]storage.UploadedFile(nil), existing...) + + for i, f := range r.Files { + var existingSize int64 + idx := -1 + for j, u := range updated { + if u.Name == f.Name { + existingSize, idx = u.Size, j + break + } + } + // Replacing a name reuses its slot rather than consuming a new one. + if idx == -1 && len(updated) >= store.MaxFiles() { + return nil, uploadErr(UploadErrTooManyFiles, + "session already holds %d files, limit is %d", len(updated), store.MaxFiles()) + } + + st, err := store.Stage(r.CorrelationID, f.Name, decoded[i], existingSize) + if err != nil { + return nil, err + } + staged = append(staged, st) + + record := storage.UploadedFile{Name: f.Name, Size: st.size, SHA256: st.sha256, Timestamp: time.Now()} + if idx == -1 { + updated = append(updated, record) + } else { + updated[idx] = record + } + response = append(response, UploadedFileResponse{ + Name: f.Name, + Size: st.size, + SHA256: st.sha256, + HTTPPath: path.Join("/f", f.Name), + FTPPath: path.Join(ftpPrefix, f.Name), + }) + } + + // Every file is on disk under a temporary name; publish them together. + // A rename failing here is severe -- the write into the same directory + // has already succeeded -- so the batch still fails, and Abort unwinds + // what it safely can. + for _, st := range staged { + if err := store.Commit(st); err != nil { + return nil, err + } + } + return updated, nil + }) + + if err != nil { + h.writeUploadError(w, err) + return + } + // Committed and recorded: there is nothing left to unwind. + staged = nil + + gologger.Debug().Msgf("Stored %d uploaded file(s) for %s\n", len(response), r.CorrelationID) + w.Header().Set("Content-Type", "application/json; charset=utf-8") + w.Header().Set("X-Content-Type-Options", "nosniff") + _ = json.NewEncoder(w).Encode(&UploadResponse{Message: "upload successful", Files: response}) +} + +// serveUploadedFile serves a file hosted against the correlation id in the Host +// header, and records the fetch as an interaction. +// +// This runs on its own route, outside the logger middleware, and records the +// interaction itself. The logger buffers the whole response into a recorder and +// dumps the body into Interaction.RawResponse; for a hosted file that means +// every fetch retains a multiple of the file size in the session's interaction +// buffer, which has no cap in memory mode, and the retained copy is mangled by +// JSON escaping anyway. Recording explicitly keeps the evidence that the fetch +// happened -- which is the entire point for second-stage OOB verification -- +// without the payload. +// +// Serving here rather than from defaultHandler also avoids being shadowed by +// -dhr, by the .json/.xml suffix branches, and by -dr header injection. +func (h *HTTPServer) serveUploadedFile(w http.ResponseWriter, req *http.Request) { + // Every exit below is recorded, by one deferred call rather than a call per + // return: a miss is the operator's evidence that the target fetched *some* + // path -- the wrong name, or a file that has since expired -- and adding an + // early return here must not be able to silently drop that again. + rec := &hostedFetchRecorder{ResponseWriter: w} + uniqueID, fullID := h.options.extractCorrelationID(req.Host) + var meta *storage.UploadedFile + defer func() { h.recordHostedFetch(req, uniqueID, fullID, meta, rec) }() + + store := h.options.UploadStore + uploadStorage := h.options.UploadStorage() + if store == nil || uploadStorage == nil { + http.NotFound(rec, req) + return + } + if uniqueID == "" { + http.NotFound(rec, req) + return + } + correlationID := uniqueID[:h.options.CorrelationIdLength] + + // Only the /f/ subtree is ours. CutPrefix rather than TrimPrefix so that a + // path which does not carry the prefix is rejected instead of being read as + // a file name; ServeMux redirects a bare /f to /f/ before we are reached. + name, ok := strings.CutPrefix(req.URL.Path, "/f/") + if !ok || !isSafeUploadName(name) { + http.NotFound(rec, req) + return + } + + // Consult the metadata first: it makes a miss cheap, and means a file can + // only be served to the session that actually owns it. + files, ok := uploadStorage.ListUploads(correlationID) + if !ok { + http.NotFound(rec, req) + return + } + for i := range files { + if files[i].Name == name { + meta = &files[i] + break + } + } + if meta == nil { + http.NotFound(rec, req) + return + } + + f, fi, err := store.Open(correlationID, name) + if err != nil { + gologger.Debug().Msgf("Could not open uploaded file %s/%s: %s\n", correlationID, name, err) + meta = nil // served nothing, so the record must not claim a hit + http.NotFound(rec, req) + return + } + defer f.Close() + + // Always octet-stream with an attachment disposition: DTD, XSLT and JNDI + // consumers ignore content type entirely, so nothing is lost for the + // intended use, while the server never renders client-supplied HTML or SVG + // on its own domain. + rec.Header().Set("Content-Type", "application/octet-stream") + rec.Header().Set("Content-Disposition", mime.FormatMediaType("attachment", map[string]string{"filename": name})) + rec.Header().Set("X-Content-Type-Options", "nosniff") + if !h.options.NoVersionHeader { + rec.Header().Set("X-Interactsh-Version", h.options.Version) + } + + // ServeContent handles Range and conditional requests. The empty name + // argument keeps it from re-deriving a content type from the extension. + http.ServeContent(rec, req, "", fi.ModTime(), f) +} + +// hostedFetchRecorder passes writes straight through to the real +// ResponseWriter while noting what the response actually was, so the stored +// interaction can state it rather than assume it. ServeContent answers a +// conditional request with 304 and a ranged one with 206, and a record that +// claimed 200 with the full length would be evidence of a delivery that never +// happened. +type hostedFetchRecorder struct { + http.ResponseWriter + status int + written int64 +} + +func (r *hostedFetchRecorder) WriteHeader(code int) { + if r.status == 0 { + r.status = code + } + r.ResponseWriter.WriteHeader(code) +} + +func (r *hostedFetchRecorder) Write(b []byte) (int, error) { + if r.status == 0 { + r.status = http.StatusOK + } + n, err := r.ResponseWriter.Write(b) + r.written += int64(n) + return n, err +} + +// statusCode reports the status sent, defaulting to 200 for a handler that +// wrote neither a header nor a body. +func (r *hostedFetchRecorder) statusCode() int { + if r.status == 0 { + return http.StatusOK + } + return r.status +} + +// recordHostedFetch stores an interaction for any request to the /f/ subtree, +// hit or miss, with the response body replaced by a summary. +// +// A miss matters as much as a hit: it is how the operator tells "the target +// never fetched the payload" from "the target fetched a name I am not hosting", +// or from a fetch that arrived after the file had expired. meta is nil for +// every miss, which is what selects the summary. +func (h *HTTPServer) recordHostedFetch(req *http.Request, uniqueID, fullID string, meta *storage.UploadedFile, rec *hostedFetchRecorder) { + if uniqueID == "" { + // Nothing to attribute it to: interactions are indexed by correlation id, + // and handleInteraction slices one out of uniqueID unconditionally. + return + } + // Counted where it is recorded, so /metrics and the interaction stream + // cannot disagree about what arrived. + atomic.AddUint64(&h.options.Stats.Http, 1) + + var host string + if originIP := req.Header.Get(h.options.OriginIPHeader); originIP != "" { + host = originIP + } else { + host, _, _ = net.SplitHostPort(req.RemoteAddr) + } + + // Request dumped without its body: a file fetch is a GET, and an attacker + // controlling the body must not be able to inflate the stored record. + reqDump, _ := httputil.DumpRequest(req, false) + + status := rec.statusCode() + var resp strings.Builder + fmt.Fprintf(&resp, "HTTP/1.1 %d %s\r\n", status, http.StatusText(status)) + if meta == nil { + fmt.Fprintf(&resp, "\r\n[no hosted file for %q on this session]\n", req.URL.Path) + h.handleInteraction(req, uniqueID, fullID, string(reqDump), resp.String(), host) + return + } + fmt.Fprintf(&resp, "Content-Type: application/octet-stream\r\n") + fmt.Fprintf(&resp, "Content-Disposition: attachment; filename=%q\r\n", meta.Name) + fmt.Fprintf(&resp, "Content-Length: %d\r\n\r\n", rec.written) + // The byte count is what ServeContent actually wrote, so a 304 records as + // zero bytes and a 206 as the size of the range. + fmt.Fprintf(&resp, "[body elided: %d of %d bytes of uploaded file %q, sha256 %s]\n", + rec.written, meta.Size, meta.Name, meta.SHA256) + + h.handleInteraction(req, uniqueID, fullID, string(reqDump), resp.String(), host) +} + +// recordUploadProbe records an interaction for a request to /upload that no +// legitimate client could have sent, so that a target poking at the endpoint +// reaches the operator's stream instead of vanishing behind a 401 or a 501. +// +// The request body is summarised rather than stored. It is attacker-controlled +// and may be megabytes -- persisting it is precisely what keeping /upload off the +// logger middleware avoids -- but its size is evidence worth keeping. +// +// status and body describe the reply the caller is about to send, so the stored +// record cannot drift from what the target actually received. +func (h *HTTPServer) recordUploadProbe(req *http.Request, status int, body string) { + uniqueID, fullID := h.options.extractCorrelationID(req.Host) + if uniqueID == "" { + // Nothing to attribute it to: interactions are indexed by correlation id, + // and handleInteraction slices one out of uniqueID unconditionally. + return + } + atomic.AddUint64(&h.options.Stats.Http, 1) + + var host string + if originIP := req.Header.Get(h.options.OriginIPHeader); originIP != "" { + host = originIP + } else { + host, _, _ = net.SplitHostPort(req.RemoteAddr) + } + + reqDump, _ := httputil.DumpRequest(req, false) + reqString := string(reqDump) + if req.ContentLength > 0 { + reqString += fmt.Sprintf("[request body elided: %d bytes]\n", req.ContentLength) + } + + var resp strings.Builder + fmt.Fprintf(&resp, "HTTP/1.1 %d %s\r\n", status, http.StatusText(status)) + if body != "" { + // Mirrors jsonBody, which is what the caller writes. + encoded, err := json.Marshal(map[string]interface{}{"error": body}) + if err != nil { + return + } + fmt.Fprintf(&resp, "Content-Type: application/json; charset=utf-8\r\n") + fmt.Fprintf(&resp, "X-Content-Type-Options: nosniff\r\n") + fmt.Fprintf(&resp, "Content-Length: %d\r\n\r\n%s\n", len(encoded)+1, encoded) + } else { + resp.WriteString("\r\n") + } + + h.handleInteraction(req, uniqueID, fullID, reqString, resp.String(), host) +} + +// writeUploadError maps a failure to a status code the client can act on. +func (h *HTTPServer) writeUploadError(w http.ResponseWriter, err error) { + switch { + case errors.Is(err, storage.ErrCorrelationIdNotFound): + jsonError(w, "unknown correlation-id", http.StatusNotFound) + return + case errors.Is(err, storage.ErrInvalidSecretKey): + // Distinct from 400 so the client can tell "wrong session" apart from + // "malformed request". + jsonError(w, "invalid secret key for correlation-id", http.StatusForbidden) + return + } + + var ue *UploadError + if errors.As(err, &ue) { + switch ue.Kind { + case UploadErrBadName: + jsonError(w, ue.Error(), http.StatusBadRequest) + case UploadErrTooLarge, UploadErrTooManyFiles: + jsonError(w, ue.Error(), http.StatusRequestEntityTooLarge) + case UploadErrOutOfSpace: + jsonError(w, ue.Error(), http.StatusInsufficientStorage) + default: + jsonError(w, ue.Error(), http.StatusBadRequest) + } + return + } + + gologger.Warning().Msgf("Could not store uploaded files: %s\n", err) + jsonError(w, "could not store uploaded files", http.StatusInternalServerError) +} diff --git a/pkg/server/upload_handler_test.go b/pkg/server/upload_handler_test.go new file mode 100644 index 00000000..ad9ebd7c --- /dev/null +++ b/pkg/server/upload_handler_test.go @@ -0,0 +1,321 @@ +package server + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/base64" + "encoding/json" + "encoding/pem" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/google/uuid" + "github.com/projectdiscovery/interactsh/pkg/settings" + "github.com/projectdiscovery/interactsh/pkg/storage" + "github.com/rs/xid" + "github.com/stretchr/testify/require" +) + +// testPublicKey returns a base64-encoded PEM RSA public key accepted by +// Storage.SetIDPublicKey. +func testPublicKey(t *testing.T) string { + t.Helper() + + priv, err := rsa.GenerateKey(rand.Reader, 2048) + require.NoError(t, err, "could not generate rsa key") + + pubkeyBytes, err := x509.MarshalPKIXPublicKey(priv.Public()) + require.NoError(t, err, "could not marshal public key") + + pubkeyPem := pem.EncodeToMemory(&pem.Block{Type: "RSA PUBLIC KEY", Bytes: pubkeyBytes}) + return base64.StdEncoding.EncodeToString(pubkeyPem) +} + +// uploadTestServer returns an HTTPServer with uploads enabled and a registered +// session, plus that session's correlation ID and secret. +func uploadTestServer(t *testing.T, enableUploads bool) (*HTTPServer, string, string) { + t.Helper() + + store, err := storage.New(&storage.Options{EvictionTTL: time.Hour}) + require.NoError(t, err) + t.Cleanup(func() { _ = store.Close() }) + + options := &Options{ + Storage: store, + Stats: &Metrics{}, + CorrelationIdLength: settings.CorrelationIdLengthDefault, + CorrelationIdNonceLength: settings.CorrelationIdNonceLengthDefault, + UploadDirectory: t.TempDir(), + UploadMaxFileSize: 1024, + UploadMaxFiles: 3, + UploadMaxTotalSize: 1 << 20, + UploadTTL: time.Hour, + } + if enableUploads { + us, err := NewUploadStore(options) + require.NoError(t, err) + t.Cleanup(func() { _ = us.Close() }) + options.Upload = true + options.UploadStore = us + } + + h := &HTTPServer{options: options} + + secret := uuid.New().String() + correlationID := xid.New().String() + require.NoError(t, store.SetIDPublicKey(correlationID, secret, testPublicKey(t))) + + return h, correlationID, secret +} + +func uploadBody(t *testing.T, correlationID, secret string, files map[string][]byte) string { + t.Helper() + + req := UploadRequest{CorrelationID: correlationID, SecretKey: secret} + for name, data := range files { + req.Files = append(req.Files, UploadFileRequest{ + Name: name, + Data: base64.StdEncoding.EncodeToString(data), + }) + } + encoded, err := json.Marshal(req) + require.NoError(t, err) + return string(encoded) +} + +func doUpload(t *testing.T, h *HTTPServer, body string) *http.Response { + t.Helper() + req := httptest.NewRequest(http.MethodPost, "http://example.com/upload", strings.NewReader(body)) + w := httptest.NewRecorder() + h.uploadHandler(w, req) + return w.Result() +} + +func TestUploadHandler(t *testing.T) { + t.Run("stores a file", func(t *testing.T) { + h, id, secret := uploadTestServer(t, true) + content := []byte("") + + resp := doUpload(t, h, uploadBody(t, id, secret, map[string][]byte{"evil.dtd": content})) + require.Equal(t, http.StatusOK, resp.StatusCode) + + out := &UploadResponse{} + require.NoError(t, json.NewDecoder(resp.Body).Decode(out)) + require.Len(t, out.Files, 1) + require.Equal(t, "evil.dtd", out.Files[0].Name) + require.EqualValues(t, len(content), out.Files[0].Size) + require.Equal(t, "/f/evil.dtd", out.Files[0].HTTPPath) + require.Equal(t, "/"+uploadsDirName+"/"+id+"/evil.dtd", out.Files[0].FTPPath) + + files, ok := h.options.UploadStorage().ListUploads(id) + require.True(t, ok) + require.Len(t, files, 1, "metadata should be recorded against the session") + }) + + t.Run("501 when uploads disabled", func(t *testing.T) { + h, id, secret := uploadTestServer(t, false) + resp := doUpload(t, h, uploadBody(t, id, secret, map[string][]byte{"a.dtd": []byte("x")})) + require.Equal(t, http.StatusNotImplemented, resp.StatusCode, + "the client uses 501 as the capability signal") + }) + + t.Run("404 for unknown correlation id", func(t *testing.T) { + h, _, secret := uploadTestServer(t, true) + resp := doUpload(t, h, uploadBody(t, xid.New().String(), secret, map[string][]byte{"a.dtd": []byte("x")})) + require.Equal(t, http.StatusNotFound, resp.StatusCode) + }) + + t.Run("403 for wrong secret", func(t *testing.T) { + h, id, _ := uploadTestServer(t, true) + resp := doUpload(t, h, uploadBody(t, id, uuid.New().String(), map[string][]byte{"a.dtd": []byte("x")})) + require.Equal(t, http.StatusForbidden, resp.StatusCode) + }) + + t.Run("413 for oversize file", func(t *testing.T) { + h, id, secret := uploadTestServer(t, true) + resp := doUpload(t, h, uploadBody(t, id, secret, map[string][]byte{"big.bin": make([]byte, 2048)})) + require.Equal(t, http.StatusRequestEntityTooLarge, resp.StatusCode) + }) + + t.Run("413 for too many files in one request", func(t *testing.T) { + h, id, secret := uploadTestServer(t, true) + files := map[string][]byte{} + for i := 0; i < 5; i++ { + files[fmt.Sprintf("f%d.dtd", i)] = []byte("x") + } + resp := doUpload(t, h, uploadBody(t, id, secret, files)) + require.Equal(t, http.StatusRequestEntityTooLarge, resp.StatusCode) + }) + + t.Run("413 when session quota is reached across requests", func(t *testing.T) { + h, id, secret := uploadTestServer(t, true) + for i := 0; i < 3; i++ { + resp := doUpload(t, h, uploadBody(t, id, secret, + map[string][]byte{fmt.Sprintf("f%d.dtd", i): []byte("x")})) + require.Equal(t, http.StatusOK, resp.StatusCode) + } + resp := doUpload(t, h, uploadBody(t, id, secret, map[string][]byte{"overflow.dtd": []byte("x")})) + require.Equal(t, http.StatusRequestEntityTooLarge, resp.StatusCode) + }) + + t.Run("replacing a name does not consume a new slot", func(t *testing.T) { + h, id, secret := uploadTestServer(t, true) + for i := 0; i < 3; i++ { + resp := doUpload(t, h, uploadBody(t, id, secret, + map[string][]byte{fmt.Sprintf("f%d.dtd", i): []byte("x")})) + require.Equal(t, http.StatusOK, resp.StatusCode) + } + resp := doUpload(t, h, uploadBody(t, id, secret, map[string][]byte{"f0.dtd": []byte("replaced")})) + require.Equal(t, http.StatusOK, resp.StatusCode, "overwriting an existing name must be allowed at quota") + + files, _ := h.options.UploadStorage().ListUploads(id) + require.Len(t, files, 3) + }) + + t.Run("400 for traversal name", func(t *testing.T) { + h, id, secret := uploadTestServer(t, true) + for _, name := range []string{"../evil", "a/b", "/etc/passwd", `..\evil`, ""} { + resp := doUpload(t, h, uploadBody(t, id, secret, map[string][]byte{name: []byte("x")})) + require.Equal(t, http.StatusBadRequest, resp.StatusCode, "name %q must be rejected", name) + } + }) + + t.Run("400 for malformed base64", func(t *testing.T) { + h, id, secret := uploadTestServer(t, true) + body := fmt.Sprintf(`{"correlation-id":%q,"secret-key":%q,"files":[{"name":"a.dtd","data":"!!!not base64!!!"}]}`, id, secret) + resp := doUpload(t, h, body) + require.Equal(t, http.StatusBadRequest, resp.StatusCode) + }) + + t.Run("400 for duplicate names in one request", func(t *testing.T) { + h, id, secret := uploadTestServer(t, true) + body := fmt.Sprintf( + `{"correlation-id":%q,"secret-key":%q,"files":[{"name":"a.dtd","data":"eA=="},{"name":"a.dtd","data":"eQ=="}]}`, + id, secret) + resp := doUpload(t, h, body) + require.Equal(t, http.StatusBadRequest, resp.StatusCode) + }) + + t.Run("400 for empty file list", func(t *testing.T) { + h, id, secret := uploadTestServer(t, true) + resp := doUpload(t, h, uploadBody(t, id, secret, nil)) + require.Equal(t, http.StatusBadRequest, resp.StatusCode) + }) + + t.Run("405 for non-POST", func(t *testing.T) { + h, _, _ := uploadTestServer(t, true) + req := httptest.NewRequest(http.MethodGet, "http://example.com/upload", nil) + w := httptest.NewRecorder() + h.uploadHandler(w, req) + require.Equal(t, http.StatusMethodNotAllowed, w.Result().StatusCode) + }) + + // A failure partway through must not leave the session holding some files. + t.Run("rejects the whole request if any file is invalid", func(t *testing.T) { + h, id, secret := uploadTestServer(t, true) + body := fmt.Sprintf( + `{"correlation-id":%q,"secret-key":%q,"files":[{"name":"good.dtd","data":"eA=="},{"name":"../bad","data":"eQ=="}]}`, + id, secret) + resp := doUpload(t, h, body) + require.Equal(t, http.StatusBadRequest, resp.StatusCode) + + files, _ := h.options.UploadStorage().ListUploads(id) + require.Empty(t, files, "no file should have been stored") + }) +} + +func TestRegisterAdvertisesCapabilities(t *testing.T) { + t.Run("uploads enabled", func(t *testing.T) { + h, _, _ := uploadTestServer(t, true) + h.options.Ftp = true + // Set at startup once the FTP and upload roots are known to be the same + // directory; without it FTP is not advertised for hosted files. + h.options.FTPServesUploads = true + + body := fmt.Sprintf(`{"public-key":%q,"secret-key":%q,"correlation-id":%q}`, + testPublicKey(t), uuid.New().String(), xid.New().String()) + w := httptest.NewRecorder() + h.registerHandler(w, httptest.NewRequest(http.MethodPost, "http://example.com/register", strings.NewReader(body))) + + resp := w.Result() + require.Equal(t, http.StatusOK, resp.StatusCode) + + out := &RegisterResponse{} + require.NoError(t, json.NewDecoder(resp.Body).Decode(out)) + require.Equal(t, "registration successful", out.Message, + "legacy clients match on this exact string") + require.NotNil(t, out.Capabilities) + require.True(t, out.Capabilities.Upload) + require.True(t, out.Capabilities.FTP) + require.EqualValues(t, 1024, out.Capabilities.UploadMaxFileSize) + require.Equal(t, 3, out.Capabilities.UploadMaxFiles) + }) + + // An ftp:// URL that resolves to nothing is worse than no URL: the target + // follows it, gets a 550, and the operator reads the silence as "not + // vulnerable". So a split root must not be advertised. + t.Run("ftp not advertised when it cannot reach the uploads", func(t *testing.T) { + h, _, _ := uploadTestServer(t, true) + h.options.Ftp = true + h.options.FTPServesUploads = false + + body := fmt.Sprintf(`{"public-key":%q,"secret-key":%q,"correlation-id":%q}`, + testPublicKey(t), uuid.New().String(), xid.New().String()) + w := httptest.NewRecorder() + h.registerHandler(w, httptest.NewRequest(http.MethodPost, "http://example.com/register", strings.NewReader(body))) + + out := &RegisterResponse{} + require.NoError(t, json.NewDecoder(w.Result().Body).Decode(out)) + require.NotNil(t, out.Capabilities) + require.True(t, out.Capabilities.Upload, "http hosting still works") + require.False(t, out.Capabilities.FTP, "a split root must not be advertised as ftp-capable") + }) + + t.Run("uploads disabled", func(t *testing.T) { + h, _, _ := uploadTestServer(t, false) + + body := fmt.Sprintf(`{"public-key":%q,"secret-key":%q,"correlation-id":%q}`, + testPublicKey(t), uuid.New().String(), xid.New().String()) + w := httptest.NewRecorder() + h.registerHandler(w, httptest.NewRequest(http.MethodPost, "http://example.com/register", strings.NewReader(body))) + + out := &RegisterResponse{} + require.NoError(t, json.NewDecoder(w.Result().Body).Decode(out)) + require.NotNil(t, out.Capabilities) + require.False(t, out.Capabilities.Upload) + }) +} + +// TestDeregisterRemovesFilesSynchronously pins the ordering guarantee that a +// client which has just deregistered can no longer fetch its hosted files. +// +// The eviction hook reached through RemoveID only queues the directory, so a +// queued-only deregistration would leave the files readable until a background +// goroutine got to them. The fixture deliberately never starts that goroutine, +// which is what makes the distinction observable here. +func TestDeregisterRemovesFilesSynchronously(t *testing.T) { + h, id, secret := uploadTestServer(t, true) + + resp := doUpload(t, h, uploadBody(t, id, secret, map[string][]byte{"evil.dtd": []byte("payload")})) + require.Equal(t, http.StatusOK, resp.StatusCode) + + dir := h.options.UploadStore.sessionDir(id) + require.DirExists(t, dir, "upload should have created the session directory") + + body := fmt.Sprintf(`{"secret-key":%q,"correlation-id":%q}`, secret, id) + w := httptest.NewRecorder() + h.deregisterHandler(w, httptest.NewRequest(http.MethodPost, "http://example.com/deregister", strings.NewReader(body))) + require.Equal(t, http.StatusOK, w.Result().StatusCode) + + // No sleep and no polling: the files must be gone by the time the handler + // has returned, not merely scheduled for removal. + require.NoDirExists(t, dir, "deregistration must remove hosted files before returning") + + _, _, err := h.options.UploadStore.Open(id, "evil.dtd") + require.Error(t, err, "file must not be servable after deregistration") +} diff --git a/pkg/server/upload_probe_test.go b/pkg/server/upload_probe_test.go new file mode 100644 index 00000000..ccd146e4 --- /dev/null +++ b/pkg/server/upload_probe_test.go @@ -0,0 +1,122 @@ +package server + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// payloadHost builds a host of the shape a client payload URL would use. +func payloadHost(correlationID string) string { + return correlationID + strings.Repeat("a", 13) + ".oast.test" +} + +// probeUpload issues an upload request the way a target would: against a payload +// host, with whatever token the caller supplies. +func probeUpload(t *testing.T, h *HTTPServer, host, token, body string) *http.Response { + t.Helper() + req := httptest.NewRequest(http.MethodPost, "/upload", strings.NewReader(body)) + req.Host = host + req.RemoteAddr = "203.0.113.9:4444" + if token != "" { + req.Header.Set("Authorization", token) + } + w := httptest.NewRecorder() + h.uploadHandler(w, req) + return w.Result() +} + +// storedInteractions returns the interaction records held for a correlation id. +func storedInteractions(t *testing.T, h *HTTPServer, correlationID string) []string { + t.Helper() + item, err := h.options.Storage.GetCacheItem(correlationID) + require.NoError(t, err) + return item.Data +} + +// A legitimate client never reaches /upload unauthenticated, nor at all on a +// server whose advertised capabilities say uploads are off. So anything arriving +// there is a target probing the endpoint, and the operator should see it. +func TestUploadProbeRecording(t *testing.T) { + payload := `{"correlation-id":"probe","secret-key":"x","files":[{"name":"a.dtd","data":"QUFBQUFB"}]}` + + t.Run("unauthenticated probe is recorded and refused", func(t *testing.T) { + h, id, _ := uploadTestServer(t, true) + h.options.Auth = true + h.options.Token = "server-token" + + resp := probeUpload(t, h, payloadHost(id), "", payload) + require.Equal(t, http.StatusUnauthorized, resp.StatusCode) + + data := storedInteractions(t, h, id) + require.Len(t, data, 1, "a probe of /upload must reach the operator's stream") + + record := &Interaction{} + require.NoError(t, json.Unmarshal([]byte(data[0]), record)) + // UniqueID carries the nonce as well; the record is filed under the id. + require.True(t, strings.HasPrefix(record.UniqueID, id), "got %q", record.UniqueID) + require.Contains(t, record.RawRequest, "POST /upload") + require.Contains(t, record.RawResponse, "401 Unauthorized", + "the record must state the status the target actually received") + require.Equal(t, uint64(1), h.options.Stats.Http) + }) + + t.Run("probe against a server without uploads is recorded", func(t *testing.T) { + h, id, _ := uploadTestServer(t, false) + + resp := probeUpload(t, h, payloadHost(id), "", payload) + require.Equal(t, http.StatusNotImplemented, resp.StatusCode) + + data := storedInteractions(t, h, id) + require.Len(t, data, 1) + record := &Interaction{} + require.NoError(t, json.Unmarshal([]byte(data[0]), record)) + require.Contains(t, record.RawResponse, "501 Not Implemented") + require.Contains(t, record.RawResponse, "file upload is not enabled on this server") + }) + + t.Run("the request body is summarised, never stored", func(t *testing.T) { + h, id, _ := uploadTestServer(t, false) + big := strings.Repeat("Q", 200000) + body := `{"files":[{"name":"a.dtd","data":"` + big + `"}]}` + + resp := probeUpload(t, h, payloadHost(id), "", body) + require.Equal(t, http.StatusNotImplemented, resp.StatusCode) + + record := &Interaction{} + require.NoError(t, json.Unmarshal([]byte(storedInteractions(t, h, id)[0]), record)) + require.NotContains(t, record.RawRequest, big, + "an attacker-controlled body must not be persisted") + require.Contains(t, record.RawRequest, fmt.Sprintf("[request body elided: %d bytes]", len(body))) + require.Less(t, len(record.RawRequest), 1000, "the record must not scale with the body") + }) + + t.Run("an authenticated upload is not recorded as an interaction", func(t *testing.T) { + h, id, secret := uploadTestServer(t, true) + h.options.Auth = true + h.options.Token = "server-token" + + resp := probeUpload(t, h, payloadHost(id), "server-token", + uploadBody(t, id, secret, map[string][]byte{"evil.dtd": []byte("payload")})) + require.Equal(t, http.StatusOK, resp.StatusCode) + + require.Empty(t, storedInteractions(t, h, id), + "the operator's own upload must not be attributed to the target") + require.Zero(t, h.options.Stats.Http) + }) + + t.Run("a probe with no correlation id is dropped, not recorded", func(t *testing.T) { + h, id, _ := uploadTestServer(t, false) + + // No panic: handleInteraction slices a correlation id out of uniqueID. + resp := probeUpload(t, h, "example.com", "", payload) + require.Equal(t, http.StatusNotImplemented, resp.StatusCode) + require.Empty(t, storedInteractions(t, h, id)) + require.Zero(t, h.options.Stats.Http) + }) +} diff --git a/pkg/server/upload_record_test.go b/pkg/server/upload_record_test.go new file mode 100644 index 00000000..120e28c2 --- /dev/null +++ b/pkg/server/upload_record_test.go @@ -0,0 +1,162 @@ +package server + +import ( + "fmt" + "net/http" + "net/http/httptest" + "testing" + + jsoniter "github.com/json-iterator/go" + "github.com/stretchr/testify/require" +) + +// serveRequestWithHeaders is serveRequest with request headers, for the +// conditional and ranged cases ServeContent handles on its own. +func serveRequestWithHeaders(t *testing.T, h *HTTPServer, host, path string, hdr map[string]string) *http.Response { + t.Helper() + req := httptest.NewRequest(http.MethodGet, path, nil) + req.Host = host + req.RemoteAddr = "203.0.113.7:5555" + for k, v := range hdr { + req.Header.Set(k, v) + } + w := httptest.NewRecorder() + h.serveUploadedFile(w, req) + return w.Result() +} + +// records returns the interaction records stored for a session, newest last. +func records(t *testing.T, h *HTTPServer, correlationID string) []*Interaction { + t.Helper() + item, err := h.options.Storage.GetCacheItem(correlationID) + require.NoError(t, err) + out := make([]*Interaction, 0, len(item.Data)) + for _, raw := range item.Data { + record := &Interaction{} + require.NoError(t, jsoniter.Unmarshal([]byte(raw), record)) + out = append(out, record) + } + return out +} + +// A fetch that misses is evidence too: it separates "the target never came" from +// "the target asked for a name I am not hosting". +func TestServeUploadedFileRecordsMisses(t *testing.T) { + setup := func(t *testing.T) (*HTTPServer, string) { + t.Helper() + h, id, secret := uploadTestServer(t, true) + require.Equal(t, http.StatusOK, + doUpload(t, h, uploadBody(t, id, secret, map[string][]byte{"evil.dtd": []byte("payload")})).StatusCode) + return h, id + } + + t.Run("a name that is not hosted", func(t *testing.T) { + h, id := setup(t) + resp := serveRequestWithHeaders(t, h, payloadHost(id), "/f/evil.dtd.txt", nil) + require.Equal(t, http.StatusNotFound, resp.StatusCode) + + got := records(t, h, id) + require.Len(t, got, 1, "a miss must reach the operator") + require.Contains(t, got[0].RawRequest, "GET /f/evil.dtd.txt") + require.Contains(t, got[0].RawResponse, "404 Not Found") + require.Contains(t, got[0].RawResponse, `[no hosted file for "/f/evil.dtd.txt" on this session]`) + require.NotContains(t, got[0].RawResponse, "body elided", "nothing was served") + }) + + t.Run("a rejected file name", func(t *testing.T) { + h, id := setup(t) + require.Equal(t, http.StatusNotFound, + serveRequestWithHeaders(t, h, payloadHost(id), "/f/../../etc/passwd", nil).StatusCode) + require.Len(t, records(t, h, id), 1) + }) + + t.Run("a server with uploads disabled", func(t *testing.T) { + h, id, _ := uploadTestServer(t, false) + require.Equal(t, http.StatusNotFound, + serveRequestWithHeaders(t, h, payloadHost(id), "/f/evil.dtd", nil).StatusCode) + + got := records(t, h, id) + require.Len(t, got, 1, "a probe of /f/ is worth recording even with uploads off") + require.Contains(t, got[0].RawResponse, "404 Not Found") + }) + + t.Run("a host carrying no correlation id is dropped", func(t *testing.T) { + h, id := setup(t) + require.Equal(t, http.StatusNotFound, + serveRequestWithHeaders(t, h, "example.com", "/f/evil.dtd", nil).StatusCode) + require.Empty(t, records(t, h, id)) + require.Zero(t, h.options.Stats.Http, "nothing recorded, so nothing counted") + }) +} + +// The stored record must state what ServeContent actually sent. Claiming a full +// 200 for a 304 is evidence of a delivery that never happened. +func TestServeUploadedFileRecordsActualResponse(t *testing.T) { + content := []byte("PAYLOAD") + setup := func(t *testing.T) (*HTTPServer, string) { + t.Helper() + h, id, secret := uploadTestServer(t, true) + require.Equal(t, http.StatusOK, + doUpload(t, h, uploadBody(t, id, secret, map[string][]byte{"evil.dtd": content})).StatusCode) + return h, id + } + + t.Run("a conditional request answered 304 records zero bytes", func(t *testing.T) { + h, id := setup(t) + first := serveRequestWithHeaders(t, h, payloadHost(id), "/f/evil.dtd", nil) + lastMod := first.Header.Get("Last-Modified") + require.NotEmpty(t, lastMod) + + resp := serveRequestWithHeaders(t, h, payloadHost(id), "/f/evil.dtd", + map[string]string{"If-Modified-Since": lastMod}) + require.Equal(t, http.StatusNotModified, resp.StatusCode) + + got := records(t, h, id) + require.Len(t, got, 2) + require.Contains(t, got[1].RawResponse, "304 Not Modified") + require.Contains(t, got[1].RawResponse, fmt.Sprintf("[body elided: 0 of %d bytes", len(content))) + require.NotContains(t, got[1].RawResponse, "200 OK") + }) + + t.Run("a ranged request answered 206 records the range size", func(t *testing.T) { + h, id := setup(t) + resp := serveRequestWithHeaders(t, h, payloadHost(id), "/f/evil.dtd", + map[string]string{"Range": "bytes=0-1"}) + require.Equal(t, http.StatusPartialContent, resp.StatusCode) + + got := records(t, h, id) + require.Len(t, got, 1) + require.Contains(t, got[0].RawResponse, "206 Partial Content") + require.Contains(t, got[0].RawResponse, fmt.Sprintf("[body elided: 2 of %d bytes", len(content))) + }) + + t.Run("a plain fetch still records the full length", func(t *testing.T) { + h, id := setup(t) + require.Equal(t, http.StatusOK, + serveRequestWithHeaders(t, h, payloadHost(id), "/f/evil.dtd", nil).StatusCode) + + got := records(t, h, id) + require.Len(t, got, 1) + require.Contains(t, got[0].RawResponse, "200 OK") + require.Contains(t, got[0].RawResponse, fmt.Sprintf("Content-Length: %d", len(content))) + require.Contains(t, got[0].RawResponse, fmt.Sprintf("[body elided: %d of %d bytes", len(content), len(content))) + }) +} + +// /metrics and the interaction stream must not disagree about what arrived. +func TestServeUploadedFileMetricsMatchRecords(t *testing.T) { + h, id, secret := uploadTestServer(t, true) + require.Equal(t, http.StatusOK, + doUpload(t, h, uploadBody(t, id, secret, map[string][]byte{"evil.dtd": []byte("payload")})).StatusCode) + + // two hits, two misses, and one request that cannot be attributed + serveRequestWithHeaders(t, h, payloadHost(id), "/f/evil.dtd", nil) + serveRequestWithHeaders(t, h, payloadHost(id), "/f/evil.dtd", nil) + serveRequestWithHeaders(t, h, payloadHost(id), "/f/absent.dtd", nil) + serveRequestWithHeaders(t, h, payloadHost(id), "/f/also-absent.dtd", nil) + serveRequestWithHeaders(t, h, "example.com", "/f/evil.dtd", nil) + + require.Len(t, records(t, h, id), 4) + require.Equal(t, uint64(4), h.options.Stats.Http, + "the http counter must equal the interactions actually recorded") +} diff --git a/pkg/server/upload_serve_test.go b/pkg/server/upload_serve_test.go new file mode 100644 index 00000000..f263f4af --- /dev/null +++ b/pkg/server/upload_serve_test.go @@ -0,0 +1,256 @@ +package server + +import ( + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/projectdiscovery/interactsh/pkg/storage" + stringsutil "github.com/projectdiscovery/utils/strings" + "github.com/stretchr/testify/require" +) + +// serveRequest issues a GET for a hosted file against the given host. +func serveRequest(t *testing.T, h *HTTPServer, host, path string) *http.Response { + t.Helper() + // Path-only target, so the dumped request line matches what a real server + // sees rather than an absolute-URI form. + req := httptest.NewRequest(http.MethodGet, path, nil) + req.Host = host + req.RemoteAddr = "203.0.113.7:5555" + w := httptest.NewRecorder() + h.serveUploadedFile(w, req) + return w.Result() +} + +func TestServeUploadedFile(t *testing.T) { + content := []byte(``) + + setup := func(t *testing.T, name string) (*HTTPServer, string, string) { + t.Helper() + h, id, secret := uploadTestServer(t, true) + resp := doUpload(t, h, uploadBody(t, id, secret, map[string][]byte{name: content})) + require.Equal(t, http.StatusOK, resp.StatusCode) + return h, id, secret + } + + t.Run("serves the exact bytes with hardened headers", func(t *testing.T) { + h, id, _ := setup(t, "evil.dtd") + + resp := serveRequest(t, h, payloadHost(id), "/f/evil.dtd") + require.Equal(t, http.StatusOK, resp.StatusCode) + + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + require.Equal(t, content, body) + + require.Equal(t, "application/octet-stream", resp.Header.Get("Content-Type")) + require.Equal(t, `attachment; filename=evil.dtd`, resp.Header.Get("Content-Disposition")) + require.Equal(t, "nosniff", resp.Header.Get("X-Content-Type-Options")) + }) + + // The old design served from defaultHandler, where these two branches + // would have hijacked the request and returned reflection markup. + t.Run("xml and json names are not hijacked", func(t *testing.T) { + for _, name := range []string{"payload.xml", "payload.json"} { + h, id, _ := setup(t, name) + resp := serveRequest(t, h, payloadHost(id), "/f/"+name) + require.Equal(t, http.StatusOK, resp.StatusCode) + + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + require.Equal(t, content, body, "%s must return the file, not reflected markup", name) + } + }) + + // -dhr returns early for every request inside defaultHandler. + t.Run("default http response file does not shadow it", func(t *testing.T) { + h, id, _ := setup(t, "evil.dtd") + h.defaultResponse = "default response" + + resp := serveRequest(t, h, payloadHost(id), "/f/evil.dtd") + require.Equal(t, http.StatusOK, resp.StatusCode) + body, _ := io.ReadAll(resp.Body) + require.Equal(t, content, body) + }) + + // -dr lets query parameters inject headers on the /s/ static path. + t.Run("dynamic response cannot override the hardened headers", func(t *testing.T) { + h, id, _ := setup(t, "evil.dtd") + h.options.DynamicResp = true + + resp := serveRequest(t, h, payloadHost(id), "/f/evil.dtd?header=Content-Type:text/html&status=201") + require.Equal(t, http.StatusOK, resp.StatusCode, "status must not be attacker controlled") + require.Equal(t, "application/octet-stream", resp.Header.Get("Content-Type")) + }) + + t.Run("404s", func(t *testing.T) { + h, id, _ := setup(t, "evil.dtd") + + t.Run("host without a correlation id", func(t *testing.T) { + resp := serveRequest(t, h, "www.oast.test", "/f/evil.dtd") + require.Equal(t, http.StatusNotFound, resp.StatusCode) + }) + t.Run("unknown file", func(t *testing.T) { + resp := serveRequest(t, h, payloadHost(id), "/f/absent.dtd") + require.Equal(t, http.StatusNotFound, resp.StatusCode) + }) + t.Run("traversal", func(t *testing.T) { + resp := serveRequest(t, h, payloadHost(id), "/f/../../etc/passwd") + require.Equal(t, http.StatusNotFound, resp.StatusCode) + }) + t.Run("file owned by a different session", func(t *testing.T) { + other, otherID, otherSecret := uploadTestServer(t, true) + r := doUpload(t, other, uploadBody(t, otherID, otherSecret, map[string][]byte{"secret.dtd": []byte("theirs")})) + require.Equal(t, http.StatusOK, r.StatusCode) + + // Ask our server, using our session, for their file name. + resp := serveRequest(t, h, payloadHost(id), "/f/secret.dtd") + require.Equal(t, http.StatusNotFound, resp.StatusCode) + }) + t.Run("uploads disabled", func(t *testing.T) { + off, offID, _ := uploadTestServer(t, false) + resp := serveRequest(t, off, payloadHost(offID), "/f/evil.dtd") + require.Equal(t, http.StatusNotFound, resp.StatusCode) + }) + }) +} + +func TestServeUploadedFileRecordsInteraction(t *testing.T) { + h, id, secret := uploadTestServer(t, true) + content := []byte("dtd content") + require.Equal(t, http.StatusOK, + doUpload(t, h, uploadBody(t, id, secret, map[string][]byte{"evil.dtd": content})).StatusCode) + + resp := serveRequest(t, h, payloadHost(id), "/f/evil.dtd") + require.Equal(t, http.StatusOK, resp.StatusCode) + + // Read the raw buffer rather than GetInteractions: in memory mode the + // latter encrypts on the way out, and we want to inspect what was stored. + item, err := h.options.Storage.GetCacheItem(id) + require.NoError(t, err) + require.Len(t, item.Data, 1, "the fetch must be visible to the client that owns the session") + + record := &Interaction{} + require.NoError(t, json.Unmarshal([]byte(item.Data[0]), record)) + + require.Equal(t, "http", record.Protocol) + require.Contains(t, record.RawRequest, "GET /f/evil.dtd") + require.Equal(t, "203.0.113.7", record.RemoteAddress) + require.Contains(t, record.RawResponse, "200 OK") + require.Contains(t, record.RawResponse, "body elided") +} + +// The reason serving sits outside the logger middleware: routed through it, a +// fetch would retain a multiple of the file size in the session's interaction +// buffer. +func TestServeUploadedFileElidesBody(t *testing.T) { + h, id, secret := uploadTestServer(t, true) + h.options.UploadStore.maxFileSize = 1 << 20 + + // Bytes that JSON-escape badly, which is what drove the amplification. + content := make([]byte, 512*1024) + for i := range content { + content[i] = 0xff + } + require.Equal(t, http.StatusOK, + doUpload(t, h, uploadBody(t, id, secret, map[string][]byte{"big.bin": content})).StatusCode) + + for i := 0; i < 5; i++ { + require.Equal(t, http.StatusOK, serveRequest(t, h, payloadHost(id), "/f/big.bin").StatusCode) + } + + item, err := h.options.Storage.GetCacheItem(id) + require.NoError(t, err) + require.Len(t, item.Data, 5) + + var total int + for _, raw := range item.Data { + total += len(raw) + require.NotContains(t, raw, strings.Repeat("\\ufffd", 64), + "file bytes must not be copied into the interaction record") + } + t.Logf("5 fetches of a 512KiB file retained %d bytes of interaction data", total) + require.Less(t, total, 32*1024, + fmt.Sprintf("5 fetches of a 512KiB file retained %d bytes; body elision is not working", total)) +} + +func TestExtractCorrelationIDMatchesLogger(t *testing.T) { + options := &Options{CorrelationIdLength: 20, CorrelationIdNonceLength: 13} + id := "c6rj61aciaeutn2ae680" + nonce := "xk4tqy8pqhwty" + + cases := []string{ + id + nonce + ".oast.test", + id + nonce + ".sub.oast.test", + strings.ToUpper(id+nonce) + ".oast.test", + id + nonce + ".oast.test:8080", + "www.oast.test", + "", + id + ".oast.test", // too short to be a full unique id + } + + for _, host := range cases { + wantUnique, wantFull := loggerStyleExtract(options, host) + gotUnique, gotFull := options.extractCorrelationID(host) + require.Equal(t, wantUnique, gotUnique, "unique id mismatch for host %q", host) + require.Equal(t, wantFull, gotFull, "full id mismatch for host %q", host) + } +} + +// loggerStyleExtract reproduces the extraction the logger middleware performs, +// so the two implementations can be compared directly. +func loggerStyleExtract(options *Options, host string) (string, string) { + if hostOnly, _, err := net.SplitHostPort(host); err == nil { + host = hostOnly + } + parts := strings.Split(host, ".") + for i, part := range parts { + for partChunk := range stringsutil.SlideWithLength(part, options.GetIdLength()) { + normalized := strings.ToLower(partChunk) + if options.isCorrelationID(normalized) { + fullID := part + if i+1 <= len(parts) { + fullID = strings.Join(parts[:i+1], ".") + } + return normalized, fullID + } + } + } + return "", "" +} + +// noUploadStorage is a Storage without the UploadStorage capability, standing in +// for a shared backend such as Redis. Embedding the interface supplies every +// Storage method while deliberately omitting UpdateUploads and ListUploads. +type noUploadStorage struct{ storage.Storage } + +// TestUploadsDegradeWhenBackendCannotTrackThem covers the path a deployment on a +// shared storage backend would take. The server must decline uploads outright +// rather than advertise a capability it cannot honour, or panic reaching for a +// nil capability. +func TestUploadsDegradeWhenBackendCannotTrackThem(t *testing.T) { + h, id, secret := uploadTestServer(t, true) + require.True(t, h.capabilities().Upload, "precondition: a capable backend advertises uploads") + + resp := doUpload(t, h, uploadBody(t, id, secret, map[string][]byte{"evil.dtd": []byte("payload")})) + require.Equal(t, http.StatusOK, resp.StatusCode, "precondition: upload works before swapping the backend") + + // Swap in a backend that cannot track uploads. The files stay on disk, so + // this isolates the capability check from the storage contents. + h.options.Storage = noUploadStorage{h.options.Storage} + require.Nil(t, h.options.UploadStorage(), "backend must not satisfy UploadStorage") + + require.False(t, h.capabilities().Upload, "capability must not be advertised without backend support") + + resp = doUpload(t, h, uploadBody(t, id, secret, map[string][]byte{"other.dtd": []byte("x")})) + require.Equal(t, http.StatusNotImplemented, resp.StatusCode, "upload must report unsupported, not fail late") + + served := serveRequest(t, h, payloadHost(id), "/f/evil.dtd") + require.Equal(t, http.StatusNotFound, served.StatusCode, "serving must 404 rather than dereference a nil capability") +} diff --git a/pkg/server/upload_test.go b/pkg/server/upload_test.go new file mode 100644 index 00000000..99bd66b6 --- /dev/null +++ b/pkg/server/upload_test.go @@ -0,0 +1,363 @@ +package server + +import ( + "errors" + "io" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/projectdiscovery/interactsh/pkg/settings" + "github.com/rs/xid" + "github.com/stretchr/testify/require" +) + +func TestIsSafeUploadName(t *testing.T) { + valid := []string{ + "evil.dtd", "a", "a-b_c.1", "payload.xml", "x.tar.gz", + strings.Repeat("a", 128), + } + for _, name := range valid { + require.True(t, isSafeUploadName(name), "expected %q to be accepted", name) + } + + invalid := []string{ + "", ".", "..", "...", + "../evil", "../../etc/passwd", "a/b", "/etc/passwd", + `..\evil`, `a\b`, `C:\evil`, + ".hidden", "-leading-dash", + "evil\x00.dtd", "evil\n.dtd", "evil\r.dtd", "tab\t.dtd", + "space name.dtd", `quote".dtd`, "semi;colon", "pipe|d", + "percent%2e%2e", "unicodeé.dtd", "emoji\U0001f600", + strings.Repeat("a", 129), + } + for _, name := range invalid { + require.False(t, isSafeUploadName(name), "expected %q to be rejected", name) + } +} + +func newTestUploadStore(t *testing.T, tune func(*Options)) *UploadStore { + t.Helper() + + dir := t.TempDir() + options := &Options{ + CorrelationIdLength: settings.CorrelationIdLengthDefault, + UploadDirectory: dir, + UploadMaxFileSize: 1024, + UploadMaxFiles: 5, + UploadMaxTotalSize: 1 << 20, + UploadTTL: time.Hour, + } + if tune != nil { + tune(options) + } + + store, err := NewUploadStore(options) + require.NoError(t, err) + t.Cleanup(func() { _ = store.Close() }) + return store +} + +// newCorrelationID returns an id of the default configured length. +func newCorrelationID(t *testing.T) string { + t.Helper() + id := xid.New().String() + strings.Repeat("z", settings.CorrelationIdLengthDefault) + return id[:settings.CorrelationIdLengthDefault] +} + +func TestUploadStoreSaveOpen(t *testing.T) { + store := newTestUploadStore(t, nil) + id := newCorrelationID(t) + + t.Run("round trip", func(t *testing.T) { + content := []byte(``) + size, sum, err := store.Save(id, "evil.dtd", content, 0) + require.NoError(t, err) + require.EqualValues(t, len(content), size) + require.Len(t, sum, 64, "sha256 hex digest") + + f, fi, err := store.Open(id, "evil.dtd") + require.NoError(t, err) + defer f.Close() + + got, err := io.ReadAll(f) + require.NoError(t, err) + require.Equal(t, content, got) + require.EqualValues(t, len(content), fi.Size()) + }) + + t.Run("no temp files left visible", func(t *testing.T) { + entries, err := os.ReadDir(filepath.Join(store.sessionsRoot, id)) + require.NoError(t, err) + for _, e := range entries { + require.False(t, strings.HasPrefix(e.Name(), ".upload-"), + "partial write %q should have been renamed into place", e.Name()) + } + }) + + t.Run("rejects unsafe name", func(t *testing.T) { + _, _, err := store.Save(id, "../escape", []byte("x"), 0) + require.Error(t, err) + var ue *UploadError + require.True(t, errors.As(err, &ue)) + require.Equal(t, UploadErrBadName, ue.Kind) + }) + + t.Run("rejects empty file", func(t *testing.T) { + _, _, err := store.Save(id, "empty.dtd", nil, 0) + require.Error(t, err) + }) + + t.Run("rejects oversize file", func(t *testing.T) { + _, _, err := store.Save(id, "big.dtd", make([]byte, 2048), 0) + require.Error(t, err) + var ue *UploadError + require.True(t, errors.As(err, &ue)) + require.Equal(t, UploadErrTooLarge, ue.Kind) + }) + + t.Run("rejects invalid correlation id", func(t *testing.T) { + _, _, err := store.Save("../..", "x.dtd", []byte("x"), 0) + require.Error(t, err) + }) + + t.Run("open rejects traversal", func(t *testing.T) { + _, _, err := store.Open(id, "../../etc/passwd") + require.Error(t, err) + }) + + t.Run("open rejects unknown file", func(t *testing.T) { + _, _, err := store.Open(id, "absent.dtd") + require.Error(t, err) + }) +} + +func TestUploadStoreGlobalQuota(t *testing.T) { + store := newTestUploadStore(t, func(o *Options) { + o.UploadMaxFileSize = 1024 + o.UploadMaxTotalSize = 2048 + }) + + id := newCorrelationID(t) + _, _, err := store.Save(id, "a.bin", make([]byte, 1024), 0) + require.NoError(t, err) + _, _, err = store.Save(id, "b.bin", make([]byte, 1024), 0) + require.NoError(t, err) + + _, _, err = store.Save(id, "c.bin", make([]byte, 1024), 0) + require.Error(t, err, "third file should exhaust the global cap") + var ue *UploadError + require.True(t, errors.As(err, &ue)) + require.Equal(t, UploadErrOutOfSpace, ue.Kind) + + // Overwriting an existing name must account for the space it already holds + // rather than double-counting it. + _, _, err = store.Save(id, "a.bin", make([]byte, 1024), 1024) + require.NoError(t, err, "replacing a file of equal size must not exceed the cap") +} + +func TestUploadStoreRemoveSession(t *testing.T) { + store := newTestUploadStore(t, nil) + store.Start() + + id := newCorrelationID(t) + _, _, err := store.Save(id, "evil.dtd", []byte("payload"), 0) + require.NoError(t, err) + require.DirExists(t, filepath.Join(store.sessionsRoot, id)) + + store.RemoveSession(id) + require.Eventually(t, func() bool { + _, err := os.Stat(filepath.Join(store.sessionsRoot, id)) + return os.IsNotExist(err) + }, 2*time.Second, 10*time.Millisecond) + + // Idempotent: a second removal of a gone session must not error or panic. + store.RemoveSession(id) + store.RemoveSession(newCorrelationID(t)) + time.Sleep(50 * time.Millisecond) +} + +func TestUploadStoreSweep(t *testing.T) { + store := newTestUploadStore(t, func(o *Options) { o.UploadTTL = time.Hour }) + + stale := newCorrelationID(t) + fresh := newCorrelationID(t) + _, _, err := store.Save(stale, "old.dtd", []byte("old"), 0) + require.NoError(t, err) + _, _, err = store.Save(fresh, "new.dtd", []byte("new"), 0) + require.NoError(t, err) + + // Operator files sharing the root (the -ftp-dir case) must survive, including + // a directory whose name happens to look exactly like a correlation id: the + // sweeper only ever descends into its own directory, so a collision in the + // operator's namespace cannot cost them data. + sibling := filepath.Join(store.Root(), "operator-notes.txt") + require.NoError(t, os.WriteFile(sibling, []byte("keep me"), 0o600)) + siblingDir := filepath.Join(store.Root(), "operator-dir") + require.NoError(t, os.Mkdir(siblingDir, 0o700)) + require.NoError(t, os.Chtimes(siblingDir, time.Now().Add(-48*time.Hour), time.Now().Add(-48*time.Hour))) + lookalike := filepath.Join(store.Root(), newCorrelationID(t)) + require.NoError(t, os.Mkdir(lookalike, 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(lookalike, "payload.txt"), []byte("operator payload"), 0o600)) + require.NoError(t, os.Chtimes(lookalike, time.Now().Add(-48*time.Hour), time.Now().Add(-48*time.Hour))) + + old := time.Now().Add(-2 * time.Hour) + require.NoError(t, os.Chtimes(filepath.Join(store.sessionsRoot, stale), old, old)) + + store.sweep() + + require.NoDirExists(t, filepath.Join(store.sessionsRoot, stale), "expired session should be swept") + require.DirExists(t, filepath.Join(store.sessionsRoot, fresh), "live session should survive") + require.FileExists(t, sibling, "non-session file must not be swept") + require.DirExists(t, siblingDir, "non-session directory must not be swept even when old") + require.FileExists(t, filepath.Join(lookalike, "payload.txt"), + "an operator directory shaped like a correlation id must survive") +} + +func TestUploadStorePurgeOnStart(t *testing.T) { + dir := t.TempDir() + + orphan := newCorrelationID(t) + require.NoError(t, os.MkdirAll(filepath.Join(dir, uploadsDirName, orphan), 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(dir, uploadsDirName, orphan, "stale.dtd"), []byte("x"), 0o600)) + + keepFile := filepath.Join(dir, "index.html") + require.NoError(t, os.WriteFile(keepFile, []byte("operator content"), 0o600)) + keepDir := filepath.Join(dir, "assets") + require.NoError(t, os.Mkdir(keepDir, 0o700)) + // The startup purge is name-shape driven, so an operator directory that + // matches that shape has to be out of its reach structurally. + lookalike := filepath.Join(dir, newCorrelationID(t)) + require.NoError(t, os.Mkdir(lookalike, 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(lookalike, "payload.txt"), []byte("operator payload"), 0o600)) + + store := newTestUploadStore(t, func(o *Options) { o.UploadDirectory = dir }) + + require.NoDirExists(t, filepath.Join(dir, uploadsDirName, orphan), "orphaned session dir should be purged at startup") + require.FileExists(t, keepFile, "operator file must survive startup purge") + require.DirExists(t, keepDir, "operator directory must survive startup purge") + require.FileExists(t, filepath.Join(lookalike, "payload.txt"), + "an operator directory shaped like a correlation id must survive the purge") + _ = store +} + +func TestUploadStoreRootResolution(t *testing.T) { + t.Run("prefers upload directory", func(t *testing.T) { + up, ftp := t.TempDir(), t.TempDir() + store, err := NewUploadStore(&Options{ + CorrelationIdLength: settings.CorrelationIdLengthDefault, + UploadDirectory: up, FTPDirectory: ftp, UploadTTL: time.Hour, + }) + require.NoError(t, err) + defer store.Close() + require.Equal(t, up, store.Root()) + }) + + t.Run("falls back to ftp directory", func(t *testing.T) { + ftp := t.TempDir() + store, err := NewUploadStore(&Options{ + CorrelationIdLength: settings.CorrelationIdLengthDefault, + FTPDirectory: ftp, UploadTTL: time.Hour, + }) + require.NoError(t, err) + defer store.Close() + require.Equal(t, ftp, store.Root(), "sharing the FTP root is what makes FTP serving work") + }) + + t.Run("creates a shared ftp directory that does not exist yet", func(t *testing.T) { + // -ftp alone never required -ftp-dir to exist, so adopting it as the + // upload root must not turn a working deployment into a boot failure. + ftp := filepath.Join(t.TempDir(), "not-created-yet") + store, err := NewUploadStore(&Options{ + CorrelationIdLength: settings.CorrelationIdLengthDefault, + FTPDirectory: ftp, UploadTTL: time.Hour, + }) + require.NoError(t, err) + defer store.Close() + require.DirExists(t, ftp) + require.DirExists(t, filepath.Join(ftp, uploadsDirName)) + }) + + t.Run("sessions live under the uploads directory", func(t *testing.T) { + up := t.TempDir() + store, err := NewUploadStore(&Options{ + CorrelationIdLength: settings.CorrelationIdLengthDefault, + UploadDirectory: up, UploadTTL: time.Hour, UploadMaxFileSize: 1024, UploadMaxTotalSize: 4096, + }) + require.NoError(t, err) + defer store.Close() + + id := newCorrelationID(t) + _, _, err = store.Save(id, "evil.dtd", []byte("payload"), 0) + require.NoError(t, err) + require.FileExists(t, filepath.Join(up, uploadsDirName, id, "evil.dtd")) + require.NoDirExists(t, filepath.Join(up, id), "nothing of ours belongs at the root") + }) + + t.Run("falls back to a temp directory", func(t *testing.T) { + store, err := NewUploadStore(&Options{ + CorrelationIdLength: settings.CorrelationIdLengthDefault, + UploadTTL: time.Hour, + }) + require.NoError(t, err) + defer func() { + root := store.Root() + _ = store.Close() + _ = os.RemoveAll(root) + }() + require.DirExists(t, store.Root()) + }) +} + +// A directory can exist and still be unwritable, in which case MkdirAll succeeds +// and every upload fails later. The server must refuse to start instead. +func TestUploadStoreRejectsUnwritableDirectory(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("running as root bypasses the permission bits this test relies on") + } + + t.Run("read-only root", func(t *testing.T) { + dir := filepath.Join(t.TempDir(), "uploads") + require.NoError(t, os.Mkdir(dir, 0o500)) + t.Cleanup(func() { _ = os.Chmod(dir, 0o700) }) + + _, err := NewUploadStore(&Options{ + CorrelationIdLength: settings.CorrelationIdLengthDefault, + UploadDirectory: dir, UploadTTL: time.Hour, + }) + require.Error(t, err, "an unwritable upload directory must not start the server") + require.Contains(t, err.Error(), dir) + }) + + t.Run("sessions directory exists but is read-only", func(t *testing.T) { + // The case MkdirAll cannot catch: the directory it would have created is + // already there, so it returns nil and only a write reveals the problem. + dir := t.TempDir() + sessions := filepath.Join(dir, uploadsDirName) + require.NoError(t, os.Mkdir(sessions, 0o500)) + t.Cleanup(func() { _ = os.Chmod(sessions, 0o700) }) + + _, err := NewUploadStore(&Options{ + CorrelationIdLength: settings.CorrelationIdLengthDefault, + UploadDirectory: dir, UploadTTL: time.Hour, + }) + require.Error(t, err) + require.Contains(t, err.Error(), "not writable") + }) + + t.Run("writable root is accepted and left clean", func(t *testing.T) { + dir := t.TempDir() + store, err := NewUploadStore(&Options{ + CorrelationIdLength: settings.CorrelationIdLengthDefault, + UploadDirectory: dir, UploadTTL: time.Hour, + }) + require.NoError(t, err) + defer store.Close() + + entries, err := os.ReadDir(filepath.Join(dir, uploadsDirName)) + require.NoError(t, err) + require.Empty(t, entries, "the probe file must not be left behind") + }) +} diff --git a/pkg/server/util.go b/pkg/server/util.go index f7313bb3..8cc15295 100644 --- a/pkg/server/util.go +++ b/pkg/server/util.go @@ -7,6 +7,7 @@ import ( "strings" "github.com/projectdiscovery/gologger" + stringsutil "github.com/projectdiscovery/utils/strings" ) // Correlation ids are produced from xid (cidl prefix) and zbase32 (cidn suffix), @@ -85,3 +86,32 @@ func (options *Options) storeRootTLDInteraction(interaction *Interaction, id str gologger.Warning().Msgf("Could not store root tld %s interaction: %s\n", interaction.Protocol, err) } } + +// extractCorrelationID finds the first correlation id embedded in a host, +// returning the full unique id (correlation id plus nonce) and the host label +// prefix it was found in. +// +// This mirrors the extraction the logger middleware performs, so that a request +// served from the file route is attributed to the same session the logger would +// have attributed it to. TestExtractCorrelationIDMatchesLogger keeps the two in +// step. +func (options *Options) extractCorrelationID(host string) (uniqueID, fullID string) { + if hostOnly, _, err := net.SplitHostPort(host); err == nil { + host = hostOnly + } + parts := strings.Split(host, ".") + for i, part := range parts { + for chunk := range stringsutil.SlideWithLength(part, options.GetIdLength()) { + normalized := strings.ToLower(chunk) + if !options.isCorrelationID(normalized) { + continue + } + fullID := part + if i+1 <= len(parts) { + fullID = strings.Join(parts[:i+1], ".") + } + return normalized, fullID + } + } + return "", "" +} diff --git a/pkg/storage/error.go b/pkg/storage/error.go index b9a2aa22..23b97fd6 100644 --- a/pkg/storage/error.go +++ b/pkg/storage/error.go @@ -3,3 +3,9 @@ package storage import "github.com/projectdiscovery/utils/errkit" var ErrCorrelationIdNotFound = errkit.New("could not get correlation-id from cache") + +// ErrInvalidSecretKey is returned when the secret key presented for a +// correlation-id does not match the one stored at registration. It is +// distinguishable from ErrCorrelationIdNotFound so callers can tell "wrong +// session" apart from "no such session". +var ErrInvalidSecretKey = errkit.New("invalid secret key passed for correlation-id") diff --git a/pkg/storage/option.go b/pkg/storage/option.go index 1d91fb49..a72d6acd 100644 --- a/pkg/storage/option.go +++ b/pkg/storage/option.go @@ -10,14 +10,20 @@ const ( ) type Options struct { - DbPath string - EvictionTTL time.Duration - MaxSize int - MaxSharedInteractions int - EvictionStrategy EvictionStrategy + DbPath string + EvictionTTL time.Duration + MaxSize int + MaxSharedInteractions int + EvictionStrategy EvictionStrategy // OnRemoval is called for each client session removed from cache // (deregistration, TTL expiry, size eviction, or cache close). OnRemoval func() + // OnEviction is invoked when a correlation-id leaves the cache for any + // reason: explicit removal, TTL expiry, capacity eviction or Close. + // Unlike OnRemoval it fires for every entry, not just client sessions, and + // receives the evicted data so callers can release resources keyed off it. + // It runs on the cache's single event goroutine and must not block. + OnEviction func(correlationID string, data *CorrelationData) } func (options *Options) UseDisk() bool { diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 60e582b4..8b4885f6 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -15,3 +15,24 @@ type Storage interface { GetCacheItem(token string) (*CorrelationData, error) Close() error } + +// UploadStorage is the optional capability of tracking per-session uploaded +// file metadata, implemented by StorageDB. +// +// It is deliberately kept out of Storage. The uploaded bytes live on the local +// filesystem of a single server instance and the capacity quota is an +// in-process counter, so the feature is only coherent for instance-local +// backends. A shared backend such as Redis would let one instance advertise +// files whose bytes only exist on another instance's disk, so it does not +// implement this and file hosting is refused when it is selected. +type UploadStorage interface { + // UpdateUploads verifies the secret key for a correlation-id and then runs + // fn under that correlation-id's lock, replacing the upload metadata with + // whatever fn returns. Callers perform their disk writes inside fn so the + // quota check and the commit are atomic against concurrent uploads for the + // same session. fn is invoked at most once. + UpdateUploads(correlationID, secret string, fn func([]UploadedFile) ([]UploadedFile, error)) error + // ListUploads returns the upload metadata for a correlation-id, and whether + // the correlation-id is known at all. + ListUploads(correlationID string) ([]UploadedFile, bool) +} diff --git a/pkg/storage/storagedb.go b/pkg/storage/storagedb.go index 0bc65185..1cca55d1 100644 --- a/pkg/storage/storagedb.go +++ b/pkg/storage/storagedb.go @@ -83,13 +83,17 @@ func (s *StorageDB) onCacheRemoval(key cache.Key, value cache.Value) { if s.Options.UseDisk() && s.db != nil { _ = s.db.Delete([]byte(k), &opt.WriteOptions{}) } + cd, _ := value.(*CorrelationData) // Only fire for client sessions (entries with a SecretKey), // not for token/domain entries created via SetID. if s.Options.OnRemoval != nil { - if cd, ok := value.(*CorrelationData); ok && cd.SecretKey != "" { + if cd != nil && cd.SecretKey != "" { s.Options.OnRemoval() } } + if s.Options.OnEviction != nil { + s.Options.OnEviction(k, cd) + } } func (s *StorageDB) GetCacheMetrics() (*CacheMetrics, error) { @@ -461,6 +465,57 @@ func (s *StorageDB) RemoveID(correlationID, secret string) error { return nil } +// UpdateUploads verifies the secret key for a correlation-id and then runs fn +// while holding that correlation-id's lock, replacing the upload metadata with +// whatever fn returns. Callers perform their disk writes inside fn so that the +// quota check and the commit are atomic with respect to concurrent uploads +// against the same session. +func (s *StorageDB) UpdateUploads(correlationID, secret string, fn func([]UploadedFile) ([]UploadedFile, error)) error { + item, ok := s.cache.GetIfPresent(correlationID) + if !ok { + return ErrCorrelationIdNotFound + } + value, ok := item.(*CorrelationData) + if !ok { + return errors.New("invalid correlation-id cache value found") + } + if !strings.EqualFold(value.SecretKey, secret) { + return ErrInvalidSecretKey + } + value.Lock() + defer value.Unlock() + + updated, err := fn(value.Files) + if err != nil { + return err + } + value.Files = updated + return nil +} + +// ListUploads returns the upload metadata for a correlation-id. No secret is +// required: this backs the file-serving path, where possession of the +// correlation-id is the only credential. +func (s *StorageDB) ListUploads(correlationID string) ([]UploadedFile, bool) { + item, ok := s.cache.GetIfPresent(correlationID) + if !ok { + return nil, false + } + value, ok := item.(*CorrelationData) + if !ok { + return nil, false + } + value.Lock() + defer value.Unlock() + + if len(value.Files) == 0 { + return nil, true + } + files := make([]UploadedFile, len(value.Files)) + copy(files, value.Files) + return files, true +} + // GetCacheItem returns an item as is func (s *StorageDB) GetCacheItem(token string) (*CorrelationData, error) { item, ok := s.cache.GetIfPresent(token) @@ -529,3 +584,5 @@ func (s *StorageDB) Close() error { os.RemoveAll(s.dbpath), ) } + +var _ UploadStorage = (*StorageDB)(nil) diff --git a/pkg/storage/types.go b/pkg/storage/types.go index c7750cbc..0b8084bc 100644 --- a/pkg/storage/types.go +++ b/pkg/storage/types.go @@ -16,6 +16,16 @@ type CacheMetrics struct { EvictionCount uint64 `json:"eviction-count"` } +// UploadedFile is metadata for a file uploaded by the owner of a correlation-id. +// The bytes themselves live on disk, managed by the server's upload store; this +// record exists so that cache eviction and deregistration can drive file cleanup. +type UploadedFile struct { + Name string `json:"name"` + Size int64 `json:"size"` + SHA256 string `json:"sha256"` + Timestamp time.Time `json:"timestamp"` +} + // CorrelationData is the data for a correlation-id. type CorrelationData struct { sync.Mutex @@ -26,7 +36,11 @@ type CorrelationData struct { // AESKey is the AES encryption key in encrypted format. AESKeyEncrypted string `json:"aes-key"` // decrypted AES key for signing - AESKey []byte `json:"-"` + AESKey []byte `json:"-"` ReadOffsets map[string]int `json:"-"` LastSeen map[string]time.Time `json:"-"` + // Files is metadata for files uploaded against this correlation-id. + // Guarded by the embedded Mutex. Not persisted: only interaction blobs + // are written to disk, and uploads do not survive a restart. + Files []UploadedFile `json:"-"` } diff --git a/pkg/storage/uploads_test.go b/pkg/storage/uploads_test.go new file mode 100644 index 00000000..9aff7e42 --- /dev/null +++ b/pkg/storage/uploads_test.go @@ -0,0 +1,240 @@ +package storage + +import ( + "os" + "sync" + "testing" + "time" + + "github.com/google/uuid" + "github.com/rs/xid" + "github.com/stretchr/testify/require" +) + +func newUploadTestDB(t *testing.T, opts *Options) (*StorageDB, string, string) { + t.Helper() + + db, err := New(opts) + require.NoError(t, err) + t.Cleanup(func() { _ = db.Close() }) + + secret := uuid.New().String() + correlationID := xid.New().String() + _, pubKey := generateRSAKeyPair(t) + require.NoError(t, db.SetIDPublicKey(correlationID, secret, pubKey)) + + return db, correlationID, secret +} + +func TestUpdateUploads(t *testing.T) { + t.Run("stores metadata", func(t *testing.T) { + db, id, secret := newUploadTestDB(t, &Options{EvictionTTL: time.Hour}) + + err := db.UpdateUploads(id, secret, func(existing []UploadedFile) ([]UploadedFile, error) { + require.Empty(t, existing, "new session should start with no uploads") + return append(existing, UploadedFile{Name: "evil.dtd", Size: 42}), nil + }) + require.NoError(t, err) + + files, ok := db.ListUploads(id) + require.True(t, ok) + require.Len(t, files, 1) + require.Equal(t, "evil.dtd", files[0].Name) + }) + + t.Run("rejects wrong secret", func(t *testing.T) { + db, id, _ := newUploadTestDB(t, &Options{EvictionTTL: time.Hour}) + + called := false + err := db.UpdateUploads(id, uuid.New().String(), func(f []UploadedFile) ([]UploadedFile, error) { + called = true + return f, nil + }) + require.ErrorIs(t, err, ErrInvalidSecretKey) + require.False(t, called, "callback must not run for an unauthorised caller") + }) + + t.Run("rejects unknown correlation id", func(t *testing.T) { + db, _, secret := newUploadTestDB(t, &Options{EvictionTTL: time.Hour}) + + err := db.UpdateUploads(xid.New().String(), secret, func(f []UploadedFile) ([]UploadedFile, error) { + return f, nil + }) + require.ErrorIs(t, err, ErrCorrelationIdNotFound) + }) + + t.Run("callback error leaves metadata untouched", func(t *testing.T) { + db, id, secret := newUploadTestDB(t, &Options{EvictionTTL: time.Hour}) + + require.NoError(t, db.UpdateUploads(id, secret, func(f []UploadedFile) ([]UploadedFile, error) { + return append(f, UploadedFile{Name: "keep.dtd"}), nil + })) + + err := db.UpdateUploads(id, secret, func(f []UploadedFile) ([]UploadedFile, error) { + return append(f, UploadedFile{Name: "discard.dtd"}), os.ErrInvalid + }) + require.ErrorIs(t, err, os.ErrInvalid) + + files, ok := db.ListUploads(id) + require.True(t, ok) + require.Len(t, files, 1, "failed update must not commit") + require.Equal(t, "keep.dtd", files[0].Name) + }) + + // The quota check and the commit both run inside the callback under the + // correlation-id lock, so concurrent uploads to one session cannot both + // observe the same "slots remaining" and overshoot. + t.Run("concurrent updates respect a quota", func(t *testing.T) { + db, id, secret := newUploadTestDB(t, &Options{EvictionTTL: time.Hour}) + + const maxFiles = 5 + var wg sync.WaitGroup + for i := 0; i < 25; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + _ = db.UpdateUploads(id, secret, func(f []UploadedFile) ([]UploadedFile, error) { + if len(f) >= maxFiles { + return nil, os.ErrPermission + } + return append(f, UploadedFile{Name: xid.New().String()}), nil + }) + }(i) + } + wg.Wait() + + files, ok := db.ListUploads(id) + require.True(t, ok) + require.Len(t, files, maxFiles, "quota must hold under concurrency") + }) +} + +func TestListUploads(t *testing.T) { + db, id, _ := newUploadTestDB(t, &Options{EvictionTTL: time.Hour}) + + t.Run("known id with no uploads", func(t *testing.T) { + files, ok := db.ListUploads(id) + require.True(t, ok, "session exists") + require.Empty(t, files) + }) + + t.Run("unknown id", func(t *testing.T) { + files, ok := db.ListUploads(xid.New().String()) + require.False(t, ok) + require.Nil(t, files) + }) + + t.Run("returns a copy", func(t *testing.T) { + db, id, secret := newUploadTestDB(t, &Options{EvictionTTL: time.Hour}) + require.NoError(t, db.UpdateUploads(id, secret, func(f []UploadedFile) ([]UploadedFile, error) { + return append(f, UploadedFile{Name: "orig.dtd"}), nil + })) + + files, _ := db.ListUploads(id) + files[0].Name = "mutated.dtd" + + again, _ := db.ListUploads(id) + require.Equal(t, "orig.dtd", again[0].Name, "caller must not be able to mutate stored metadata") + }) +} + +// TestOnEvictionMemoryMode covers OnEviction firing in memory mode, which is +// the default and the mode upload cleanup most depends on: with no leveldb +// handle in play, the hook is the only signal that a session's files may go. +func TestOnEvictionMemoryMode(t *testing.T) { + t.Run("fires on RemoveID", func(t *testing.T) { + var mu sync.Mutex + var evicted []string + + db, id, secret := newUploadTestDB(t, &Options{ + EvictionTTL: time.Hour, + OnEviction: func(correlationID string, _ *CorrelationData) { + mu.Lock() + defer mu.Unlock() + evicted = append(evicted, correlationID) + }, + }) + + require.NoError(t, db.RemoveID(id, secret)) + require.Eventually(t, func() bool { + mu.Lock() + defer mu.Unlock() + return len(evicted) == 1 && evicted[0] == id + }, 2*time.Second, 10*time.Millisecond) + }) + + t.Run("fires on ttl expiry", func(t *testing.T) { + var mu sync.Mutex + var evicted []string + + db, id, _ := newUploadTestDB(t, &Options{ + EvictionTTL: 100 * time.Millisecond, + OnEviction: func(correlationID string, _ *CorrelationData) { + mu.Lock() + defer mu.Unlock() + evicted = append(evicted, correlationID) + }, + }) + + time.Sleep(200 * time.Millisecond) + // goburrow/cache has no janitor; expiry is only processed on cache + // activity, so poke it. + db.cache.GetIfPresent(id) + + require.Eventually(t, func() bool { + mu.Lock() + defer mu.Unlock() + return len(evicted) == 1 && evicted[0] == id + }, 2*time.Second, 10*time.Millisecond) + }) + + t.Run("fires on close", func(t *testing.T) { + var mu sync.Mutex + var evicted []string + + db, err := New(&Options{ + EvictionTTL: time.Hour, + OnEviction: func(correlationID string, _ *CorrelationData) { + mu.Lock() + defer mu.Unlock() + evicted = append(evicted, correlationID) + }, + }) + require.NoError(t, err) + + secret := uuid.New().String() + id := xid.New().String() + _, pubKey := generateRSAKeyPair(t) + require.NoError(t, db.SetIDPublicKey(id, secret, pubKey)) + + require.NoError(t, db.Close()) + require.Eventually(t, func() bool { + mu.Lock() + defer mu.Unlock() + return len(evicted) == 1 && evicted[0] == id + }, 2*time.Second, 10*time.Millisecond) + }) + + // Without the s.db nil guard this panics on the cache's event goroutine, + // which then deadlocks every subsequent cache operation. + t.Run("memory mode eviction does not panic", func(t *testing.T) { + db, id, secret := newUploadTestDB(t, &Options{EvictionTTL: time.Hour}) + + require.NoError(t, db.RemoveID(id, secret)) + time.Sleep(100 * time.Millisecond) + + // If the event goroutine had died, this would block forever. + done := make(chan struct{}) + go func() { + defer close(done) + newID := xid.New().String() + _, pubKey := generateRSAKeyPair(t) + _ = db.SetIDPublicKey(newID, uuid.New().String(), pubKey) + }() + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("cache is deadlocked: the removal listener killed the event goroutine") + } + }) +}