feat: ✨ Add option to unescape strings inside log string (#48)

* feat: ✨ Add option to unescape strings inside log string

* fix: 🎨 Minor changes

* fix: arrow_down: Remove testify dependency

Author: adrienyhuel

README.md

@@ -33,6 +33,7 @@ Apache Common Log Format. The body of the block accepts the custom `placeholder`
 log {
 	format transform [<template>] {
 		placeholder <string>
+		unescape_strings
 		# other fields accepted by JSON encoder
 	}
 }

caddyfile.go

@@ -30,34 +30,37 @@ import (
 // See the godoc on the LogEncoderConfig type for the syntax of
 // subdirectives that are common to most/all encoders.
 func (se *TransformEncoder) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
-	foundTemplate := 0
-outerloop:
 	for d.Next() {
 		args := d.RemainingArgs()
 		switch len(args) {
 		case 0:
 			se.Template = commonLogFormat
 		default:
-			foundTemplate = len(args)
 			se.Template = strings.Join(args, " ")
 		}
 
 		for nesting := d.Nesting(); d.NextBlock(nesting); {
-			if d.Val() == "placeholder" {
+			switch d.Val() {
+			case "placeholder":
 				d.AllArgs(&se.Placeholder)
 				// delete the `placeholder` token and the value, and reset the cursor
 				d.Delete()
 				d.Delete()
-				break outerloop
+			case "unescape_strings":
+				if d.NextArg() {
+					return d.ArgErr()
+				}
+				d.Delete()
+				se.UnescapeStrings = true
+			default:
+				d.RemainingArgs() //consume line without getting values
 			}
 		}
 	}
 
 	d.Reset()
 	// consume the directive and the template
-	d.Next()
-	for ; foundTemplate > 0; foundTemplate-- {
-		d.Next()
-	}
+	d.RemainingArgs()
+
 	return (&se.LogEncoderConfig).UnmarshalCaddyfile(d)
 }

caddyfile_test.go

@@ -29,6 +29,7 @@ func TestUnmarshalCaddyfile(t *testing.T) {
 		Encoder          zapcore.Encoder
 		Template         string
 		Placeholder      string
+		UnescapeStrings  bool
 	}
 	type args struct {
 		d *caddyfile.Dispenser
@@ -295,17 +296,64 @@ func TestUnmarshalCaddyfile(t *testing.T) {
 			},
 			wantErr: false,
 		},
+		{
+			name: "transform: not template but given unescape_strings",
+			fields: fields{
+				Template:        commonLogFormat,
+				UnescapeStrings: true,
+			},
+			args: args{
+				d: caddyfile.NewTestDispenser(`transform {
+					unescape_strings
+				}`),
+			},
+			wantErr: false,
+		},
+		{
+			name: "transform: given template and given unescape_strings",
+			fields: fields{
+				Template:        "{obj1>obj2>[0]}",
+				UnescapeStrings: true,
+			},
+			args: args{
+				d: caddyfile.NewTestDispenser(`transform "{obj1>obj2>[0]}" {
+					unescape_strings
+				}`),
+			},
+			wantErr: false,
+		},
+		{
+			name: "transform: `placeholder` `unescape_strings` and  property sitting between other properties",
+			fields: fields{
+				Template:        "{obj1>obj2>[0]}",
+				Placeholder:     "-",
+				UnescapeStrings: true,
+				LogEncoderConfig: logging.LogEncoderConfig{
+					TimeLocal:  true,
+					TimeFormat: "iso8601",
+				},
+			},
+			args: args{
+				d: caddyfile.NewTestDispenser(`transform "{obj1>obj2>[0]}" {
+					time_local
+					placeholder -
+					unescape_strings
+					time_format iso8601
+				}`),
+			},
+			wantErr: false,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			se := &TransformEncoder{
 				Encoder: new(logging.JSONEncoder),
 			}
 			if err := se.UnmarshalCaddyfile(tt.args.d); (err != nil) != tt.wantErr {
-				t.Errorf("TransformEncoder.UnmarshalCaddyfile() error = %v, wantErr %v", err, tt.wantErr)
+				t.Fatalf("TransformEncoder.UnmarshalCaddyfile() error = %v, wantErr %v", err, tt.wantErr)
 			}
-			if se.Template != tt.fields.Template || se.Placeholder != tt.fields.Placeholder || !reflect.DeepEqual(se.LogEncoderConfig, tt.fields.LogEncoderConfig) {
-				t.Errorf("Unexpected marshalling error: expected = %+v, received: %+v", tt.fields, *se)
+			if se.Template != tt.fields.Template || se.Placeholder != tt.fields.Placeholder || se.UnescapeStrings != tt.fields.UnescapeStrings || !reflect.DeepEqual(se.LogEncoderConfig, tt.fields.LogEncoderConfig) {
+				t.Fatalf("Unexpected marshalling error: expected = %+v, received: %+v", tt.fields, *se)
 			}
 		})
 	}

formatencoder.go

@@ -64,6 +64,7 @@ type TransformEncoder struct {
 	zapcore.Encoder `json:"-"`
 	Template        string `json:"template,omitempty"`
 	Placeholder     string `json:"placeholder,omitempty"`
+	UnescapeStrings bool   `json:"unescape_strings,omitempty"`
 }
 
 func (TransformEncoder) CaddyModule() caddy.ModuleInfo {
@@ -106,6 +107,7 @@ func (se TransformEncoder) Clone() zapcore.Encoder {
 		Encoder:          se.Encoder.Clone(),
 		Template:         se.Template,
 		Placeholder:      se.Placeholder,
+		UnescapeStrings:  se.UnescapeStrings,
 	}
 }
 
@@ -119,7 +121,7 @@ func (se TransformEncoder) EncodeEntry(ent zapcore.Entry, fields []zapcore.Field
 	repl.Map(func(key string) (interface{}, bool) {
 		if strings.Contains(key, ":") {
 			for _, slice := range strings.Split(key, ":") {
-				val, found := getValue(buf, slice)
+				val, found := getValue(buf, slice, se.UnescapeStrings)
 				if found {
 					return val, found
 				}
@@ -128,7 +130,7 @@ func (se TransformEncoder) EncodeEntry(ent zapcore.Entry, fields []zapcore.Field
 			return nil, false
 		}
 
-		return getValue(buf, key)
+		return getValue(buf, key, se.UnescapeStrings)
 	})
 
 	out := repl.ReplaceAll(se.Template, se.Placeholder)
@@ -144,7 +146,7 @@ func (se TransformEncoder) EncodeEntry(ent zapcore.Entry, fields []zapcore.Field
 	return buf, err
 }
 
-func getValue(buf *buffer.Buffer, key string) (interface{}, bool) {
+func getValue(buf *buffer.Buffer, key string, unescapeStrings bool) (interface{}, bool) {
 	path := strings.Split(key, ">")
 	value, dataType, _, err := jsonparser.Get(buf.Bytes(), path...)
 	if err != nil {
@@ -153,7 +155,13 @@ func getValue(buf *buffer.Buffer, key string) (interface{}, bool) {
 	switch dataType {
 	case jsonparser.NotExist:
 		return nil, false
-	case jsonparser.Array, jsonparser.Boolean, jsonparser.Null, jsonparser.Number, jsonparser.Object, jsonparser.String, jsonparser.Unknown:
+	case jsonparser.String:
+		if !unescapeStrings {
+			return value, true
+		}
+		str, _ := jsonparser.ParseString(value)
+		return str, true
+	case jsonparser.Array, jsonparser.Boolean, jsonparser.Null, jsonparser.Number, jsonparser.Object, jsonparser.Unknown:
 		// if a value exists, return it as is. A byte is a byte is a byte. The replacer handles them just fine.
 		return value, true
 	default:

formatencoder_test.go

@@ -0,0 +1,85 @@
+// Copyright 2015 Matthew Holt and The Caddy Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package transformencoder
+
+import (
+	"testing"
+
+	"github.com/caddyserver/caddy/v2"
+	"github.com/caddyserver/caddy/v2/modules/logging"
+	"go.uber.org/zap"
+	"go.uber.org/zap/zapcore"
+)
+
+func TestEncodeEntry(t *testing.T) {
+	tests := []struct {
+		name              string
+		se                TransformEncoder
+		entry             zapcore.Entry
+		fields            []zapcore.Field
+		expectedLogString string
+	}{
+		{
+			name: "encode entry: no unescape field",
+			se: TransformEncoder{
+				Encoder:  new(logging.JSONEncoder),
+				Template: "{msg} {username}",
+			},
+			entry: zapcore.Entry{
+				Message: "lob\nlaw",
+			},
+			fields: []zapcore.Field{
+				zap.String("username", "john\ndoe"),
+			},
+			expectedLogString: "lob\\nlaw john\\ndoe\n",
+		},
+		{
+			name: "encode entry: unescape field",
+			se: TransformEncoder{
+				Encoder:         new(logging.JSONEncoder),
+				Template:        "{msg} {username}",
+				UnescapeStrings: true,
+			},
+			entry: zapcore.Entry{
+				Message: "lob\nlaw",
+			},
+			fields: []zapcore.Field{
+				zap.String("username", "john\ndoe"),
+			},
+			expectedLogString: "lob\nlaw john\ndoe\n",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+
+			err := tt.se.Provision(caddy.Context{})
+			if err != nil {
+				t.Fatalf("TransformEncoder.Provision() error = %v", err)
+			}
+
+			buf, err := tt.se.EncodeEntry(tt.entry, tt.fields)
+
+			if err != nil {
+				t.Fatalf("TransformEncoder.EncodeEntry() error = %v", err)
+			}
+
+			if tt.expectedLogString != buf.String() {
+				t.Fatalf("Unexpected encoding error: expected = %+v, received: %+v", tt.expectedLogString, buf)
+			}
+
+		})
+	}
+}