APP-14482 - Downgrade logs for signaling/ratelimiting (#519)

Co-authored-by: Benjamin Rewis <[email protected]>

Author: cheukt

rpc/wrtc_call_queue_mongodb.go

@@ -620,7 +620,7 @@ func (queue *mongoDBWebRTCCallQueue) operatorLivenessLoop() {
 					//nolint:goconst
 					reason = "other"
 				}
-				queue.logger.Errorw("failed to update operator document for self", "error", err)
+				queue.logger.Infow("failed to update operator document for self", "error", err)
 			}
 			operatorsCollUpdateFailures.Inc(queue.operatorID, reason)
 		} else if result.UpsertedCount == 1 {
@@ -738,7 +738,7 @@ func (queue *mongoDBWebRTCCallQueue) changeStreamManager() {
 		if err != nil {
 			callChangeStreamFailures.Inc(queue.operatorID)
 			queue.csManagerSeq.Add(1)
-			queue.logger.Errorw("failed to create calls change stream", "error", err)
+			queue.logger.Infow("failed to create calls change stream", "error", err)
 			continue
 		}
 

rpc/wrtc_rate_limiter.go

@@ -28,6 +28,14 @@ var rateLimitDenials = statz.NewCounter1[string]("signaling/rate_limits_denials"
 	},
 })
 
+var rateLimitErrors = statz.NewCounter1[string]("signaling/rate_limit_errors", statz.MetricConfig{
+	Description: "Total number of errors while applying the rate limit.",
+	Unit:        units.Dimensionless,
+	Labels: []statz.Label{
+		{Name: "reason", Description: "Reason that applying the rate limit failed."},
+	},
+})
+
 // Database configuration and collection names for MongoDB rate limiter.
 var (
 	mongodbWebRTCRateLimiterCollName = "rate_limiter"
@@ -102,8 +110,10 @@ func (rl *MongoDBRateLimiter) Allow(ctx context.Context, key string) error {
 		}},
 		options.Update().SetUpsert(true))
 	if err != nil {
-		rl.logger.Errorw("rate limit doc existence check failed", "error", err, "key", key)
-		return err
+		// to not erroneously rate limit requests, we log the error but do not return the error.
+		rl.logger.Infow("rate limit doc existence check failed", "error", err, "key", key)
+		rateLimitErrors.Inc("existence_check_failed")
+		return nil
 	}
 
 	// Filter: only match if request count within the most recent window for this key is < MaxRequests
@@ -176,8 +186,10 @@ func (rl *MongoDBRateLimiter) Allow(ctx context.Context, key string) error {
 
 	result, err := rl.rateLimitColl.UpdateOne(ctx, filter, update)
 	if err != nil {
-		rl.logger.Errorw("rate limit operation failed", "error", err, "key", key)
-		return err
+		// to not erroneously rate limit requests, we log the error but do not return the error.
+		rl.logger.Infow("rate limit operation failed", "error", err, "key", key)
+		rateLimitErrors.Inc("update_operation_failed")
+		return nil
 	}
 
 	// No match means rate limit exceeded