Merge pull request #3617 from sysown/v2.x-client_err_limit

Implements feature 'Client Error Limit'

Author: JavierJF

include/MySQL_Thread.h

@@ -118,6 +118,7 @@ enum MySQL_Thread_status_variable {
 	st_var_aws_aurora_replicas_skipped_during_query,
 	st_var_automatic_detected_sqli,
 	st_var_whitelisted_sqli_fingerprint,
+	st_var_client_host_error_killed_connections,
 	st_var_END
 };
 
@@ -320,6 +321,7 @@ struct p_th_counter {
 		whitelisted_sqli_fingerprint,
 		mysql_killed_backend_connections,
 		mysql_killed_backend_queries,
+		client_host_error_killed_connections,
 		__size
 	};
 };
@@ -359,6 +361,20 @@ struct th_metrics_map_idx {
 	};
 };
 
+/**
+ * @brief Structure holding the data for a Client_Host_Cache entry.
+ */
+typedef struct _MySQL_Client_Host_Cache_Entry {
+	/**
+	 * @brief Last time the entry was updated.
+	 */
+	uint64_t updated_at;
+	/**
+	 * @brief Error count associated with the entry.
+	 */
+	uint32_t error_count;
+} MySQL_Client_Host_Cache_Entry;
+
 class MySQL_Threads_Handler
 {
 	private:
@@ -382,6 +398,22 @@ class MySQL_Threads_Handler
 	//   variable address
 	//   special variable : if true, further input validation is required
 	std::unordered_map<std::string, std::tuple<bool *, bool>> VariablesPointers_bool;
+	/**
+	 * @brief Holds the clients host cache. It keeps track of the number of
+	 *   errors associated to a specific client:
+	 *     - Key: client identifier, based on 'clientaddr'.
+	 *     - Value: Structure of type 'MySQL_Client_Host_Cache_Entry' holding
+	 *       the last time the entry was updated and the error count associated
+	 *       with the client.
+	 */
+	std::unordered_map<std::string, MySQL_Client_Host_Cache_Entry> client_host_cache;
+	/**
+	 * @brief Holds the mutex for accessing 'client_host_cache', since every
+	 *   access can potentially perform 'read/write' operations, a regular mutex
+	 *   is enough.
+	 */
+	pthread_mutex_t mutex_client_host_cache;
+
 	public:
 	struct {
 		int monitor_history;
@@ -430,6 +462,8 @@ class MySQL_Threads_Handler
 		int query_retries_on_failure;
 		bool client_multi_statements;
 		bool connection_warming;
+		int client_host_cache_size;
+		int client_host_error_counts;
 		int connect_retries_on_failure;
 		int connect_retries_delay;
 		int connection_delay_multiplex_ms;
@@ -549,6 +583,77 @@ class MySQL_Threads_Handler
 		std::array<prometheus::Counter*, p_th_counter::__size> p_counter_array {};
 		std::array<prometheus::Gauge*, p_th_gauge::__size> p_gauge_array {};
 	} status_variables;
+	/**
+	 * @brief Update the client host cache with the supplied 'client_sockaddr',
+	 *   and the supplied 'error' parameter specifying if there was a connection
+	 *   error or not.
+	 *
+	 *   NOTE: This function is not safe, the supplied 'client_sockaddr' should
+	 *   have been initialized by 'accept' or 'getpeername'. NULL checks are not
+	 *   performed.
+	 *
+	 * @details The 'client_sockaddr' parameter is inspected, and the
+	 *   'client_host_cache' map is only updated in case of:
+	 *    - 'address_family' is either 'AF_INET' or 'AF_INET6'.
+	 *    - The address obtained from it isn't '127.0.0.1'.
+	 *
+	 *   In case 'client_sockaddr' matches the previous description, the update
+	 *   of the client host cache is performed in the following way:
+	 *     1. If the cache is full, the oldest element in the cache is searched.
+	 *     In case the oldest element address doesn't match the supplied
+	 *     address, the oldest element is removed.
+	 *     2. The cache is searched looking for the supplied address, in case of
+	 *     being found, the entry is updated, otherwise the entry is inserted in
+	 *     the cache.
+	 *
+	 * @param client_sockaddr A 'sockaddr' holding the required client information
+	 *   to update the 'client_host_cache_map'.
+	 * @param error 'true' if there was an error in the connection that should be
+	 *   register, 'false' otherwise.
+	 */
+	void update_client_host_cache(struct sockaddr* client_sockaddr, bool error);
+	/**
+	 * @brief Retrieves the entry of the underlying 'client_host_cache' map for
+	 *   the supplied 'client_sockaddr' in case of existing. In case it doesn't
+	 *   exist or the supplied 'client_sockaddr' doesn't met the requirements
+	 *   for being registered in the map, and zeroed 'MySQL_Client_Host_Cache_Entry'
+	 *   is returned.
+	 *
+	 *   NOTE: This function is not safe, the supplied 'client_sockaddr' should
+	 *   have been initialized by 'accept' or 'getpeername'. NULL checks are not
+	 *   performed.
+	 *
+	 * @details The 'client_sockaddr' parameter is inspected, and the
+	 *   'client_host_cache' map is only searched in case of:
+	 *    - 'address_family' is either 'AF_INET' or 'AF_INET6'.
+	 *    - The address obtained from it isn't '127.0.0.1'.
+	 *
+	 * @param client_sockaddr A 'sockaddr' holding the required client information
+	 *   to update the 'client_host_cache_map'.
+	 * @return If found, the corresponding entry for the supplied 'client_sockaddr',
+	 *   a zeroed 'MySQL_Client_Host_Cache_Entry' otherwise.
+	 */
+	MySQL_Client_Host_Cache_Entry find_client_host_cache(struct sockaddr* client_sockaddr);
+	/**
+	 * @brief Delete all the entries in the 'client_host_cache' internal map.
+	 */
+	void flush_client_host_cache();
+	/**
+	 * @brief Returns the current entries of 'client_host_cache' in a
+	 *   'SQLite3_result'. In case the param 'reset' is specified, the structure
+	 *   is cleaned after being queried.
+	 *
+	 * @param reset If 'true' the entries of the internal structure
+	 *   'client_host_cache' will be cleaned after scrapping.
+	 *
+	 * @return SQLite3_result holding the current entries of the
+	 *   'client_host_cache'. In the following format:
+	 *
+	 *    [ 'client_address', 'error_num', 'last_updated' ]
+	 *
+	 *    Where 'last_updated' is the last updated time expressed in 'ns'.
+	 */
+	SQLite3_result* get_client_host_cache(bool reset);
 	/**
 	 * @brief Callback to update the metrics.
 	 */

include/proxysql_admin.h

@@ -355,6 +355,7 @@ class ProxySQL_Admin {
 	void stats___proxysql_servers_metrics();
 	void stats___mysql_prepared_statements_info();
 	void stats___mysql_gtid_executed();
+	void stats___mysql_client_host_cache(bool reset);
 
 	// Update prometheus metrics
 	void p_stats___memory_metrics();

include/proxysql_structs.h

@@ -799,6 +799,8 @@ __thread bool mysql_thread___enable_client_deprecate_eof;
 __thread bool mysql_thread___enable_server_deprecate_eof;
 __thread bool mysql_thread___log_mysql_warnings_enabled;
 __thread bool mysql_thread___enable_load_data_local_infile;
+__thread int mysql_thread___client_host_cache_size;
+__thread int mysql_thread___client_host_error_counts;
 
 /* variables used for Query Cache */
 __thread int mysql_thread___query_cache_size_MB;
@@ -955,6 +957,8 @@ extern __thread bool mysql_thread___enable_client_deprecate_eof;
 extern __thread bool mysql_thread___enable_server_deprecate_eof;
 extern __thread bool mysql_thread___log_mysql_warnings_enabled;
 extern __thread bool mysql_thread___enable_load_data_local_infile;
+extern __thread int mysql_thread___client_host_cache_size;
+extern __thread int mysql_thread___client_host_error_counts;
 
 /* variables used for Query Cache */
 extern __thread int mysql_thread___query_cache_size_MB;

lib/MySQL_Session.cpp

@@ -4816,6 +4816,7 @@ void MySQL_Session::handler___status_CHANGING_USER_CLIENT___STATE_CLIENT_HANDSHA
 void MySQL_Session::handler___status_CONNECTING_CLIENT___STATE_SERVER_HANDSHAKE(PtrSize_t *pkt, bool *wrong_pass) {
 	bool is_encrypted = client_myds->encrypted;
 	bool handshake_response_return = client_myds->myprot.process_pkt_handshake_response((unsigned char *)pkt->ptr,pkt->size);
+	bool handshake_err = true;
 
 	proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION,8,"Session=%p , DS=%p , handshake_response=%d , switching_auth_stage=%d , is_encrypted=%d , client_encrypted=%d\n", this, client_myds, handshake_response_return, client_myds->switching_auth_stage, is_encrypted, client_myds->encrypted);
 	if (
@@ -5001,6 +5002,7 @@ void MySQL_Session::handler___status_CONNECTING_CLIENT___STATE_SERVER_HANDSHAKE(
 					) {
 						// we are good!
 						client_myds->myprot.generate_pkt_OK(true,NULL,NULL, _pid, 0,0,0,0,NULL);
+						handshake_err = false;
 						GloMyLogger->log_audit_entry(PROXYSQL_MYSQL_AUTH_OK, this, NULL);
 						status=WAITING_CLIENT_DATA;
 						client_myds->DSS=STATE_CLIENT_AUTH_OK;
@@ -5037,6 +5039,7 @@ void MySQL_Session::handler___status_CONNECTING_CLIENT___STATE_SERVER_HANDSHAKE(
 						proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION,8,"Session=%p , DS=%p . STATE_CLIENT_AUTH_OK\n", this, client_myds);
 						GloMyLogger->log_audit_entry(PROXYSQL_MYSQL_AUTH_OK, this, NULL);
 						client_myds->myprot.generate_pkt_OK(true,NULL,NULL, _pid, 0,0,0,0,NULL);
+						handshake_err = false;
 						status=WAITING_CLIENT_DATA;
 						client_myds->DSS=STATE_CLIENT_AUTH_OK;
 					}
@@ -5103,6 +5106,10 @@ void MySQL_Session::handler___status_CONNECTING_CLIENT___STATE_SERVER_HANDSHAKE(
 		__sync_add_and_fetch(&MyHGM->status.client_connections_aborted,1);
 		client_myds->DSS=STATE_SLEEP;
 	}
+
+	if (mysql_thread___client_host_cache_size) {
+		GloMTH->update_client_host_cache(client_myds->client_addr, handshake_err);
+	}
 }
 
 // Note: as commented in issue #546 and #547 , some clients ignore the status of CLIENT_MULTI_STATEMENTS