Fix memory leak on handling QoS 2 PUBLISH.

In some circumstances, Mosquitto could leak memory when handling PUBLISH messages. This is limited to incoming QoS 2 messages, and is related to the combination of the broker having persistence enabled, a clean session=false client, which was connected prior to the broker restarting, then has reconnected and has now sent messages at a sufficiently high rate that the incoming queue at the broker has filled up and hence messages are being dropped. This is more likely to have an effect where max_queued_messages is a small value. This has now been fixed.

Closes #1793.

ChangeLog.txt

@@ -1,3 +1,16 @@
+1.5.10 - 2020-08-19
+===================
+
+Security:
+- In some circumstances, Mosquitto could leak memory when handling PUBLISH
+ messages. This is limited to incoming QoS 2 messages, and is related
+ to the combination of the broker having persistence enabled, a clean
+ session=false client, which was connected prior to the broker restarting,
+ then has reconnected and has now sent messages at a sufficiently high rate
+ that the incoming queue at the broker has filled up and hence messages are
+ being dropped. This is more likely to have an effect where
+ max_queued_messages is a small value. This has now been fixed. Closes #1793.
+
 1.5.9 - 20190917
 ================
 

src/database.c

@@ -36,7 +36,7 @@ static unsigned long max_queued_bytes = 0;
  * @param qos qos for the packet of interest
  * @return true if more in flight are allowed.
  */
-static bool db__ready_for_flight(struct mosquitto *context, int qos)
+bool db__ready_for_flight(struct mosquitto *context, int qos)
 {
  if(qos == 0 || (max_inflight == 0 && max_inflight_bytes == 0)){
  return true;
@@ -64,7 +64,7 @@ static bool db__ready_for_flight(struct mosquitto *context, int qos)
  * @param qos destination qos for the packet of interest
  * @return true if queuing is allowed, false if should be dropped
  */
-static bool db__ready_for_queue(struct mosquitto *context, int qos)
+bool db__ready_for_queue(struct mosquitto *context, int qos)
 {
  if(max_queued == 0 && max_queued_bytes == 0){
  return true;

src/handle_publish.c

@@ -184,9 +184,23 @@ int handle__publish(struct mosquitto_db *db, struct mosquitto *context)
  db__message_store_find(context, mid, &stored);
  }
  if(!stored){
- dup = 0;
- if(db__message_store(db, context, mid, topic, qos, payloadlen, &payload, retain, &stored, 0)){
- return 1;
+ if(qos == 0
+ || db__ready_for_flight(context, qos)
+ || db__ready_for_queue(context, qos)){
+
+ dup = 0;
+ if(db__message_store(db, context, mid, topic, qos, payloadlen, &payload, retain, &stored, 0)){
+ return 1;
+ }
+ }else{
+ /* Client isn't allowed any more incoming messages, so fail early */
+ mosquitto__free(topic);
+ UHPA_FREE(payload, payloadlen);
+ if(qos == 1){
+ return send__puback(context, mid);
+ }else{
+ return send__pubrec(context, mid);
+ }
  }
  }else{
  mosquitto__free(topic);

src/mosquitto_broker_internal.h

@@ -567,6 +567,8 @@ void db__msg_store_deref(struct mosquitto_db *db, struct mosquitto_msg_store **s
 void db__msg_store_clean(struct mosquitto_db *db);
 void db__msg_store_compact(struct mosquitto_db *db);
 int db__message_reconnect_reset(struct mosquitto_db *db, struct mosquitto *context);
+bool db__ready_for_flight(struct mosquitto *context, int qos);
+bool db__ready_for_queue(struct mosquitto *context, int qos);
 void sys_tree__init(struct mosquitto_db *db);
 void sys_tree__update(struct mosquitto_db *db, int interval, time_t start_time);