ESP32 + Paho + PubSubclient Issue

[imankareem]

Hello Everyone,
I am using an ESP32 with PubSubClient and Paho and Mosquitto (1.5.8) running on a Raspi 3 on Ubuntu. ESP32 and Raspi are on the same local network . PubSubClient's MQTT_MAX_PACKET_SIZE has been set to 256 bytes. Code used to work fine before but not anymore.

Error is "Attempting MQTT connection...failed, rc=-4 try again in 5 seconds"

The code running on my ESP32 is below. Setup is 2 Pro mins connected to IMUs over UART/Serial and a Temp sensor.

#include <Wire.h>
#include <DallasTemperature.h>
#include <OneWire.h>
#include <WiFi.h>
#include <PubSubClient.h>

const char* ssid = "Me";
const char* password = "";

const char* mqtt_server = "192.168.1.124"; //ip address of the mqtt server

const char* mqtt_topic = "/hms/data";

#define ONE_WIRE_BUS 18 // D18 ESP32, use 4.7K pull up

WiFiClient espClient;

PubSubClient client(espClient);

void MQTT_connect();

OneWire oneWire(ONE_WIRE_BUS);
DallasTemperature temperatureSensor(&oneWire);
float temperature = 0.0F;

unsigned long prevTime = 0;

HardwareSerial proMiniIMU1(2); // RX1:23, TX1:22 on ESP32. 
                              // Edit the file 'HardwareSerial.cpp 'like in video.
    
HardwareSerial proMiniIMU2(2); // RX2:16, TX2:17 on ESP32. This is by default.

String imu1Data;
String imu2Data;

int BPM = -1;

void setup() {
  // put your setup code here, to run once:
  Serial.begin(115200);
  proMiniIMU1.begin(9600);
  proMiniIMU2.begin(9600);

  connectWifi();
  client.setServer(mqtt_server, 1883);

  // temperatureSensor.begin();
}

void loop() {
  // put your main code here, to run repeatedly:
  if (!client.connected()) {
    reconnect();
  }
  client.loop();

  //temperatureSensor.requestTemperatures();
  //temperature = temperatureSensor.getTempCByIndex(0);

  if (millis() - prevTime > 1000) {
    //  publishData();

    displayData();

    prevTime = millis();

  }
}

void publishData() {

  char msg[200];

  sprintf(msg, "%.2f,%d,%s,%s",temperature, BPM,imu1Data,imu2Data);

  client.publish(mqtt_topic, msg);
  //Serial.println(msg);
}

void displayData() {
  Serial.println();
  Serial.println();
  Serial.print("Temperature = ");
  Serial.println(temperature);
  Serial.println();
  Serial.println("-----------------------[ IMU-1 ]-----------------------]");
  Serial.print(imu1Data);
  Serial.println();
  Serial.println("-----------------------[ IMU-2 ]-----------------------]");
  Serial.print(imu2Data);
  Serial.println();
}

void readIMU1() {

  proMiniIMU1.println("imu");

  if(proMiniIMU1.available())
  {
    imu1Data = proMiniIMU1.readStringUntil('\n');
    imu1Data.trim();
  }
}

void readIMU2() {

  proMiniIMU2.println("imu");

  if(proMiniIMU2.available())
  {
    imu2Data = proMiniIMU2.readStringUntil('\n');
    imu2Data.trim();
  }
}

void connectWifi() {

  // Connect to WiFi access point.
  Serial.println(); Serial.println();
  Serial.print("[*] Connecting to ");
  Serial.println(ssid);

  WiFi.begin(ssid, password);
  while (WiFi.status() != WL_CONNECTED) {
    delay(500);
    Serial.print(".");
    yield();
  }
  Serial.println();

  Serial.println("[+] WiFi connected");
  Serial.print("    >> IP address : ");
  Serial.println(WiFi.localIP());
}


void reconnect() {
  // Loop until we're reconnected
  while (!client.connected()) {
    Serial.print("Attempting MQTT connection...");
    // Attempt to connect
    if (client.connect("ESP32_HMS")) {
      Serial.println("connected");
      // Subscribe
      //client.subscribe("Master/commands");
    } else {
      Serial.print("failed, rc=");
      Serial.print(client.state());
      Serial.println(" try again in 5 seconds");
      // Wait 5 seconds before retrying
      delay(5000);
    }
    yield();
  }
}

Using this python script on the Raspi to receive mqtt packets and save to CSV file

import paho.mqtt.client as mqtt
import json
import time
import csv
import os
from datetime import datetime

host_name = "localhost"
topic = "/hms/data"

def store_data(msg):
    file_path = "./hms-data.csv"

    csv_title = ["Timestamp","Temperature","BPM","IMU1-Accel-X","IMU1-Accel-Y","IMU1-Accel-Z",
                  "IMU1-Gyro-X","IMU1-Gyro-Y","IMU1-Gyro-Z","IMU1-Mag-X","IMU1-Mag-Y","IMU1-Mag-Z",
                  "IMU2-Accel-X","IMU2-Accel-Y","IMU2-Accel-Z","IMU2-Gyro-X","IMU2-Gyro-Y","IMU2-Gyro-Z",
                  "IMU2-Mag-X","IMU2-Mag-Y","IMU2-Mag-Z"]

    msg = [float(x.strip()) for x in msg.split(",")]
    msg.insert(0, str(datetime.now()))

    if os.path.isfile(file_path):
        with open(file_path, "a") as fp:
            csv_writer = csv.writer(fp, delimiter=',')
            csv_writer.writerow(msg)
    else:
        with open(file_path, "w+") as fp:
            csv_writer = csv.writer(fp, delimiter=',')
            csv_writer.writerow(csv_title)
            csv_writer.writerow(msg)


def on_message(client, userdata, message):
    print("message received ", str(message.payload.decode("utf-8")))
    print("message topic = ", message.topic)
    print("message qos = ", message.qos)
    print("message retain flag = ", message.retain)
    msg = str(message.payload.decode("utf-8")).strip()
    if(message.topic == "/hms/data"):
        store_data(msg)

mqtt_client = mqtt.Client("omega31")
mqtt_client.on_message = on_message

broker = mqtt_client.connect(host_name)

print("Subscribing to topic : {}".format(topic))
while(True):
    mqtt_client.loop_start()

    mqtt_client.subscribe(topic)

    mqtt_client.loop_stop()

Please help as my thesis is due soon. Thank you!

[ralight]

Have you done any other debugging? At the moment it doesn't sound like you know what component the problem is in - the broker or the esp I am guessing. Does publishing with another mqtt client like mosquitto_pub work in place of the esp? Do you have any information from the broker logs?

[sislakd]

@imaniroman When do you start to see this issue? I've noticed similar issue in my setup where I use ESP8266 devices with PubSubClient-EspEasy-2.6.09 on Arduino 2.5.0 (Tasmota firmware) and Mosquitto on Raspberry Pi 3B+. I had the issue when I've upgraded from Moquitto 1.4.10-3+deb9u2 to Mosquitto 1.4.10-3+deb9u4 (yes just different update) from Rasbian default repository. Then I've tried to use Mosquitto repository https://repo.mosquitto.org/debian stretch main with version 1.5.6 but it contains the same issue. The only way to make my setup working again was Mosquitto downgrade back to 1.4.10-3+deb9u2. Before downgrading Mosquitto, I tried to downgrade kernel and various system libraries which were upgraded during the same R Pi upgrade cycles but without any success.

Later, when I had some time I did more debugging with newer Mosquitto and it seems that the problem lies somewhere in authentication. PubSubClient was not able to connect to Mosquitto and does regular retries but without any success. When I've disabled acl_file and allow anonymous access to mosquitto, it works on new version. However, running Mosquitto without authentication and acl is not acceptable for IoT devices.

I was able to run tcpdump and collect few attempts to connect + auth into mqtt both on old version and new version of mosquitto using the same configuration on connection without TLS. Initial communication from client looks exactly same but newer mosquitto is not accepting this connection. Thus it seems that there must be some change around initial authentication.

[imankareem]

Have you done any other debugging? At the moment it doesn't sound like you know what component the problem is in - the broker or the esp I am guessing. Does publishing with another mqtt client like mosquitto_pub work in place of the esp? Do you have any information from the broker logs?

@ralight Thanks for the reply! I installed mosquitto on Ubuntu on Windows 10 and tried a simple mqtt sketch. Both failed to publish. I then tried pinging my Ras pi which failed to. Currently having weird connection issues with my raspi to my Mobile hotspot. Trying to fix that first :)

[imankareem]

@sislakd
Thank you so much for the detailed response! At the moment I having basic connectivety issues even tho my Pi says its connected to the network.. Unable to ping local devices and google. Will dive in to your suggestion once I get basic connectivity going! THANKS!

[imankareem]

Turns out my Wifi hot stop and my Pi's wifi module was dead!!! Got it all running with a new pi!

[ralight]

Glad you got it sorted!