Extend db_dump tests, and fix minor bugs.

apps/db_dump/Makefile

@@ -1,7 +1,7 @@
 R=../..
 include ${R}/config.mk
 
-CFLAGS_FINAL=${CFLAGS} ${APP_CFLAGS} -I${R}/include -I${R}/ -I${R}/lib -I${R}/src -I${R}/deps -I${R}/common -DWITH_BROKER -DWITH_PERSISTENCE
+CFLAGS_FINAL=${CFLAGS} ${BROKER_CFLAGS} ${BROKER_CPPFLAGS} -I${R}/include -I${R}/ -I${R}/lib -I${R}/src -I${R}/deps -I${R}/common
 
 OBJS = \
 	   db_dump.o \

apps/db_dump/db_dump.c

@@ -120,7 +120,7 @@ static int dump__cfg_chunk_process(FILE *db_fd, uint32_t length)
 		rc = persist__chunk_cfg_read_v234(db_fd, &chunk);
 	}
 	if(rc){
-		fprintf(stderr, "Error: Corrupt persistent database.");
+		fprintf(stderr, "Error: Corrupt persistent database.\n");
 		fclose(db_fd);
 		return rc;
 	}
@@ -157,7 +157,7 @@ static int dump__client_chunk_process(FILE *db_fd, uint32_t length)
 		rc = persist__chunk_client_read_v234(db_fd, &chunk, db_version);
 	}
 	if(rc){
-		fprintf(stderr, "Error: Corrupt persistent database.");
+		fprintf(stderr, "Error: Corrupt persistent database.\n");
 		return rc;
 	}
 
@@ -198,7 +198,7 @@ static int dump__client_msg_chunk_process(FILE *db_fd, uint32_t length)
 		rc = persist__chunk_client_msg_read_v234(db_fd, &chunk);
 	}
 	if(rc){
-		fprintf(stderr, "Error: Corrupt persistent database.");
+		fprintf(stderr, "Error: Corrupt persistent database.\n");
 		fclose(db_fd);
 		return rc;
 	}
@@ -242,7 +242,7 @@ static int dump__base_msg_chunk_process(FILE *db_fptr, uint32_t length)
 		rc = persist__chunk_base_msg_read_v234(db_fptr, &chunk, db_version);
 	}
 	if(rc){
-		fprintf(stderr, "Error: Corrupt persistent database.");
+		fprintf(stderr, "Error: Corrupt persistent database.\n");
 		fclose(db_fptr);
 		return rc;
 	}
@@ -328,6 +328,7 @@ static int dump__retain_chunk_process(FILE *db_fd, uint32_t length)
 		rc = persist__chunk_retain_read_v234(db_fd, &chunk);
 	}
 	if(rc){
+		fprintf(stderr, "Error: Corrupt persistent database.\n");
 		fclose(db_fd);
 		return rc;
 	}
@@ -352,7 +353,7 @@ static int dump__sub_chunk_process(FILE *db_fd, uint32_t length)
 		rc = persist__chunk_sub_read_v234(db_fd, &chunk);
 	}
 	if(rc){
-		fprintf(stderr, "Error: Corrupt persistent database.");
+		fprintf(stderr, "Error: Corrupt persistent database.\n");
 		fclose(db_fd);
 		return rc;
 	}
@@ -458,16 +459,13 @@ int main(int argc, char *argv[])
 					break;
 
 				default:
-					fprintf(stderr, "Warning: Unsupported chunk \"%d\" in persistent database file. Ignoring.\n", chunk);
-					if(fseek(fd, length, SEEK_CUR) < 0){
-						fprintf(stderr, "Error seeking in file.\n");
-						return 1;
-					}
+					fprintf(stderr, "Warning: Unsupported chunk \"%d\" of length %d in persistent database file at position %ld. Ignoring.\n", chunk, length, ftell(fd));
+					fseek(fd, length, SEEK_CUR);
 					break;
 			}
 		}
 	}else{
-		fprintf(stderr, "Error: Unrecognised file format.");
+		fprintf(stderr, "Error: Unrecognised file format.\n");
 		rc = 1;
 	}
 
@@ -497,7 +495,7 @@ int main(int argc, char *argv[])
 	return rc;
 error:
 	cleanup_msg_store();
-	fprintf(stderr, "Error: %s.", strerror(errno));
+	fprintf(stderr, "Error: Corrupt persistent database.\n");
 	if(fd) fclose(fd);
 	return 1;
 }

apps/db_dump/print.c

@@ -37,90 +37,19 @@ static void print__properties(mosquitto_property *properties)
 
 	while(properties){
 		switch(properties->identifier){
+			/* Only properties for base messages are valid for saving */
 			case MQTT_PROP_PAYLOAD_FORMAT_INDICATOR:
 				printf("\t\tPayload format indicator: %d\n", properties->value.i8);
 				break;
-			case MQTT_PROP_REQUEST_PROBLEM_INFORMATION:
-				printf("\t\tRequest problem information: %d\n", properties->value.i8);
-				break;
-			case MQTT_PROP_REQUEST_RESPONSE_INFORMATION:
-				printf("\t\tRequest response information: %d\n", properties->value.i8);
-				break;
-			case MQTT_PROP_MAXIMUM_QOS:
-				printf("\t\tMaximum QoS: %d\n", properties->value.i8);
-				break;
-			case MQTT_PROP_RETAIN_AVAILABLE:
-				printf("\t\tRetain available: %d\n", properties->value.i8);
-				break;
-			case MQTT_PROP_WILDCARD_SUB_AVAILABLE:
-				printf("\t\tWildcard sub available: %d\n", properties->value.i8);
-				break;
-			case MQTT_PROP_SUBSCRIPTION_ID_AVAILABLE:
-				printf("\t\tSubscription ID available: %d\n", properties->value.i8);
-				break;
-			case MQTT_PROP_SHARED_SUB_AVAILABLE:
-				printf("\t\tShared subscription available: %d\n", properties->value.i8);
-				break;
-
-			case MQTT_PROP_SERVER_KEEP_ALIVE:
-				printf("\t\tServer keep alive: %d\n", properties->value.i16);
-				break;
-			case MQTT_PROP_RECEIVE_MAXIMUM:
-				printf("\t\tReceive maximum: %d\n", properties->value.i16);
-				break;
-			case MQTT_PROP_TOPIC_ALIAS_MAXIMUM:
-				printf("\t\tTopic alias maximum: %d\n", properties->value.i16);
-				break;
-			case MQTT_PROP_TOPIC_ALIAS:
-				printf("\t\tTopic alias: %d\n", properties->value.i16);
-				break;
-
-			case MQTT_PROP_MESSAGE_EXPIRY_INTERVAL:
-				printf("\t\tMessage expiry interval: %d\n", properties->value.i32);
-				break;
-			case MQTT_PROP_SESSION_EXPIRY_INTERVAL:
-				printf("\t\tSession expiry interval: %d\n", properties->value.i32);
-				break;
-			case MQTT_PROP_WILL_DELAY_INTERVAL:
-				printf("\t\tWill delay interval: %d\n", properties->value.i32);
-				break;
-			case MQTT_PROP_MAXIMUM_PACKET_SIZE:
-				printf("\t\tMaximum packet size: %d\n", properties->value.i32);
-				break;
-
-			case MQTT_PROP_SUBSCRIPTION_IDENTIFIER:
-				printf("\t\tSubscription identifier: %d\n", properties->value.varint);
-				break;
 
 			case MQTT_PROP_CONTENT_TYPE:
 				printf("\t\tContent type: %s\n", properties->value.s.v);
 				break;
+
 			case MQTT_PROP_RESPONSE_TOPIC:
 				printf("\t\tResponse topic: %s\n", properties->value.s.v);
 				break;
-			case MQTT_PROP_ASSIGNED_CLIENT_IDENTIFIER:
-				printf("\t\tAssigned client identifier: %s\n", properties->value.s.v);
-				break;
-			case MQTT_PROP_AUTHENTICATION_METHOD:
-				printf("\t\tAuthentication method: %s\n", properties->value.s.v);
-				break;
-			case MQTT_PROP_RESPONSE_INFORMATION:
-				printf("\t\tResponse information: %s\n", properties->value.s.v);
-				break;
-			case MQTT_PROP_SERVER_REFERENCE:
-				printf("\t\tServer reference: %s\n", properties->value.s.v);
-				break;
-			case MQTT_PROP_REASON_STRING:
-				printf("\t\tReason string: %s\n", properties->value.s.v);
-				break;
 
-			case MQTT_PROP_AUTHENTICATION_DATA:
-				printf("\t\tAuthentication data: ");
-				for(i=0; i<properties->value.bin.len; i++){
-					printf("%02X", properties->value.bin.v[i]);
-				}
-				printf("\n");
-				break;
 			case MQTT_PROP_CORRELATION_DATA:
 				printf("\t\tCorrelation data: ");
 				for(i=0; i<properties->value.bin.len; i++){

apps/db_dump/stubs.c

@@ -10,63 +10,24 @@
 #  define UNUSED(A) (void)(A)
 #endif
 
-struct mosquitto *context__init(void)
-{
-	return NULL;
-}
+struct mosquitto *context__init(void) { return NULL; }
 
-void context__add_to_by_id(struct mosquitto *context)
-{
-	UNUSED(context);
-}
+void context__add_to_by_id(struct mosquitto *context) { UNUSED(context); }
 
-int db__message_store(const struct mosquitto *source, struct mosquitto_base_msg *base_msg, uint32_t message_expiry_interval, dbid_t store_id, enum mosquitto_msg_origin origin)
-{
-	UNUSED(source); UNUSED(base_msg); UNUSED(message_expiry_interval); UNUSED(store_id); UNUSED(origin);
-    return 0;
-}
+int db__message_store(const struct mosquitto *source, struct mosquitto_base_msg *base_msg, uint32_t message_expiry_interval, dbid_t store_id, enum mosquitto_msg_origin origin) { UNUSED(source); UNUSED(base_msg); UNUSED(message_expiry_interval); UNUSED(store_id); UNUSED(origin); return 0; }
 
-void db__msg_store_ref_inc(struct mosquitto_base_msg *base_msg)
-{
-	UNUSED(base_msg);
-}
+void db__msg_store_ref_inc(struct mosquitto_base_msg *base_msg) { UNUSED(base_msg); }
 
-int log__printf(struct mosquitto *mosq, unsigned int level, const char *fmt, ...)
-{
-	UNUSED(mosq); UNUSED(level); UNUSED(fmt);
-	return 0;
-}
+int log__printf(struct mosquitto *mosq, unsigned int level, const char *fmt, ...) { UNUSED(mosq); UNUSED(level); UNUSED(fmt); return 0; }
 
-FILE *mosquitto__fopen(const char *path, const char *mode, bool restrict_read)
-{
-	UNUSED(path); UNUSED(mode); UNUSED(restrict_read);
-	return NULL;
-}
+FILE *mosquitto__fopen(const char *path, const char *mode, bool restrict_read) { UNUSED(path); UNUSED(mode); UNUSED(restrict_read); return NULL; }
 
-int retain__store(const char *topic, struct mosquitto_base_msg *base_msg, char **split_topics, bool persist)
-{
-	UNUSED(topic); UNUSED(base_msg); UNUSED(split_topics); UNUSED(persist);
-	return 0;
-}
+int retain__store(const char *topic, struct mosquitto_base_msg *base_msg, char **split_topics, bool persist) { UNUSED(topic); UNUSED(base_msg); UNUSED(split_topics); UNUSED(persist); return 0; }
 
-int sub__add(struct mosquitto *context, const char *sub, uint8_t qos, uint32_t identifier, int options)
-{
-	UNUSED(context); UNUSED(sub); UNUSED(qos); UNUSED(identifier); UNUSED(options);
-	return 0;
-}
+int sub__add(struct mosquitto *context, const char *sub, uint8_t qos, uint32_t identifier, int options) { UNUSED(context); UNUSED(sub); UNUSED(qos); UNUSED(identifier); UNUSED(options); return 0; }
 
-void db__msg_add_to_inflight_stats(struct mosquitto_msg_data *msg_data, struct mosquitto_client_msg *msg)
-{
-	UNUSED(msg_data); UNUSED(msg);
-}
+void db__msg_add_to_inflight_stats(struct mosquitto_msg_data *msg_data, struct mosquitto_client_msg *msg) { UNUSED(msg_data); UNUSED(msg); }
 
-void db__msg_add_to_queued_stats(struct mosquitto_msg_data *msg_data, struct mosquitto_client_msg *msg)
-{
-	UNUSED(msg_data); UNUSED(msg);
-}
+void db__msg_add_to_queued_stats(struct mosquitto_msg_data *msg_data, struct mosquitto_client_msg *msg) { UNUSED(msg_data); UNUSED(msg); }
 
-int session_expiry__add_from_persistence(struct mosquitto *context, time_t expiry_time)
-{
-	UNUSED(context); UNUSED(expiry_time);
-	return 0;
-}
+int session_expiry__add_from_persistence(struct mosquitto *context, time_t expiry_time) { UNUSED(context); UNUSED(expiry_time); return 0; }

test/apps/01-db-dump-client-stats.py

@@ -9,7 +9,7 @@ def do_test(file, counts):
         f"MS: {counts[3]} " + \
         f"  {counts[4]}\n"
 
-    cmd = ['../../apps/db_dump/mosquitto_db_dump',
+    cmd = [mosq_test.get_build_root()+'/apps/db_dump/mosquitto_db_dump',
             '--client-stats',
             f'db_dump/{file}'
             ]

test/apps/01-db-dump-corrupt.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python3
+
+from mosq_test_helper import *
+
+def do_test(file, stderr, rc_expected):
+
+    cmd = [mosq_test.get_build_root()+'/apps/db_dump/mosquitto_db_dump',
+            f'db_dump/{file}'
+            ]
+
+    res = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=1, encoding='utf-8')
+    if res.stderr != stderr:
+        print(res.stderr)
+        raise mosq_test.TestError
+    if res.returncode != rc_expected:
+        print(file)
+        print(res.returncode)
+        raise mosq_test.TestError
+
+do_test('missing.test-db', "Error: Unable to open db_dump/missing.test-db\n", 0)
+do_test('bad-magic.test-db', "Error: Unrecognised file format.\n", 1)
+do_test('short.test-db', "Error: Corrupt persistent database.\n", 1)
+do_test('bad-dbid-size.test-db', "Error: Incompatible database configuration (dbid size is 5 bytes, expected 8)", 1)
+do_test('bad-chunk.test-db', 'Warning: Unsupported chunk "2816" of length 65696 in persistent database file at position 29. Ignoring.\n', 0)
+do_test('v3-corrupt.test-db', "Error: Corrupt persistent database.\n", 1)
+do_test('v4-corrupt.test-db', "Error: Corrupt persistent database.\n", 1)
+do_test('v5-corrupt.test-db', "Error: Corrupt persistent database.\n", 1)
+do_test('v6-corrupt.test-db', "Error: Corrupt persistent database.\n", 1)
+do_test('v6-corrupt-client.test-db', "Error: Corrupt persistent database.\n", 1)
+do_test('v6-corrupt-cmsg.test-db', "Error: Corrupt persistent database.\n", 1)
+do_test('v6-corrupt-retain.test-db', "Error: Corrupt persistent database.\n", 1)
+do_test('v6-corrupt-sub.test-db', "Error: Corrupt persistent database.\n", 1)

test/apps/01-db-dump-print-empty.py

@@ -4,7 +4,7 @@ from mosq_test_helper import *
 
 def do_test(file, stdout):
 
-    cmd = ['../../apps/db_dump/mosquitto_db_dump',
+    cmd = [mosq_test.get_build_root()+'/apps/db_dump/mosquitto_db_dump',
             f'db_dump/{file}'
             ]
 

test/apps/01-db-dump-print-v6-all.py

@@ -4,12 +4,13 @@ from mosq_test_helper import *
 
 def do_test(file, stdout):
 
-    cmd = ['../../apps/db_dump/mosquitto_db_dump',
+    cmd = [mosq_test.get_build_root()+'/apps/db_dump/mosquitto_db_dump',
             f'db_dump/{file}'
             ]
 
     res = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=1, encoding='utf-8')
     if res.stdout != stdout:
+        print(res.stdout)
         raise mosq_test.TestError
 
 stdout = """Mosquitto DB dump
@@ -25,7 +26,7 @@ DB_CHUNK_BASE_MSG:
 	Store ID: 208508774941868
 	Source Port: 1883
 	Source MID: 1
-	Topic: (null)
+	Topic: topic
 	QoS: 1
 	Retain: 1
 	Payload Length: 7

test/apps/01-db-dump-print-v6-mqtt-v5-props.py

@@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+
+from mosq_test_helper import *
+
+def do_test(file, stdout):
+
+    cmd = [mosq_test.get_build_root() + '/apps/db_dump/mosquitto_db_dump',
+            f'db_dump/{file}'
+            ]
+
+    res = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=3, encoding='utf-8')
+    if res.stdout != stdout:
+        print(res.stdout)
+        raise mosq_test.TestError
+
+stdout = """Mosquitto DB dump
+CRC: 0
+DB version: 6
+DB_CHUNK_CFG:
+	Length: 16
+	Shutdown: 1
+	DB ID size: 8
+	Last DB ID: 273732472648936
+DB_CHUNK_BASE_MSG:
+	Length: 187
+	Store ID: 273732462748327
+	Source Port: 1883
+	Source MID: 1
+	Topic: test-topic
+	QoS: 1
+	Retain: 0
+	Payload Length: 7
+	Expiry Time: 1669799825
+	Payload: message
+	Properties:
+		Content type: text/plain
+		Correlation data: 35636638653064652D356666612D346131302D393036622D346535623266393038363162
+		Payload format indicator: 1
+		Response topic: pub-response-topic
+		User property: pub-key , pub-value
+DB_CHUNK_BASE_MSG:
+	Length: 132
+	Store ID: 273732472648936
+	Source Port: 0
+	Source MID: 0
+	Topic: will-topic
+	QoS: 2
+	Retain: 1
+	Payload Length: 12
+	Expiry Time: 1669799786
+	Payload: will-payload
+	Properties:
+		Content type: text/plain
+		Correlation data: 636F7272656C6174696F6E2D64617461
+		Payload format indicator: 1
+		Response topic: will-response-topic
+		User property: key , value
+DB_CHUNK_CLIENT:
+	Length: 32
+	Client ID: clientid
+	Last MID: 1
+	Session expiry time: 1669799784
+	Session expiry interval: 60
+DB_CHUNK_CLIENT_MSG:
+	Length: 27
+	Client ID: clientid
+	Store ID: 273732462748327
+	MID: 1
+	QoS: 1
+	Retain: 0
+	Direction: 1
+	State: 11
+	Dup: 0
+	Subscription identifier: 42
+DB_CHUNK_SUB:
+	Length: 30
+	Client ID: clientid
+	Topic: test-topic
+	QoS: 1
+	Subscription ID: 42
+	Options: 0x00
+DB_CHUNK_RETAIN:
+	Length: 8
+	Store ID: 273732472648936
+"""
+do_test('v6-mqtt-v5-props.test-db', stdout)

test/apps/01-db-dump-stats-current.py

@@ -20,7 +20,7 @@ def check_db(port, counts):
         f"DB_CHUNK_SUB:        {counts[4]}\n" + \
         f"DB_CHUNK_CLIENT:     {counts[5]}\n"
 
-    cmd = ['../../apps/db_dump/mosquitto_db_dump',
+    cmd = [mosq_test.get_build_root()+'/apps/db_dump/mosquitto_db_dump',
             '--stats',
             f'{port}/mosquitto.db'
             ]
@@ -51,7 +51,7 @@ def do_test(counts):
             }
 
         # Set up persistent client session, including a subscription
-        cmd = ['../../client/mosquitto_sub',
+        cmd = [mosq_test.get_build_root()+'/client/mosquitto_sub',
             '-c',
             '-i', 'client-id',
             '-p', str(port),
@@ -62,7 +62,7 @@ def do_test(counts):
         subprocess.run(cmd, timeout=1, env=env)
 
         # Publish a retained message which is also queued for the subscriber
-        cmd = ['../../client/mosquitto_pub',
+        cmd = [mosq_test.get_build_root()+'/client/mosquitto_pub',
             '-p', str(port),
             '-q', '1',
             '-t', 'sub-topic',

test/apps/01-db-dump-stats.py

@@ -10,7 +10,7 @@ def do_test(file, counts):
         f"DB_CHUNK_SUB:        {counts[4]}\n" + \
         f"DB_CHUNK_CLIENT:     {counts[5]}\n"
 
-    cmd = ['../../apps/db_dump/mosquitto_db_dump',
+    cmd = [mosq_test.get_build_root()+'/apps/db_dump/mosquitto_db_dump',
             '--stats',
             f'db_dump/{file}'
             ]

test/apps/Makefile

@@ -2,6 +2,7 @@ R=../..
 include ${R}/config.mk
 
 .PHONY: all check test ptest clean
+.NOTPARALLEL:
 
 all :
 
@@ -11,9 +12,11 @@ test : 01
 
 01 :
 	./01-db-dump-client-stats.py
+	./01-db-dump-corrupt.py
 	./01-db-dump-print-empty.py
 	./01-db-dump-print-v6-all.py
-	./01-db-dump-stats.py
+	./01-db-dump-print-v6-mqtt-v5-props.py
 	./01-db-dump-stats-current.py
+	./01-db-dump-stats.py
 
 clean:

test/apps/db_dump/bad-magic.test-db

@@ -0,0 +1 @@
+This is a text file, not a persistence file.