pack 3 DatabaseLookup messages into one TunnelData while processing DatabaseSearchReply

I2NPProtocol.cpp

@@ -175,31 +175,6 @@ namespace i2p
 		}	
 	}	
 
-	void HandleDatabaseSearchReplyMsg (uint8_t * buf, size_t len)
-	{	
-#pragma pack(1)
-		struct
-		{
-			uint8_t key[32];
-			uint8_t num;
-		} * msg;	
-#pragma pack()	
-		msg = (decltype(msg))buf;
-		char key[48];
-		int l = i2p::data::ByteStreamToBase64 (msg->key, 32, key, 48);
-		key[l] = 0;
-		LogPrint ("DatabaseSearchReply for ", key, " num=", (int)msg->num);
-		for (int i = 0; i < msg->num; i++)
-		{
-			char peerHash[48];
-			int l1 = i2p::data::ByteStreamToBase64 (buf + sizeof (*msg) +i*32, 32, peerHash, 48);
-			peerHash[l1] = 0;
-			LogPrint (i,": ", peerHash);	
-
-			i2p::data::netdb.HandleDatabaseSearchReply (msg->key, buf + sizeof (*msg) +i*32);
-		}	
-	}	
-
 	I2NPBuildRequestRecordClearText CreateBuildRequestRecord (
 		const uint8_t * ourIdent, uint32_t receiveTunnelID, 
 	    const uint8_t * nextIdent, uint32_t nextTunnelID, 
@@ -432,10 +407,6 @@ namespace i2p
 				LogPrint ("Garlic");
 			break;	
 			break;	
-			case eI2NPDatabaseSearchReply:
-				LogPrint ("DatabaseSearchReply");
-				HandleDatabaseSearchReplyMsg (buf, size);
-			break;	
 			case eI2NPDeliveryStatus:
 				LogPrint ("DeliveryStatus");
 			break;	
@@ -468,7 +439,11 @@ namespace i2p
 				break;
 				case eI2NPDatabaseStore:
 					LogPrint ("DatabaseStore");
-					i2p::data::netdb.PostDatabaseStoreMsg (msg);
+					i2p::data::netdb.PostI2NPMsg (msg);
+				break;	
+				case eI2NPDatabaseSearchReply:
+					LogPrint ("DatabaseSearchReply");
+					i2p::data::netdb.PostI2NPMsg (msg);
 				break;	
 				default:
 					HandleI2NPMessage (msg->GetBuffer (), msg->GetLength ());

I2NPProtocol.h

@@ -106,11 +106,9 @@ namespace i2p
 	I2NPMessage * CreateDeliveryStatusMsg ();
 	I2NPMessage * CreateDatabaseLookupMsg (const uint8_t * key, const uint8_t * from, 
 		uint32_t replyTunnelID, bool exploratory = false);
+
 	I2NPMessage * CreateDatabaseStoreMsg ();
-	
 	void HandleDatabaseStoreMsg (uint8_t * buf, size_t len);
-	void HandleDatabaseSearchReplyMsg (uint8_t * buf, size_t len);
-	
 	
 	I2NPBuildRequestRecordClearText CreateBuildRequestRecord (
 		const uint8_t * ourIdent, uint32_t receiveTunnelID, 

NetDb.cpp

@@ -1,5 +1,6 @@
 #include <fstream>
 #include <boost/filesystem.hpp>
+#include "base64.h"
 #include "Log.h"
 #include "Timestamp.h"
 #include "I2NPProtocol.h"
@@ -60,6 +61,8 @@ namespace data
 						i2p::HandleDatabaseStoreMsg (msg->GetPayload (), msg->GetLength ()); // TODO
 						i2p::DeleteI2NPMessage (msg);
 					}
+					else if (msg->GetHeader ()->typeID == eI2NPDatabaseSearchReply)
+						HandleDatabaseSearchReplyMsg (msg);
 					else // WTF?
 					{
 						LogPrint ("NetDb: unexpected message type ", msg->GetHeader ()->typeID);
@@ -179,6 +182,42 @@ namespace data
 	//		RequestDestination (key, router);
 	}	
 
+	void NetDb::HandleDatabaseSearchReplyMsg (I2NPMessage * msg)
+	{
+		uint8_t * buf = msg->GetPayload ();
+		char key[48];
+		int l = i2p::data::ByteStreamToBase64 (buf, 32, key, 48);
+		key[l] = 0;
+		int num = buf[32]; // num
+		LogPrint ("DatabaseSearchReply for ", key, " num=", num);
+		if (num > 0)
+		{
+			if (!memcmp (m_Exploratory, buf, 32) && m_LastFloodfill)
+			{
+				i2p::tunnel::OutboundTunnel * outbound = i2p::tunnel::tunnels.GetNextOutboundTunnel ();
+				i2p::tunnel::InboundTunnel * inbound = i2p::tunnel::tunnels.GetNextInboundTunnel ();
+				for (int i = 0; i < num; i++)
+				{
+					uint8_t * router = buf + 33 + i*32;
+					char peerHash[48];
+					int l1 = i2p::data::ByteStreamToBase64 (router, 32, peerHash, 48);
+					peerHash[l1] = 0;
+					LogPrint (i,": ", peerHash);
+
+					if (outbound && inbound)
+					{
+						I2NPMessage * msg = i2p::CreateDatabaseLookupMsg (router, inbound->GetNextIdentHash (), 
+							inbound->GetNextTunnelID ());
+						outbound->GetTunnelGateway ().PutTunnelDataMsg (m_LastFloodfill->GetIdentHash (), 0, msg);
+					}	
+				}
+				if (outbound)
+					outbound->GetTunnelGateway ().SendBuffer ();
+			}	
+		}	
+		i2p::DeleteI2NPMessage (msg);
+	}	
+	
 	void NetDb::Explore ()
 	{
 		i2p::tunnel::OutboundTunnel * outbound = i2p::tunnel::tunnels.GetNextOutboundTunnel ();
@@ -225,7 +264,7 @@ namespace data
 		return nullptr;
 	}	
 
-	void NetDb::PostDatabaseStoreMsg (I2NPMessage * msg)
+	void NetDb::PostI2NPMsg (I2NPMessage * msg)
 	{
 		if (msg) m_Queue.Put (msg);	
 	}	

NetDb.h

@@ -30,11 +30,12 @@ namespace data
 			
 			void RequestDestination (const uint8_t * destination, const uint8_t * router);
 			void HandleDatabaseSearchReply (const uint8_t * key, const uint8_t * router);
+			void HandleDatabaseSearchReplyMsg (I2NPMessage * msg);
 			
 			const RouterInfo * GetRandomNTCPRouter (bool floodfillOnly = false) const;
 			const RouterInfo * GetRandomRouter () const;
 
-			void PostDatabaseStoreMsg (I2NPMessage * msg);
+			void PostI2NPMsg (I2NPMessage * msg);
 			
 		private:
 

TunnelBase.h

@@ -8,6 +8,9 @@ namespace i2p
 {
 namespace tunnel
 {
+	const size_t TUNNEL_DATA_MSG_SIZE = 1028;
+	const size_t TUNNEL_DATA_MAX_PAYLOAD_SIZE = 1003;
+	
 	enum TunnelDeliveryType 
 	{ 
 		eDeliveryTypeLocal = 0, 

TunnelGateway.cpp

@@ -16,7 +16,7 @@ namespace tunnel
 		block->deliveryInstructionsLen = 1; // flag
 		if (gwHash)
 		{	
-			block->deliveryInstructionsLen = 32; // hash
+			block->deliveryInstructionsLen += 32; // hash
 			memcpy (block->hash, gwHash, 32);
 			if (gwTunnel)
 			{	
@@ -30,32 +30,53 @@ namespace tunnel
 		else	
 			block->deliveryType = eDeliveryTypeLocal;
 		block->deliveryInstructionsLen += 2; // size
-		// we don't reserve 4 bytes for msgID because we don't if it fits
+		// we don't reserve 4 bytes for msgID yet
 		block->totalLen = block->deliveryInstructionsLen + msg->GetLength ();
 		block->data = msg;
 		m_I2NPMsgs.push_back (block);
+
+		if (!m_Remaining) m_Remaining = TUNNEL_DATA_MAX_PAYLOAD_SIZE;
+		if (block->totalLen <= m_Remaining) // message fits
+		{
+			block->isFragmented = false;
+			m_Remaining -= block->totalLen;
+		}	
+		else // message doesn't fit
+		{
+			if (block->deliveryInstructionsLen + 4 <= m_Remaining)
+			{
+				// delivery instructions of first fragment fits
+				block->isFragmented = true;
+				block->deliveryInstructionsLen += 4;
+				block->totalLen += 4;
+				m_Remaining = m_Remaining + TUNNEL_DATA_MAX_PAYLOAD_SIZE - block->totalLen - 7; // TODO: handle case if more than two fragments
+			}	
+			else
+			{
+				// delivery instructions of first fragment don't fit
+				block->isFragmented = false;
+				m_Remaining = 0;
+			}	
+		}	
 	}	
 
 	std::vector<I2NPMessage *> TunnelGatewayBuffer::GetTunnelDataMsgs (uint32_t tunnelID) 
 	{ 
+		m_Remaining = 0;
+		m_NextOffset = 0;
 		std::vector<I2NPMessage *> res;
 		int cnt = m_I2NPMsgs.size ();
-		m_NextOffset = 0;
 		if (cnt > 0)
 		{	
-			for (auto m: m_I2NPMsgs)
+			int ind = 0;
+			while (ind < cnt)
 			{
-				if (m->totalLen <= 1003)
+				auto tunnelMsg = CreateNextTunnelMessage (tunnelID, ind);
-					res.push_back (CreateNextTunnelMessage (tunnelID, m, m->totalLen));
+				if (!tunnelMsg) break;
-				else
+				res.push_back (tunnelMsg);
-				{
-					res.push_back (CreateNextTunnelMessage (tunnelID, m, 1003));
-					size_t remaining = m->data->GetLength () - m_NextOffset; // remaining payload
-					remaining += 7; // follow-on header
-					res.push_back (CreateNextTunnelMessage (tunnelID, m, remaining)); 
-				}	
-				delete m;
 			}
+			for (auto msg: m_I2NPMsgs)
+				delete msg;
 			m_I2NPMsgs.clear ();
 		}	
 		
@@ -64,7 +85,6 @@ namespace tunnel
 
 	size_t TunnelGatewayBuffer::CreateFirstFragment (TunnelMessageBlockExt * block, uint8_t * buf, size_t len)
 	{
-		if (block->deliveryInstructionsLen > len) return 0; // can't put even delivery instructions
 		size_t ret = 1;
 		buf[0] = block->deliveryType << 5; // flag
 		if (block->deliveryType == eDeliveryTypeTunnel)
@@ -80,7 +100,6 @@ namespace tunnel
 		size_t size = block->data->GetLength ();
 		if (block->totalLen > len) // entire message doesn't fit
 		{	
-			if (ret + 4 > len) return 0; // can't put delivery instructions with msgID
 			buf[0] |= 0x08; // set fragmented bit
 			m_NextMsgID = block->data->GetHeader ()->msgID;
 			*(uint32_t *)(buf + ret) = m_NextMsgID;
@@ -120,42 +139,86 @@ namespace tunnel
 		return ret;
 	}	
 
-	I2NPMessage * TunnelGatewayBuffer::CreateNextTunnelMessage (uint32_t tunnelID, 
+	I2NPMessage * TunnelGatewayBuffer::CreateNextTunnelMessage (uint32_t tunnelID, int& ind)
-		TunnelMessageBlockExt * block, size_t size)
 	{
+		int cnt = m_I2NPMsgs.size ();
+		if (ind > cnt - 1) return nullptr; // no more messages
+		// calculate payload size
+		size_t size = 0;
+		int i = ind;
+		if (m_NextOffset)
+		{	
+			size = m_I2NPMsgs[i]->data->GetLength () - m_NextOffset + 7; // including follow-on header
+			i++;
+		}	
+		while (i < cnt)
+		{	
+			auto msg = m_I2NPMsgs[i];
+			size += msg->totalLen;
+			if (size >= TUNNEL_DATA_MAX_PAYLOAD_SIZE)
+			{
+				size = TUNNEL_DATA_MAX_PAYLOAD_SIZE;
+				break;
+			}	
+			if (msg->isFragmented) break;
+			i++;
+		}
+		
 		I2NPMessage * tunnelMsg = NewI2NPMessage ();
 		uint8_t * buf = tunnelMsg->GetPayload ();
 		*(uint32_t *)(buf) = htobe32 (tunnelID);
 		CryptoPP::RandomNumberGenerator& rnd = i2p::context.GetRandomNumberGenerator ();
 		rnd.GenerateBlock (buf + 4, 16); // original IV	
-		memcpy (buf + 1028, buf + 4, 16); // copy IV for checksum 	
+		memcpy (buf + TUNNEL_DATA_MSG_SIZE, buf + 4, 16); // copy IV for checksum 	
-		size_t zero  = 1028 - size -1;
+		size_t zero  = TUNNEL_DATA_MSG_SIZE - size -1;
 		buf[zero] = 0; // zero
-	
+		size_t s = 0;
+		while (ind < cnt)
+		{
+			auto msg = m_I2NPMsgs[ind];
 			if (m_NextOffset)	
 			{	
-			size_t s = CreateFollowOnFragment (block, buf + zero + 1, 1003);
+				s += CreateFollowOnFragment (msg, buf + zero + 1 + s, size - s);
-			if (s != size)
+				m_NextOffset = 0; // TODO:
-				LogPrint ("Follow-on fragment size mismatch ", s, "!=", size);
 			}	
 			else
-			CreateFirstFragment (block, buf + zero + 1, 1003);
+			{	
+				s += CreateFirstFragment (msg, buf + zero + 1 + s, size - s);
+				if (msg->isFragmented) break; // payload is full, but we stay at the same message
+			}
+			ind++;
+			if (s >= size) break; //  payload is full but we moved to next message
+		}
+
+		if (s != size)
+		{	
+			LogPrint ("TunnelData payload size mismatch ", s, "!=", size);
+			return nullptr;
+		}	
 		
 		uint8_t hash[32];
 		CryptoPP::SHA256().CalculateDigest(hash, buf+zero+1, size+16);
 		memcpy (buf+20, hash, 4); // checksum
 		if (zero > 24)
-			memset (buf+24, 1, zero-24); // padding
+			memset (buf+24, 1, zero-24); // padding TODO: fill with random data
-		tunnelMsg->len += 1028;
+		tunnelMsg->len += TUNNEL_DATA_MSG_SIZE;
-		
 		// we can't fill message header yet because encryption is required
 		return tunnelMsg;
 	}	
 	
-
 	void TunnelGateway::SendTunnelDataMsg (const uint8_t * gwHash, uint32_t gwTunnel, i2p::I2NPMessage * msg)
+	{
+		PutTunnelDataMsg (gwHash, gwTunnel, msg);
+		SendBuffer ();
+	}	
+
+	void TunnelGateway::PutTunnelDataMsg (const uint8_t * gwHash, uint32_t gwTunnel, i2p::I2NPMessage * msg)
 	{
 		m_Buffer.PutI2NPMsg (gwHash, gwTunnel, msg);
+	}	
+
+	void TunnelGateway::SendBuffer ()
+	{
 		auto tunnelMsgs = m_Buffer.GetTunnelDataMsgs (m_Tunnel->GetNextTunnelID ());
 		for (auto tunnelMsg : tunnelMsgs)
 		{	

TunnelGateway.h

@@ -15,10 +15,13 @@ namespace tunnel
 		struct TunnelMessageBlockExt: public TunnelMessageBlock
 		{
 			size_t deliveryInstructionsLen, totalLen;
+			bool isFragmented;
 		};	
 		
 		public:
 
+			TunnelGatewayBuffer (): m_Remaining (0) {}; 
+			
 			void PutI2NPMsg (const uint8_t * gwHash, uint32_t gwTunnel, I2NPMessage * msg);	
 			std::vector<I2NPMessage *> GetTunnelDataMsgs (uint32_t tunnelID);
 
@@ -26,13 +29,13 @@ namespace tunnel
 
 			size_t CreateFirstFragment (TunnelMessageBlockExt * block, uint8_t * buf, size_t len);
 			size_t CreateFollowOnFragment (TunnelMessageBlockExt * block, uint8_t * buf, size_t len);
-			I2NPMessage * CreateNextTunnelMessage (uint32_t tunnelID, TunnelMessageBlockExt * block, size_t size);
+			I2NPMessage * CreateNextTunnelMessage (uint32_t tunnelID, int& ind);
 			
 		private:
 
 			std::vector<TunnelMessageBlockExt *> m_I2NPMsgs;
 			// for fragmented  messages
-			size_t m_NextOffset, m_NextSeqn;
+			size_t m_NextOffset, m_NextSeqn, m_Remaining;
 			uint32_t m_NextMsgID;
 	};	
 
@@ -42,6 +45,8 @@ namespace tunnel
 
 			TunnelGateway (TunnelBase * tunnel): m_Tunnel (tunnel) {};
 			void SendTunnelDataMsg (const uint8_t * gwHash, uint32_t gwTunnel, i2p::I2NPMessage * msg);
+			void PutTunnelDataMsg (const uint8_t * gwHash, uint32_t gwTunnel, i2p::I2NPMessage * msg);	
+			void SendBuffer ();
 			
 		private: