Convert a pipe delimited TCP data stream into JSON records in Python
###This script gets raw TCP data from a IOT device endpoint and generates JSON records
import socket
import time
import json
import csv
host = 'x.x.x.x' # The TCP data sending remote host
port = xxxx # TCP port used by the remote host
num_msg = 100 # number of TCP message to get per batch of rows
# Specify the column names. Note: stx=start of text row delimiter etx=end of text
# If the remote host doesnt have header fields we set up the fixed column names
column_names = ["col1","col2","col3","col4","col5","col6","col7","col8","col9","col10","col11"]
def convert_pipe_delimited_csv_to_json(data,column_names):
#print('Received1:', str(data))
rows = data.split(b'\x02') # \x02 is the row delimiter in the TCP data
#print('Received2:', str(rows))
json_data = []
for row in rows:
# Skip the first and last rows (STX and ETX)
if row == b'' or row == b'\x02':
continue
row = row.decode('utf-8')
#print('Received3:', str(row))
columns = row.split("|")
json_object = {}
for i in range(len(columns)):
json_object[column_names[i]] = columns[i]
json_object[column_names[0]] = 'STX02' #col1 is hardcoded here
json_data.append(json_object)
return json.dumps(json_data)
def main():
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((host, port))
for i in range(num_msg): # can adjust how many recv calls before json data output
#print("sock.recv=",i)
data = sock.recv(1024)
json_data = convert_pipe_delimited_csv_to_json(data,column_names)
print(json_data)
time.sleep(0.1)
sock.close()
if __name__ == "__main__":
main()
###################################################### example output ###########################################
#Received1: b'\x02|2|3|4|5|6|7|8|9|10|11\x02'
# JSON output generated from pipe delimited data stream. num_msg=100 rows at a time.
#[{"col1": "STX02", "col2": "2", "col3": "3", "col4": "4", "col5": "5", "col6": "6", "col7": "7", "col8": "8", "col9": "9", "col10": "10", "col11": "11"}]
