DEV Community

arjun sharma
arjun sharma

Posted on

Unable to fetch data and make the csv file of the serial port data.

i am fetching data using serial port in my python program. the data contain 30000 data points. which i have to save in CSV files but the data is not saving in csv files.

from configuration.config import COM_PORT, BAUD_RATE
import serial
import threading
import time
import logging
import os
import csv
import requests

buffer1 = []
buffer2 = []
active_buffer = buffer1
lock = threading.Lock()

text_file_name = 'data.txt'
csv_file_prefix = 'data'
csv_file_counter = 1
csv_file_name = f"{csv_file_prefix}_{csv_file_counter}.csv"

stop_event = threading.Event()

highest_srno = 29999

def get_new_csv_filename():
global csv_file_counter
filename = f"{csv_file_prefix}_{csv_file_counter}.csv"
csv_file_counter += 1
return filename

def read_data():
global active_buffer, highest_srno, csv_file_name
try:
ser = serial.Serial(COM_PORT, BAUD_RATE)
print(f"Connected to the sensor with the above credentials")

    while not stop_event.is_set():
        if ser.in_waiting > 0:  # Check if there is data available to read
            data = ser.readline().decode('utf-8').strip()
            try:
                SerialNo, x, y, z = map(int, data.split(','))
                print(f"SRNO:{SerialNo}, X:{x}, Y:{y}, Z:{z}")

                # # This is the code to send the data to API
                # payload = {
                #     "sr_no": SerialNo, 
                #     "x": x, 
                #     "y": y, 
                #     "z": z
                # }
                # try:
                #     response = requests.post(API_endpoints, json=payload, timeout=10)
                #     if response.status_code == 200:
                #         print("Data sent to API")
                #     else:
                #         print(f"Failed to send data to API {response.status_code} - {response.text}")
                # except requests.exceptions.RequestException as e:
                #     print(f"Request failed: {e}")

                with lock:
                    if SerialNo > highest_srno:
                        if  highest_srno == 29999:
                            csv_file_name = get_new_csv_filename()
                            highest_srno = 0
                        else:
                            highest_srno = SerialNo
                    active_buffer.append(f"SRNO:{SerialNo},X:{x},Y:{y},Z:{z}")
            except ValueError:
                print(f"Error: Data format incorrect, skipping this line")

except Exception as e:
    logging.error(f"Error: {e}")

finally:
    if ser.is_open:
        ser.close()
        print("Serial connection closed.")
Enter fullscreen mode Exit fullscreen mode

def write_data_to_file():
global active_buffer, buffer1, buffer2, csv_file_name
while not stop_event.is_set():
time.sleep(4) # Periodically write data every 4 seconds
with lock:
buffer_to_write = active_buffer
if active_buffer == buffer1:
active_buffer = buffer2
else:
active_buffer = buffer1

    if buffer_to_write:
        try:
            # Write to text file
            with open(text_file_name, 'a') as f:
                for line in buffer_to_write:
                    f.write(line + '\n')

            # Write to CSV file
            with open(csv_file_name, 'a', newline='') as csvfile:
                csv_writer = csv.writer(csvfile)
                for line in buffer_to_write:
                    parts = line.split(',')
                    serial_no = parts[0].split(':')[1]
                    x = parts[1].split(':')[1]
                    y = parts[2].split(':')[1]
                    z = parts[3].split(':')[1]
                    csv_writer.writerow([serial_no, x, y, z])

            buffer_to_write.clear()

        except IOError as e:
            logging.error(f"File error: {e}")
Enter fullscreen mode Exit fullscreen mode

if name == "main":
logging.basicConfig(level=logging.ERROR, format='%(asctime)s - %(levelname)s - %(message)s')

write_thread = threading.Thread(target=write_data_to_file)
write_thread.daemon = True
write_thread.start()

try:
    read_data()
except KeyboardInterrupt:
    print("Stopping...")
    stop_event.set()
    write_thread.join()
    print("Program terminated.")
Enter fullscreen mode Exit fullscreen mode

Top comments (0)