cpp data recorder
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
import socket
|
||||
import numpy as np
|
||||
import ctypes
|
||||
from ctypes import cdll
|
||||
import data_structures
|
||||
import threading
|
||||
import queue
|
||||
@@ -13,123 +14,32 @@ class DataRecorder:
|
||||
port=1234,
|
||||
packet_size=4096):
|
||||
|
||||
self.lib = cdll.LoadLibrary('../cpp/data_recorder.so')
|
||||
|
||||
# # TESTTTT
|
||||
# self.s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
||||
# print('SO_RCVBUF', self.s.getsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF))
|
||||
# self.s.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 4 * 1024 * 1024)
|
||||
# print('SO_RCVBUF', self.s.getsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF))
|
||||
# self.s.settimeout(1)
|
||||
# self.s.bind(("", 1234))
|
||||
# data = np.arange(16, dtype=np.uint32)
|
||||
# data = data.tobytes()
|
||||
# self.s.sendto(data, (host, 1234))
|
||||
# self.s.close()
|
||||
# # TESTTTTT
|
||||
self.lib.DataRecorder_new.argtypes = [ctypes.c_int32]
|
||||
self.lib.DataRecorder_new.restype = ctypes.c_int64
|
||||
|
||||
self.lib.DataRecorder_get_recording_rate.argtypes = [ctypes.c_int64]
|
||||
self.lib.DataRecorder_get_recording_rate.restype = ctypes.c_float
|
||||
|
||||
# self.lib.DataRecorder_get_current_recording_size.argtypes = [ctypes.c_int64]
|
||||
# self.lib.DataRecorder_get_current_recording_size.restype = ctypes.c_float
|
||||
|
||||
# UDP Socket for High Speed Data
|
||||
self.ip = host
|
||||
self.port = port
|
||||
self.s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
||||
print('SO_RCVBUF', self.s.getsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF))
|
||||
self.s.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 4 * 1024 * 1024)
|
||||
print('SO_RCVBUF', self.s.getsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF))
|
||||
self.s.settimeout(1)
|
||||
self.s.bind(("", port))
|
||||
# Need to send one udp message to set IP and port info inside FPGA
|
||||
data = np.arange(16, dtype=np.uint32)
|
||||
data = data.tobytes()
|
||||
self.s.sendto(data, (self.ip, self.port))
|
||||
self.lib.DataRecorder_start_recording.argtypes = [ctypes.c_int64, ctypes.c_char_p, ctypes.c_int64, ctypes.c_int32]
|
||||
self.lib.DataRecorder_start_recording.restype = ctypes.c_int32
|
||||
|
||||
self.lib.DataRecorder_stop_recording.argtypes = [ctypes.c_int64]
|
||||
self.lib.DataRecorder_stop_recording.restype = ctypes.c_int32
|
||||
|
||||
self.max_packet_size = packet_size
|
||||
|
||||
# Data Buffer
|
||||
# self.buffer = bytearray(512 * 1024 * 1024)
|
||||
self.buffer = mmap.mmap(-1, 512 * 1024 * 1024)
|
||||
self.buffer_view = memoryview(self.buffer)
|
||||
|
||||
self.stop_event = threading.Event()
|
||||
|
||||
self.fid = None
|
||||
self.write_to_disk = False
|
||||
self.write_offset = 0
|
||||
self.write_count = 0
|
||||
self.write_queue = queue.SimpleQueue()
|
||||
self.obj = self.lib.DataRecorder_new(port)
|
||||
|
||||
def start_recording(self, filename, write_to_disk=False):
|
||||
filename = filename.encode('utf-8')
|
||||
filename = ctypes.c_char_p(filename)
|
||||
self.lib.DataRecorder_start_recording(self.obj, filename, -1, 1)
|
||||
|
||||
self.write_to_disk = write_to_disk
|
||||
|
||||
if write_to_disk:
|
||||
self.write_offset = 0
|
||||
self.write_count = 0
|
||||
self.fid = os.open(filename, os.O_WRONLY | os.O_CREAT | os.O_TRUNC | os.O_DIRECT )
|
||||
|
||||
self.write_queue = queue.SimpleQueue()
|
||||
self.write_data_thread = threading.Thread(target=self.write_data)
|
||||
self.write_data_thread.start()
|
||||
|
||||
self.get_data_thread = threading.Thread(target=self.get_data)
|
||||
|
||||
self.get_data_thread.start()
|
||||
|
||||
def stop_recording(self):
|
||||
print('Stop Thread')
|
||||
self.stop_event.set()
|
||||
self.get_data_thread.join()
|
||||
print('Get Data Thread Joined')
|
||||
if self.write_to_disk:
|
||||
self.write_data_thread.join()
|
||||
print('Write Data Thread Joined')
|
||||
|
||||
def write_data(self):
|
||||
|
||||
write_chunk_size = 4 * 1024 * 1024
|
||||
buffer_view = memoryview(self.buffer)
|
||||
|
||||
print('Waiting For Data to Write')
|
||||
while not self.stop_event.is_set():
|
||||
|
||||
try:
|
||||
num_bytes = self.write_queue.get(timeout=1)
|
||||
|
||||
self.write_count += num_bytes
|
||||
|
||||
if self.write_count > write_chunk_size:
|
||||
# print(self.write_offset)
|
||||
# os.write(self.fid, self.buffer[self.write_offset:self.write_offset + write_chunk_size])
|
||||
os.write(self.fid, buffer_view[self.write_offset:self.write_offset + write_chunk_size])
|
||||
self.write_offset += write_chunk_size
|
||||
self.write_count -= write_chunk_size
|
||||
self.write_offset = self.write_offset % len(self.buffer)
|
||||
except queue.Empty:
|
||||
print('DR Queue Empty!', self.ip)
|
||||
|
||||
|
||||
def get_data(self):
|
||||
offset = 0
|
||||
|
||||
print('Waiting For Data From Socket')
|
||||
while not self.stop_event.is_set():
|
||||
|
||||
try:
|
||||
n = self.s.recv_into(self.buffer_view[offset:offset + self.max_packet_size])
|
||||
|
||||
if self.write_to_disk:
|
||||
# print(n)
|
||||
self.write_queue.put(n)
|
||||
|
||||
offset += n
|
||||
if offset > len(self.buffer):
|
||||
if self.port == 1234:
|
||||
print('hmmm', n, offset, len(self.buffer))
|
||||
offset = offset % len(self.buffer)
|
||||
# print(offset)
|
||||
|
||||
except socket.timeout:
|
||||
continue
|
||||
self.lib.DataRecorder_stop_recording(self.obj)
|
||||
|
||||
|
||||
|
||||
135
python/data_recorder.py.python_version
Executable file
135
python/data_recorder.py.python_version
Executable file
@@ -0,0 +1,135 @@
|
||||
import socket
|
||||
import numpy as np
|
||||
import ctypes
|
||||
import data_structures
|
||||
import threading
|
||||
import queue
|
||||
import os
|
||||
import mmap
|
||||
|
||||
class DataRecorder:
|
||||
def __init__(self,
|
||||
host="192.168.2.128",
|
||||
port=1234,
|
||||
packet_size=4096):
|
||||
|
||||
|
||||
# # TESTTTT
|
||||
# self.s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
||||
# print('SO_RCVBUF', self.s.getsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF))
|
||||
# self.s.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 4 * 1024 * 1024)
|
||||
# print('SO_RCVBUF', self.s.getsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF))
|
||||
# self.s.settimeout(1)
|
||||
# self.s.bind(("", 1234))
|
||||
# data = np.arange(16, dtype=np.uint32)
|
||||
# data = data.tobytes()
|
||||
# self.s.sendto(data, (host, 1234))
|
||||
# self.s.close()
|
||||
# # TESTTTTT
|
||||
|
||||
|
||||
|
||||
# UDP Socket for High Speed Data
|
||||
self.ip = host
|
||||
self.port = port
|
||||
self.s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
||||
print('SO_RCVBUF', self.s.getsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF))
|
||||
self.s.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 4 * 1024 * 1024)
|
||||
print('SO_RCVBUF', self.s.getsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF))
|
||||
self.s.settimeout(1)
|
||||
self.s.bind(("", port))
|
||||
# Need to send one udp message to set IP and port info inside FPGA
|
||||
data = np.arange(16, dtype=np.uint32)
|
||||
data = data.tobytes()
|
||||
self.s.sendto(data, (self.ip, self.port))
|
||||
|
||||
|
||||
self.max_packet_size = packet_size
|
||||
|
||||
# Data Buffer
|
||||
# self.buffer = bytearray(512 * 1024 * 1024)
|
||||
self.buffer = mmap.mmap(-1, 512 * 1024 * 1024)
|
||||
self.buffer_view = memoryview(self.buffer)
|
||||
|
||||
self.stop_event = threading.Event()
|
||||
|
||||
self.fid = None
|
||||
self.write_to_disk = False
|
||||
self.write_offset = 0
|
||||
self.write_count = 0
|
||||
self.write_queue = queue.SimpleQueue()
|
||||
|
||||
def start_recording(self, filename, write_to_disk=False):
|
||||
|
||||
self.write_to_disk = write_to_disk
|
||||
|
||||
if write_to_disk:
|
||||
self.write_offset = 0
|
||||
self.write_count = 0
|
||||
self.fid = os.open(filename, os.O_WRONLY | os.O_CREAT | os.O_TRUNC | os.O_DIRECT )
|
||||
|
||||
self.write_queue = queue.SimpleQueue()
|
||||
self.write_data_thread = threading.Thread(target=self.write_data)
|
||||
self.write_data_thread.start()
|
||||
|
||||
self.get_data_thread = threading.Thread(target=self.get_data)
|
||||
|
||||
self.get_data_thread.start()
|
||||
|
||||
def stop_recording(self):
|
||||
print('Stop Thread')
|
||||
self.stop_event.set()
|
||||
self.get_data_thread.join()
|
||||
print('Get Data Thread Joined')
|
||||
if self.write_to_disk:
|
||||
self.write_data_thread.join()
|
||||
print('Write Data Thread Joined')
|
||||
|
||||
def write_data(self):
|
||||
|
||||
write_chunk_size = 4 * 1024 * 1024
|
||||
buffer_view = memoryview(self.buffer)
|
||||
|
||||
print('Waiting For Data to Write')
|
||||
while not self.stop_event.is_set():
|
||||
|
||||
try:
|
||||
num_bytes = self.write_queue.get(timeout=1)
|
||||
|
||||
self.write_count += num_bytes
|
||||
|
||||
if self.write_count > write_chunk_size:
|
||||
# print(self.write_offset)
|
||||
# os.write(self.fid, self.buffer[self.write_offset:self.write_offset + write_chunk_size])
|
||||
os.write(self.fid, buffer_view[self.write_offset:self.write_offset + write_chunk_size])
|
||||
self.write_offset += write_chunk_size
|
||||
self.write_count -= write_chunk_size
|
||||
self.write_offset = self.write_offset % len(self.buffer)
|
||||
except queue.Empty:
|
||||
print('DR Queue Empty!', self.ip)
|
||||
|
||||
|
||||
def get_data(self):
|
||||
offset = 0
|
||||
|
||||
print('Waiting For Data From Socket')
|
||||
while not self.stop_event.is_set():
|
||||
|
||||
try:
|
||||
n = self.s.recv_into(self.buffer_view[offset:offset + self.max_packet_size])
|
||||
|
||||
if self.write_to_disk:
|
||||
# print(n)
|
||||
self.write_queue.put(n)
|
||||
|
||||
offset += n
|
||||
if offset > len(self.buffer):
|
||||
if self.port == 1234:
|
||||
print('hmmm', n, offset, len(self.buffer))
|
||||
offset = offset % len(self.buffer)
|
||||
# print(offset)
|
||||
|
||||
except socket.timeout:
|
||||
continue
|
||||
|
||||
|
||||
@@ -370,8 +370,8 @@ class RadarManager:
|
||||
rf_atten = [1, 2, 3, 4, 5, 6]
|
||||
self.setup_rf_attenuators(rf_atten)
|
||||
|
||||
adc_nco = 5e9 % f_adc
|
||||
dac_nco = 5.001e9 % f_dac
|
||||
adc_nco = 1e9 % f_adc
|
||||
dac_nco = 1.001e9 % f_dac
|
||||
|
||||
# adc_nco = 2e9
|
||||
# adc_nyquist_zone = np.floor(adc_nco / (f_adc / 2))
|
||||
|
||||
@@ -3,12 +3,8 @@ import os.path
|
||||
import time
|
||||
import numpy as np
|
||||
from matplotlib import pyplot as plt
|
||||
import socket
|
||||
|
||||
import data_structures
|
||||
import radar_manager
|
||||
from data_recorder import DataRecorder
|
||||
|
||||
|
||||
def db20(x):
|
||||
return 20*np.log10(np.abs(x))
|
||||
@@ -48,7 +44,6 @@ def main():
|
||||
header = data_structures.CpiHeader.from_buffer_copy(header)
|
||||
fid.seek(-ctypes.sizeof(data_structures.CpiHeader), 1)
|
||||
|
||||
|
||||
# CPI Parameters (timing values are in clk ticks)
|
||||
num_pulses = header.num_pulses
|
||||
num_samples = header.num_samples
|
||||
@@ -93,18 +88,19 @@ def main():
|
||||
|
||||
plt.figure()
|
||||
plt.plot(np.diff(cpi_times))
|
||||
plt.ylim([0, .02])
|
||||
plt.plot(np.diff(pps_frac))
|
||||
# plt.ylim([0, .04])
|
||||
|
||||
plt.figure()
|
||||
plt.plot(iq.T.real, '.-')
|
||||
plt.plot(iq.T.imag, '--.')
|
||||
plt.grid()
|
||||
|
||||
plt.figure()
|
||||
plt.imshow(db20n(iq), aspect='auto', interpolation='nearest', vmin=vmin, vmax=vmax)
|
||||
plt.ylabel('Pulse Count')
|
||||
plt.xlabel('Sample Count')
|
||||
plt.colorbar()
|
||||
# plt.figure()
|
||||
# plt.plot(iq.T.real, '.-')
|
||||
# plt.plot(iq.T.imag, '--.')
|
||||
# plt.grid()
|
||||
#
|
||||
# plt.figure()
|
||||
# plt.imshow(db20n(iq), aspect='auto', interpolation='nearest', vmin=vmin, vmax=vmax)
|
||||
# plt.ylabel('Pulse Count')
|
||||
# plt.xlabel('Sample Count')
|
||||
# plt.colorbar()
|
||||
|
||||
|
||||
plt.show()
|
||||
|
||||
@@ -45,13 +45,16 @@ def main():
|
||||
# CPI Parameters (timing values are in clk ticks)
|
||||
num_pulses = 128
|
||||
# Should be multiple of udp packet size, currently 4096 bytes, or 1024 samples
|
||||
num_samples = 5000
|
||||
num_samples = 16384
|
||||
start_sample = 2000
|
||||
tx_num_samples = 1024
|
||||
tx_start_sample = start_sample
|
||||
pri = int(.0004 * clk)
|
||||
prf = 8000
|
||||
pri = int(1/prf * clk)
|
||||
pri -= (pri % 3)
|
||||
# pri = int(.0001 * clk)
|
||||
print(pri)
|
||||
inter_cpi = 50
|
||||
inter_cpi = 2000
|
||||
tx_lo_offset = 10e6
|
||||
rx_lo_offset = 0
|
||||
|
||||
@@ -63,9 +66,9 @@ def main():
|
||||
|
||||
|
||||
recorder0 = DataRecorder("192.168.2.128", 1234, packet_size=radar.packet_size)
|
||||
recorder1 = DataRecorder("192.168.3.128", 1235, packet_size=radar.packet_size)
|
||||
# recorder1 = DataRecorder("192.168.3.128", 1235, packet_size=radar.packet_size)
|
||||
recorder0.start_recording('test0.bin', True)
|
||||
recorder1.start_recording('test1.bin', True)
|
||||
# recorder1.start_recording('test1.bin', True)
|
||||
|
||||
radar.configure_cpi(pri, inter_cpi, num_pulses, num_samples, start_sample,
|
||||
tx_num_samples, tx_start_sample, rx_lo_offset, tx_lo_offset)
|
||||
@@ -73,83 +76,83 @@ def main():
|
||||
print('Start Running')
|
||||
radar.start_running()
|
||||
# Let it run for a bit
|
||||
time.sleep(2)
|
||||
time.sleep(60)
|
||||
# Stop running
|
||||
radar.stop_running()
|
||||
# Stop the data recorder
|
||||
recorder0.stop_recording()
|
||||
recorder1.stop_recording()
|
||||
|
||||
# Parse some data
|
||||
|
||||
# Find header, recording buffer could have wrapped depending on data rate and how long we ran for
|
||||
recorders = [recorder0, recorder1]
|
||||
for recorder in recorders:
|
||||
headers = []
|
||||
offset = 0
|
||||
plot_recorder = recorder
|
||||
hdr_sync = False
|
||||
while not hdr_sync:
|
||||
data = plot_recorder.buffer[offset:offset + 4]
|
||||
sync_word = np.frombuffer(data, dtype=np.uint32)[0]
|
||||
if sync_word == 0xAABBCCDD:
|
||||
hdr_sync = True
|
||||
print('Header found at offset', offset)
|
||||
else:
|
||||
offset += 4
|
||||
|
||||
num_cpi = 1
|
||||
for i in range(num_cpi):
|
||||
# Get Header
|
||||
data = plot_recorder.buffer[offset:offset + ctypes.sizeof(data_structures.CpiHeader)]
|
||||
offset += ctypes.sizeof(data_structures.CpiHeader)
|
||||
headers.append(data_structures.CpiHeader.from_buffer_copy(data))
|
||||
num_pulses = headers[i].num_pulses
|
||||
num_samples = headers[i].num_samples
|
||||
|
||||
# Get CPI
|
||||
data_size = num_pulses * num_samples * 4
|
||||
data = plot_recorder.buffer[offset:offset + data_size]
|
||||
offset += data_size
|
||||
|
||||
# Check some header fields
|
||||
cpi_times = np.array([x.system_time for x in headers]) / 187.5e6
|
||||
pps_frac = np.array([x.pps_frac_sec for x in headers]) / 187.5e6
|
||||
pps_sec = np.array([x.pps_sec for x in headers])
|
||||
utc_time = pps_sec + pps_frac
|
||||
print(pri, inter_cpi, num_pulses * pri + inter_cpi)
|
||||
print(cpi_times - cpi_times[0])
|
||||
print(pps_frac)
|
||||
print(pps_sec - pps_sec[0])
|
||||
|
||||
# Plot last CPI
|
||||
data2 = np.frombuffer(data, dtype=np.int16)
|
||||
i = data2[0::2]
|
||||
q = data2[1::2]
|
||||
iq = i + 1j * q
|
||||
iq = iq.reshape(-1, num_samples)
|
||||
iq = iq + 1e-15
|
||||
|
||||
vmin = -60
|
||||
vmax = 0
|
||||
|
||||
fid, axs = plt.subplots(3)
|
||||
axs[0].plot(iq.T.real, '-')
|
||||
axs[0].plot(iq.T.imag, '--')
|
||||
axs[0].grid()
|
||||
|
||||
# axs[1].imshow(db20n(iq), aspect='auto', interpolation='nearest', vmin=vmin, vmax=vmax)
|
||||
axs[1].imshow(iq.real, aspect='auto', interpolation='nearest')
|
||||
axs[1].set_ylabel('Pulse Count')
|
||||
axs[1].set_xlabel('Sample Count')
|
||||
|
||||
iq_freq = np.fft.fftshift(np.fft.fft(iq, axis=1), axes=1)
|
||||
freq_axis = (np.arange(num_samples)/num_samples - 0.5) * radar_manager.BASEBAND_SAMPLE_RATE / 1e6
|
||||
axs[2].plot(freq_axis, db20n(iq_freq.T))
|
||||
axs[2].grid()
|
||||
|
||||
|
||||
plt.show()
|
||||
# recorder1.stop_recording()
|
||||
#
|
||||
# # Parse some data
|
||||
#
|
||||
# # Find header, recording buffer could have wrapped depending on data rate and how long we ran for
|
||||
# recorders = [recorder0, recorder1]
|
||||
# for recorder in recorders:
|
||||
# headers = []
|
||||
# offset = 0
|
||||
# plot_recorder = recorder
|
||||
# hdr_sync = False
|
||||
# while not hdr_sync:
|
||||
# data = plot_recorder.buffer[offset:offset + 4]
|
||||
# sync_word = np.frombuffer(data, dtype=np.uint32)[0]
|
||||
# if sync_word == 0xAABBCCDD:
|
||||
# hdr_sync = True
|
||||
# print('Header found at offset', offset)
|
||||
# else:
|
||||
# offset += 4
|
||||
#
|
||||
# num_cpi = 1
|
||||
# for i in range(num_cpi):
|
||||
# # Get Header
|
||||
# data = plot_recorder.buffer[offset:offset + ctypes.sizeof(data_structures.CpiHeader)]
|
||||
# offset += ctypes.sizeof(data_structures.CpiHeader)
|
||||
# headers.append(data_structures.CpiHeader.from_buffer_copy(data))
|
||||
# num_pulses = headers[i].num_pulses
|
||||
# num_samples = headers[i].num_samples
|
||||
#
|
||||
# # Get CPI
|
||||
# data_size = num_pulses * num_samples * 4
|
||||
# data = plot_recorder.buffer[offset:offset + data_size]
|
||||
# offset += data_size
|
||||
#
|
||||
# # Check some header fields
|
||||
# cpi_times = np.array([x.system_time for x in headers]) / 187.5e6
|
||||
# pps_frac = np.array([x.pps_frac_sec for x in headers]) / 187.5e6
|
||||
# pps_sec = np.array([x.pps_sec for x in headers])
|
||||
# utc_time = pps_sec + pps_frac
|
||||
# print(pri, inter_cpi, num_pulses * pri + inter_cpi)
|
||||
# print(cpi_times - cpi_times[0])
|
||||
# print(pps_frac)
|
||||
# print(pps_sec - pps_sec[0])
|
||||
#
|
||||
# # Plot last CPI
|
||||
# data2 = np.frombuffer(data, dtype=np.int16)
|
||||
# i = data2[0::2]
|
||||
# q = data2[1::2]
|
||||
# iq = i + 1j * q
|
||||
# iq = iq.reshape(-1, num_samples)
|
||||
# iq = iq + 1e-15
|
||||
#
|
||||
# vmin = -60
|
||||
# vmax = 0
|
||||
#
|
||||
# fid, axs = plt.subplots(3)
|
||||
# axs[0].plot(iq.T.real, '-')
|
||||
# axs[0].plot(iq.T.imag, '--')
|
||||
# axs[0].grid()
|
||||
#
|
||||
# # axs[1].imshow(db20n(iq), aspect='auto', interpolation='nearest', vmin=vmin, vmax=vmax)
|
||||
# axs[1].imshow(iq.real, aspect='auto', interpolation='nearest')
|
||||
# axs[1].set_ylabel('Pulse Count')
|
||||
# axs[1].set_xlabel('Sample Count')
|
||||
#
|
||||
# iq_freq = np.fft.fftshift(np.fft.fft(iq, axis=1), axes=1)
|
||||
# freq_axis = (np.arange(num_samples)/num_samples - 0.5) * radar_manager.BASEBAND_SAMPLE_RATE / 1e6
|
||||
# axs[2].plot(freq_axis, db20n(iq_freq.T))
|
||||
# axs[2].grid()
|
||||
#
|
||||
#
|
||||
# plt.show()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
Reference in New Issue
Block a user