DEV Community

nima
nima

Posted on

Real Time Lane Detection — python opencv

#numpy #python #opencv #lanedetection

Overview
Lane detection is one of the most crucial technique of ADAS and has received significant attention recently. In this project, we achived lane detection with real time by numpy and multi-thread.

Dependencies:

Python
Numpy
Opencv

How to Run:

Run lane_detection.py. The default video is project_video, if you want to process the "fog_video.mp4", change video_index to 1 in line 9.

full source code:
import numpy as np
import cv2
import time
from threading import Thread
from queue import Queue

defualt video number, if you want to process the "fog_video.mp4", change video_index to 1

video_index = 0

the result of lane detection, we add the road to the main frame

road = np.zeros((720, 1280, 3))

A flag which means the process is started

started = 0

Pipeline combining color and gradient thresholding

def thresholding_pipeline(img, s_thresh=(90, 255), sxy_thresh=(20, 100)):
img = np.copy(img)
# 1: Convert to HSV color space and separate the V channel
hls = cv2.cvtColor(img, cv2.COLOR_RGB2HLS).astype(np.float)
h_channel = hls[:, :, 0]
l_channel = hls[:, :, 1]
s_channel = hls[:, :, 2]

2: Calculate x directional gradient 

sobelx = cv2.Sobel(l_channel, cv2.CV_64F, 1, 0)  # Take the derivative in x
# Absolute x derivative to accentuate lines away from horizontal
abs_sobelx = np.absolute(sobelx)
scaled_sobelx = np.uint8(255 * abs_sobelx / np.max(abs_sobelx))
sxbinary = np.zeros_like(scaled_sobelx)
sxbinary[(scaled_sobelx >= sxy_thresh[0]) &
         (scaled_sobelx <= sxy_thresh[1])] = 1
grad_thresh = sxbinary
Enter fullscreen mode Exit fullscreen mode

3: Color Threshold of s channel 

s_binary = np.zeros_like(s_channel)
s_binary[(s_channel >= s_thresh[0]) & (s_channel <= s_thresh[1])] = 1
Enter fullscreen mode Exit fullscreen mode

4: Combine the two binary thresholds 

combined_binary = np.zeros_like(grad_thresh)
combined_binary[(s_binary == 1) | (grad_thresh == 1)] = 1
Enter fullscreen mode Exit fullscreen mode

return combined_binary

Apply perspective transformation to bird's eye view

def perspective_transform(img, src_mask, dst_mask):
img_size = (img.shape[1], img.shape[0])
src = np.float32(src_mask)
dst = np.float32(dst_mask)
M = cv2.getPerspectiveTransform(src, dst)
warped_img = cv2.warpPerspective(img, M, img_size, flags=cv2.INTER_LINEAR)
return warped_img

Implement Sliding Windows and Fit a Polynomial

def sliding_windows(binary_warped, nwindows=9):
histogram = np.sum(
binary_warped[int(binary_warped.shape[0]/2):, :], axis=0)

Find the peak of the left and right halves of the histogram 

# These will be the starting point for the left and right lines
midpoint = np.int(histogram.shape[0]/2)
leftx_base = np.argmax(histogram[:midpoint])
rightx_base = np.argmax(histogram[midpoint:]) + midpoint
Enter fullscreen mode Exit fullscreen mode

Set height of windows 

window_height = np.int(binary_warped.shape[0]/nwindows)
# Identify the x and y positions of all nonzero pixels in the image
nonzero = binary_warped.nonzero()
nonzeroy = np.array(nonzero[0])
nonzerox = np.array(nonzero[1])
# Current positions to be updated for each window
leftx_current = leftx_base
rightx_current = rightx_base
# Set the width of the windows +/- margin
margin = 100
# Set minimum number of pixels found to recenter window
minpix = 50
# Create empty lists to receive left and right lane pixel indices
left_lane_inds = []
right_lane_inds = []
Enter fullscreen mode Exit fullscreen mode

Step through the windows one by one 

for window in range(nwindows):
    # Identify window boundaries in x and y (and right and left)
    win_y_low = binary_warped.shape[0] - (window+1)*window_height
    win_y_high = binary_warped.shape[0] - window*window_height
    win_xleft_low = leftx_current - margin
    win_xleft_high = leftx_current + margin
    win_xright_low = rightx_current - margin
    win_xright_high = rightx_current + margin
    # Identify the nonzero pixels in x and y within the window
    good_left_inds = ((nonzeroy >= win_y_low) & (nonzeroy < win_y_high) & (
        nonzerox >= win_xleft_low) & (nonzerox < win_xleft_high)).nonzero()[0]
    good_right_inds = ((nonzeroy >= win_y_low) & (nonzeroy < win_y_high) & (
        nonzerox >= win_xright_low) & (nonzerox < win_xright_high)).nonzero()[0]
    # Append these indices to the lists
    left_lane_inds.append(good_left_inds)
    right_lane_inds.append(good_right_inds)
    # If you found > minpix pixels, recenter next window on their mean position
    if len(good_left_inds) > minpix:
        leftx_current = np.int(np.mean(nonzerox[good_left_inds]))
    if len(good_right_inds) > minpix:
        rightx_current = np.int(np.mean(nonzerox[good_right_inds]))
Enter fullscreen mode Exit fullscreen mode

Concatenate the arrays of indices 

left_lane_inds = np.concatenate(left_lane_inds)
right_lane_inds = np.concatenate(right_lane_inds)
Enter fullscreen mode Exit fullscreen mode

Extract left and right line pixel positions 

leftx = nonzerox[left_lane_inds]
lefty = nonzeroy[left_lane_inds]
rightx = nonzerox[right_lane_inds]
righty = nonzeroy[right_lane_inds]
Enter fullscreen mode Exit fullscreen mode

Fit a second order polynomial to each 

left_fit = np.polyfit(lefty, leftx, 2)
right_fit = np.polyfit(righty, rightx, 2)
Enter fullscreen mode Exit fullscreen mode

return left_fit, right_fit, lefty, leftx, righty, rightx

Warp lane line projection back to original image

def project_lanelines(binary_warped, orig_img, left_fit, right_fit, dst_mask, src_mask):
global road
global started
ploty = np.linspace(0, binary_warped.shape[0]-1, binary_warped.shape[0])
left_fitx = left_fit[0]ploty2 + left_fit[1]*ploty + left_fit[2]
right_fitx = right_fit[0]*ploty*2 + right_fit[1]*ploty + right_fit[2]

Create an image to draw the lines on 

warp_zero = np.zeros_like(binary_warped).astype(np.uint8)
color_warp = np.dstack((warp_zero, warp_zero, warp_zero))
Enter fullscreen mode Exit fullscreen mode

Recast the x and y points into usable format for cv2.fillPoly() 

pts_left = np.array([np.transpose(np.vstack([left_fitx, ploty]))])
pts_right = np.array(
    [np.flipud(np.transpose(np.vstack([right_fitx, ploty])))])
pts = np.hstack((pts_left, pts_right))
Enter fullscreen mode Exit fullscreen mode

Draw the lane onto the warped blank image 

cv2.fillPoly(color_warp, np.int_([pts]), (0, 255, 0))
# Warp the blank back to original image space using inverse perspective matrix (Minv)
warped_inv = perspective_transform(color_warp, dst_mask, src_mask)
road = warped_inv
started = 1
Enter fullscreen mode Exit fullscreen mode

Main process functions

def main_pipeline(input):

step 1 select the ROI, and we need to distort the image for fog_video 

if video_index == 0:
    image = input
    top_left = [540, 460]
    top_right = [754, 460]
    bottom_right = [1190, 670]
    bottom_left = [160, 670]
else:
    mtx = np.array([[1.15396467e+03, 0.00000000e+00, 6.69708251e+02], [0.00000000e+00, 1.14802823e+03, 3.85661017e+02],
                    [0.00000000e+00, 0.00000000e+00, 1.00000000e+00]])
    dist = np.array([[-2.41026561e-01, -5.30262184e-02, -
                      1.15775369e-03, -1.27924043e-04, 2.66417032e-02]])
    image = cv2.undistort(input, mtx, dist, None, mtx)
    top_left = [240, 270]
    top_right = [385, 270]
    bottom_right = [685, 402]
    bottom_left = [0, 402]
src_mask = np.array([[(top_left[0], top_left[1]), (top_right[0], top_right[1]),
                      (bottom_right[0], bottom_right[1]), (bottom_left[0], bottom_left[1])]], np.int32)
dst_mask = np.array([[(bottom_left[0], 0), (bottom_right[0], 0),
                      (bottom_right[0], bottom_right[1]), (bottom_left[0], bottom_left[1])]], np.int32)
Enter fullscreen mode Exit fullscreen mode

Step 2 Thresholding: color and gradient thresholds to generate a binary image 

binary_image = thresholding_pipeline(image, s_thresh=(90, 255))
Enter fullscreen mode Exit fullscreen mode

Step 3 Perspective transform on binary image: 

binary_warped = perspective_transform(binary_image, src_mask, dst_mask)
Enter fullscreen mode Exit fullscreen mode

Step 4 Fit Polynomial 

left_fit, right_fit, lefty, leftx, righty, rightx = sliding_windows(
    binary_warped, nwindows=9)
Enter fullscreen mode Exit fullscreen mode

Step 5 Project Lines 

project_lanelines(binary_warped, image, left_fit,
                  right_fit, dst_mask, src_mask)
Enter fullscreen mode Exit fullscreen mode

if name == 'main':
frames_counts = 1
if video_index == 0:
cap = cv2.VideoCapture('project_video.mp4')
else:
cap = cv2.VideoCapture('fog_video.mp4')
class MyThread(Thread):
def init(self, q):
Thread.init(self)
self.q = q
def run(self):
while(1):
if (not self.q.empty()):
image = self.q.get()
main_pipeline(image)
q = Queue()
q.queue.clear()
thd1 = MyThread(q)
thd1.setDaemon(True)
thd1.start()
while (True):
start = time.time()
ret, frame = cap.read()

Detect the lane every 5 frames 

    if frames_counts % 5 == 0:
        q.put(frame)
Enter fullscreen mode Exit fullscreen mode

Add the lane image on the original frame if started 

    if started:
        frame = cv2.addWeighted(frame, 1, road, 0.5, 0)
    cv2.imshow("RealTime_lane_detection", frame)
    if cv2.waitKey(1) & 0xFF == ord('q'):
        break
    frames_counts += 1
    cv2.waitKey(12)
    finish = time.time()
    print('FPS:  ' + str(int(1/(finish-start))))
Enter fullscreen mode Exit fullscreen mode

cap.release()
cv2.destroyAllWindows()

Github link:
https://github.com/nimadorostkar/realtime_lane_detection

Top comments (0)

Read next

spwoodcock profile image

FastAPI, Pydantic, Psycopg3: the holy trinity for Python web APIs

Sam -

m_sea_bass profile image

Janus 1.3B: A Unified Model for Multimodal Understanding and Generation Tasks

M Sea Bass -

vincenttommi profile image

2. Understanding Django’s Architecture: The MTV Pattern.

Mr #1738 -

bailon profile image

How to Set Up Selenium as a Linux Daemon with systemd

Juan Bailon -