In [1]:
"""
Weerstanden herkennen
HU elektrotechniek - beeldherkenning - september 2020
Pieter Baelde 1744256
Ewoud Dronkert 1743524
"""
import numpy as np
import cv2
from matplotlib import pyplot as plt
from math import sqrt, sin, cos, atan2, pi
from colorsys import rgb_to_hsv
from scipy.signal import argrelextrema
from glob import glob
# Names for colour numbers 0-9
colournames = np.array(['zwart', 'bruin', 'rood', 'oranje', 'geel', 'groen', 'blauw', 'paars', 'grijs', 'wit'])
multipliernames = {-1: 'goud', -2: 'zilver', -3: 'roze'}
tolerancevals = {'g': -1, 's': -2, 'p': -3}
colourrgb = {
-3: "#FF69B4",
-2: "#C0C0C0",
-1: "#CFB53B",
0 : "#000000",
1 : "#964B00",
2 : "#FF0000",
3 : "#FFA500",
4 : "#FFFF00",
5 : "#9ACD32",
6 : "#6495ED",
7 : "#9400D3",
8 : "#A0A0A0",
9 : "#FFFFFF"}
# Region of interest based on resistor position
# pos: (4x int) coordinates of endpoints of resistor
# imgshape: (2x int) height and width of image = maximum y and x coordinate + 1
# margin: (int) how many pixels to add on each side
# returns: (4x int) region of interest, can be used to crop image
def cropregion(pos, imgshape, margin=50):
# Unpack
x0, y0, x1, y1 = pos
# Sort and add margin
if x0 <= x1:
x0, x1 = x0 - margin, x1 + margin
else:
x0, x1 = x1 - margin, x0 + margin
if y0 <= y1:
y0, y1 = y0 - margin, y1 + margin
else:
y0, y1 = y1 - margin, y0 + margin
# Check image boundary
x0 = max(0, x0)
y0 = max(0, y0)
x1 = min(x1, imgshape[1])
y1 = min(y1, imgshape[0])
# Region of interest tuple
return x0, y0, x1, y1
In [2]:
"""
Detect resistor position in image
@author pieter.baelde@student.hu.nl 1744256
"""
def detectposition(filename, blurradius=15, erodesize=17, draw=False):
# Read from disk
img = cv2.imread(filename)
# Resize to lose some fine detail
shrink = 2 # reduce image size by factor
img = cv2.resize(img, (img.shape[1] // shrink, img.shape[0] // shrink))
# Need clean copy if position will be shown on image
if draw:
imgpos = img.copy()
font = cv2.FONT_HERSHEY_PLAIN
# Find colours on white paper
imghsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV) # convert BGR to HSV
lo = np.array([0, 65, 0]) # minimum values of the pixels
hi = np.array([179, 255, 255]) # maximum values of the pixels
mask = cv2.inRange(imghsv, lo, hi) # check which pixels are within the min and max
img = cv2.bitwise_and(img, img, mask=mask) # get original colours back
# Blur
img = cv2.GaussianBlur(img, (blurradius, blurradius), 0)
# Remove resistor legs
img = cv2.erode(img, np.ones((erodesize, erodesize), np.uint8))
# Find resistor
img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) # convert to grayscale
contours, _ = cv2.findContours(img, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
approx = False
for cnt in contours:
approx = cv2.approxPolyDP(cnt, 0.3 * cv2.arcLength(cnt, True), True)
if draw:
# Draw red line along the resistor
cv2.drawContours(imgpos, [approx], 0, (0, 0, 255), 3)
# Print text at contour
x = approx.ravel()[0]
y = approx.ravel()[1]
if len(approx) == 3:
cv2.putText(imgpos, "Triangle", (x, y), font, 1, (0))
elif len(approx) == 4:
cv2.putText(imgpos, "Rectangle", (x, y), font, 1, (0))
elif len(approx) == 5:
cv2.putText(imgpos, "Pentagon", (x, y), font, 1, (0))
elif 6 < len(approx) < 15:
cv2.putText(imgpos, "Ellipse", (x, y), font, 1, (0))
else:
cv2.putText(imgpos, "Circle", (x, y), font, 1, (0))
# Result for resized image (only use first contour found)
pos = (approx[0,0,0], approx[0,0,1], approx[1,0,0], approx[1,0,1])
if draw:
# Only show the area around the contour
roi = cropregion(pos, img.shape)
img = img[roi[1]:roi[3], roi[0]:roi[2]]
imgpos = imgpos[roi[1]:roi[3], roi[0]:roi[2]]
# Show processed and detected
dpi = 96 # approximately actual size
w = imgpos.shape[1] * 2.1 # 2 columns
h = imgpos.shape[0] # 1 row
plt.figure(figsize=(w/dpi, h/dpi), dpi=dpi)
plt.subplot(121)
plt.title("Bewerkt")
plt.imshow(img) # grayscale with standard colour space
plt.subplot(122)
plt.title("Positie")
plt.imshow(cv2.cvtColor(imgpos, cv2.COLOR_BGR2RGB))
plt.tight_layout()
plt.show()
# Result for original image
return np.array(pos, dtype=int) * shrink
In [3]:
"""
Detect colours in image at specified position
@author ewoud.dronkert@student.hu.nl
"""
# Sample BGR and HSV values along lines perpendicular to the resistor long axis.
# filename: (string) colour image on disk
# pos: (4x int) x0,y0,x1,y1 coordinates of endpoints of the resistor
# draw: (bool) show images yes or no
# returns: (2x 2D Numpy array) mean BGR and HSV values of each sample line
def samplecolours(filename, pos, draw=False):
img = cv2.imread(filename, 1) # 1=colour image
if draw:
detect = img.copy()
# Imagine the resistor lying horizontally and a sampling grid on top of it.
# This would be 6 lines of 3 points each:
# +-+-+-+-+-+
# | | | | | |
# --+-+-+-+-+-+--
# | | | | | |
# +-+-+-+-+-+
lines = 200 # number of sample lines along the length of the resistor
points = 50 # number of points per sample line
# Coordinates of the endpoints of the resistor = position detection line
x0, y0, x1, y1 = pos
# Draw thick red line over the length of the resistor
# if draw:
# cv2.line(detect, (x0, y0), (x1, y1), (0,0,255), 10)
# Resistor dimensions
rdx = x0 - x1 # resistor dx
rdy = y0 - y1 # resistor dy
rlen = sqrt(rdx * rdx + rdy * rdy) # resistor length
# Sample line dimensions perpendicular to the position detection line
slen = rlen / 8 # sample line length (from centre) relative to resistor length
sang = atan2(rdy, rdx) + pi / 2 # sample line angle perpendicular to the resistor
sdx = slen * cos(sang) # sample line dx (from centre)
sdy = slen * sin(sang) # sample line dy (from centre)
# Coordinates along the resistor on the position detection line
rx = np.linspace(x0, x1, num=lines)
ry = np.linspace(y0, y1, num=lines)
# BGR and HSV arrays with average colour values along the sample lines
bgr = np.zeros((lines, 3), dtype=np.float64)
hsv = np.zeros((lines, 3), dtype=np.float64)
# Sampling lines along the length of the resistor
for i in range(lines):
# Endpoint tuples of the sample line
sxy1 = (round(rx[i] + sdx), round(ry[i] + sdy))
sxy2 = (round(rx[i] - sdx), round(ry[i] - sdy))
# Draw thin yellow line on the sample line
if draw:
cv2.line(detect, sxy1, sxy2, (0,255,255), 1)
# Coordinates along the sample line
sx = np.linspace(sxy1[0], sxy2[0], num=points)
sy = np.linspace(sxy1[1], sxy2[1], num=points)
# Average of BGR colour values along the sample line
avg = np.zeros(3, dtype=np.float64)
# Summing points along the sample line
# and calculating the average
for j in range(points):
avg += img[round(sy[j]), round(sx[j])]
avg /= points
# Save BGR and HSV results of the sample line
# all values must be or are in range 0.0 - 1.0
bgr[i] = avg / 255
hsv[i] = rgb_to_hsv(bgr[i, 2], bgr[i, 1], bgr[i, 0])
# Hue correction modulo 1.0 for useable peaks
if i > 0 and hsv[i - 1, 0] >= 0.8 and hsv[i, 0] <= 0.2: # wrapped around?
hsv[i, 0] += 1 # unwrap
if draw:
# Only show the area around the resistor
roi = cropregion(pos, img.shape)
img = img[roi[1]:roi[3], roi[0]:roi[2]]
detect = detect[roi[1]:roi[3], roi[0]:roi[2]]
dpi = 96 # make image approximately actual size
w = img.shape[1] * 2.1
h = img.shape[0]
plt.figure(figsize=(w/dpi,h/dpi),dpi=dpi)
plt.subplot(121)
plt.title("Origineel")
plt.imshow(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))
plt.subplot(122)
plt.title("Kleuren")
plt.imshow(cv2.cvtColor(detect, cv2.COLOR_BGR2RGB))
plt.show()
return (bgr, hsv)
# Smoothing filter by moving average convolution
# data: (1D Numpy array) with at least 'window' values
# window: (int >= 1) kernel size
# returns: new array with length reduced by window-1
def movingaverage(data, window=11):
return np.convolve(data, np.ones(window), 'valid') / window
# Get index & value of peaks/troughs
# bgr: (1D Numpy array) with average BGR values of sample lines
# hsv: (1D Numpy array) with average HSV values of sample lines
# window: (int >= 1) kernel size for smoothing
# returns: (list)
# 1D Numpy array with indices of peaks
# 2D Numpy array with [b,g,r,h,s,v] values
def detectcolours(bgr, hsv, window=11):
# Detect the base hue of the resistor
hist = np.histogram(hsv[:, 0], bins=48, range=(0, 1.2)) # histogram of hue in 0.25 steps
imax = np.argmax(hist[0]) # bin number with the highest count
basehuemin = hist[1][imax] # lower bound
basehuemax = hist[1][imax + 1] # higher bound
# Filter the data
smooth = movingaverage(hsv[:, 0], window) # has window-1 fewer values than data!
# Indices of peaks
ip = argrelextrema(smooth, np.greater)[0] # indices of hue peaks in smoothed data
ip += window // 2 # correction for smoothing size decrease
ip = ip[hsv[ip, 0] > basehuemax] # select only peaks above base hue
# Indices of troughs
it = argrelextrema(smooth, np.less)[0] # indices of hue troughs in smoothed data
it += window // 2 # correction for smoothing size decrease
it = it[hsv[it, 0] < basehuemin] # select only troughs below base hue
ix = np.sort(np.concatenate((ip, it))) # sorted indices of peaks & troughs
val = np.concatenate((bgr[ix], hsv[ix]), axis=1) # array of [b,g,r,h,s,v] arrays
return (ix, val.astype(np.float32)) # float32 for KNN
In [4]:
"""
Helper functions for visualisation
@author ewoud.dronkert@student.hu.nl
"""
def showhisto(bgr, hsv):
plt.figure(figsize=(16, 8))
plt.subplot(231)
plt.title("Rood")
plt.hist(bgr[:, 2], bins=10, range=(0.0, 1.0))
plt.subplot(232)
plt.title("Groen")
plt.hist(bgr[:, 1], bins=10, range=(0.0, 1.0))
plt.subplot(233)
plt.title("Blauw")
plt.hist(bgr[:, 0], bins=10, range=(0.0, 1.0))
# Get resistor background hue range from maximum value here!
plt.subplot(234)
plt.title("Tint")
plt.hist(hsv[:,0], bins=24, range=(0.0, 1.2), color='r')
plt.subplot(235)
plt.title("Verzadiging")
plt.hist(hsv[:,1], bins=12, range=(0.0, 1.2))
plt.subplot(236)
plt.title("Intensiteit")
plt.hist(hsv[:,2], bins=12, range=(0.0, 1.2))
plt.tight_layout()
plt.show()
def plotsamples(bgr, hsv, peaks=None, window=11, save=False):
x = np.arange(len(bgr) - window + 1) + window // 2
plt.figure(figsize=(16, 8))
plt.subplot(121)
plt.title('RGB-waardes')
plt.xlabel('Monsterlijn')
plt.ylabel('Gemiddelde RGB-waarde')
if peaks is not None:
plt.vlines(peaks, 0.0, 1.0, colors='0.33', linestyles='--')
plt.plot(x, movingaverage(bgr[:, 2], window), color='red')
plt.plot(x, movingaverage(bgr[:, 1], window), color='green')
plt.plot(x, movingaverage(bgr[:, 0], window), color='blue')
plt.legend(('Rood', 'Groen', 'Blauw'))
plt.subplot(122)
plt.title('HSV-waardes')
plt.xlabel('Monsterlijn')
plt.ylabel('Gemiddelde HSV-waarde')
if peaks is not None:
plt.vlines(peaks, 0.0, 1.0, colors='0.33', linestyles='--')
plt.plot(x, movingaverage(hsv[:, 0], window), linewidth=3)
plt.plot(x, movingaverage(hsv[:, 1], window), linestyle=':')
plt.plot(x, movingaverage(hsv[:, 2], window), linestyle=':')
plt.legend(('Tint', 'Verzadiging', 'Intensiteit'))
plt.tight_layout()
if save:
plt.savefig('bgrhsv.png', dpi=72)
plt.show()
In [5]:
filename = '20200923200846_51031.jpg'
filename = '20200923200325_47031.jpg'
filename = '20200923194706_682g.jpg'
filename = '20200923201135_3942.jpg'
filename = '20200923195413_56011.jpg'
filename = '20200923171335_10031.jpg'
filename = '20200923193804_6842.jpg'
filename = '20200923200626_475g.jpg'
# Detect the position of the resistor
pos = detectposition(filename, draw=True)
# Sample the BGR and HSV values on the resistor
bgr, hsv = samplecolours(filename, pos, draw=True)
# Get indices and r,g,b,h,s,v values of the colour bands
ix, traindata = detectcolours(bgr, hsv, window=15)
# Plot graphs for validation
showhisto(bgr, hsv) # histograms of BGR and HSV values
plotsamples(bgr, hsv, peaks=ix, window=15, save=True) # line graphs of BGR and HSV values
# Colour numbers determined from filename
# last digit is tolerance
code = filename.split('_')[1].split('.')[0]
colournums = np.array([int(d) if d.isdigit() else tolerancevals[d] for d in code], dtype=int)
# Tolerance should be further apart and come last
if len(ix) >= 3 and ix[1] - ix[0] > ix[-1] - ix[-2]: # first peaks are further apart?
colournums = colournums[::-1] # reverse the colour numbers
# Validate these detection numbers with the graphs
print("Piekindex = %s" % ix)
print("Kleurnummer = %s" % colournums)
print("Kleurnaam = %s" % [colournames[i] if i >= 0 else multipliernames[i] for i in colournums])
In [6]:
# Response shape is (len(colournums),1) and type is float32 for use in KNN
responses = np.array([colournums], dtype=np.float32).T
knndata = np.concatenate((responses, traindata), axis=1)
# Save KNN data for later!
# Every line in text file = [num,b,g,r,h,s,v]
np.savetxt(filename.split(".")[0] + ".knn.txt", knndata)
print("Kleurnummer = %s" % responses.reshape(-1).astype(int))
print('BGR+HSV data =')
print(traindata)
In [7]:
# Read back all values from separate files and save to one file
knndata = np.zeros((1, 7), dtype=np.float32) # base for concatenating, len([num,b,g,r,h,s,v]) = 7
for f in glob("*.knn.txt"): # read every file ending in .knn.txt
knndata = np.concatenate((knndata, np.loadtxt(f, dtype=np.float32))) # add new data to the end
np.savetxt("knndata.txt", knndata)
# Split responses from training data, skip first row with zeros
knnrespo = np.array([knndata[1:, 0]], dtype=np.float32).T # transpose for use in KNN
knntrain = np.array(knndata[1:, 1:], dtype=np.float32) # array of [b,g,r,h,s,v] arrays
# Histogram of all response values (= different colour numbers) in the KNN model
hmin = min(dict.keys(colourrgb))
hmax = max(dict.keys(colourrgb)) + 1
plt.title('Aantal kleurnummers in KNN-data')
plt.xlabel('Kleurnummer')
plt.ylabel('Aantal')
plt.bar(range(hmin, hmax),
np.histogram(knnrespo.reshape(-1).astype(int),
bins=hmax - hmin,
range=(hmin, hmax))[0],
color=list(dict.values(colourrgb)),
edgecolor='k')
plt.xlim(hmin - 0.5, hmax - 0.5)
plt.show()
print("Kleurnummer = %s" % knnrespo.reshape(-1).astype(int))
print('BGR+HSV data =')
print(knntrain)
In [8]:
# Train the KNN model
knn = cv2.ml.KNearest_create()
knn.train(knntrain, cv2.ml.ROW_SAMPLE, knnrespo)
# Test with another sample
test = ix[0] + 20
newcomer = np.expand_dims(np.concatenate((bgr[test], hsv[test])), axis=0).astype(np.float32)
_,result,_,_ = knn.findNearest(newcomer, 1)
colournum = int(result[0, 0])
# Print result
print("BGR+HSV = %s" % newcomer.reshape(-1))
print("Kleurnummer = %s" % colournum)
print("Kleurnaam = %s" % colournames[colournum])
In [ ]: