| OpenCV 4.3.0 Open Source Computer Vision

Super-resolution benchmarking

## Benchmarking

The super-resolution module contains sample codes for benchmarking, in order to compare different models and algorithms. Here is presented a sample code for performing benchmarking, and then a few benchmarking results are collected. It was performed on an Intel i7-9700K CPU on an Ubuntu 18.04.02 OS.

## Source Code of the sample

1 // This file is part of OpenCV project.
2 // It is subject to the license terms in the LICENSE file found in the top-level directory
3 // of this distribution and at http://opencv.org/license.html.
4
5 #include <iostream>
6 #include <opencv2/opencv_modules.hpp>
7
8 #ifdef HAVE_OPENCV_QUALITY
9 #include <opencv2/dnn_superres.hpp>
10 #include <opencv2/quality.hpp>
11 #include <opencv2/imgproc.hpp>
12 #include <opencv2/highgui.hpp>
13
14 using namespace std;
15 using namespace cv;
16 using namespace dnn_superres;
17
18 static void showBenchmark(vector<Mat> images, string title, Size imageSize,
19 const vector<String> imageTitles,
20 const vector<double> psnrValues,
21 const vector<double> ssimValues)
22 {
23 int fontFace = FONT_HERSHEY_COMPLEX_SMALL;
24 int fontScale = 1;
25 Scalar fontColor = Scalar(255, 255, 255);
26
27 int len = static_cast<int>(images.size());
28
29 int cols = 2, rows = 2;
30
31 Mat fullImage = Mat::zeros(Size((cols * 10) + imageSize.width * cols, (rows * 10) + imageSize.height * rows),
32 images[0].type());
33
34 stringstream ss;
35 int h_ = -1;
36 for (int i = 0; i < len; i++) {
37
38 int fontStart = 15;
39 int w_ = i % cols;
40 if (i % cols == 0)
41 h_++;
42
43 Rect ROI((w_ * (10 + imageSize.width)), (h_ * (10 + imageSize.height)), imageSize.width, imageSize.height);
44 Mat tmp;
45 resize(images[i], tmp, Size(ROI.width, ROI.height));
46
47 ss << imageTitles[i];
48 putText(tmp,
49 ss.str(),
50 Point(5, fontStart),
51 fontFace,
52 fontScale,
53 fontColor,
54 1,
55 16);
56
57 ss.str("");
58 fontStart += 20;
59
60 ss << "PSNR: " << psnrValues[i];
61 putText(tmp,
62 ss.str(),
63 Point(5, fontStart),
64 fontFace,
65 fontScale,
66 fontColor,
67 1,
68 16);
69
70 ss.str("");
71 fontStart += 20;
72
73 ss << "SSIM: " << ssimValues[i];
74 putText(tmp,
75 ss.str(),
76 Point(5, fontStart),
77 fontFace,
78 fontScale,
79 fontColor,
80 1,
81 16);
82
83 ss.str("");
84 fontStart += 20;
85
86 tmp.copyTo(fullImage(ROI));
87 }
88
89 namedWindow(title, 1);
90 imshow(title, fullImage);
91 waitKey();
92 }
93
94 static Vec2d getQualityValues(Mat orig, Mat upsampled)
95 {
96 double psnr = PSNR(upsampled, orig);
97 Scalar q = quality::QualitySSIM::compute(upsampled, orig, noArray());
98 double ssim = mean(Vec3d((q[0]), q[1], q[2]))[0];
99 return Vec2d(psnr, ssim);
100 }
101
102 int main(int argc, char *argv[])
103 {
104 // Check for valid command line arguments, print usage
105 // if insufficient arguments were given.
106 if (argc < 4) {
107 cout << "usage: Arg 1: image path | Path to image" << endl;
108 cout << "\t Arg 2: algorithm | edsr, espcn, fsrcnn or lapsrn" << endl;
109 cout << "\t Arg 3: path to model file 2 \n";
110 cout << "\t Arg 4: scale | 2, 3, 4 or 8 \n";
111 return -1;
112 }
113
114 string path = string(argv[1]);
115 string algorithm = string(argv[2]);
116 string model = string(argv[3]);
117 int scale = atoi(argv[4]);
118
119 Mat img = imread(path);
120 if (img.empty()) {
121 cerr << "Couldn't load image: " << img << "\n";
122 return -2;
123 }
124
125 //Crop the image so the images will be aligned
126 int width = img.cols - (img.cols % scale);
127 int height = img.rows - (img.rows % scale);
128 Mat cropped = img(Rect(0, 0, width, height));
129
130 //Downscale the image for benchmarking
131 Mat img_downscaled;
132 resize(cropped, img_downscaled, Size(), 1.0 / scale, 1.0 / scale);
133
134 //Make dnn super resolution instance
135 DnnSuperResImpl sr;
136
137 vector <Mat> allImages;
138 Mat img_new;
139
140 //Read and set the dnn model
141 sr.readModel(model);
142 sr.setModel(algorithm, scale);
143 sr.upsample(img_downscaled, img_new);
144
145 vector<double> psnrValues = vector<double>();
146 vector<double> ssimValues = vector<double>();
147
148 //DL MODEL
149 Vec2f quality = getQualityValues(cropped, img_new);
150
151 psnrValues.push_back(quality[0]);
152 ssimValues.push_back(quality[1]);
153
154 cout << sr.getAlgorithm() << ":" << endl;
155 cout << "PSNR: " << quality[0] << " SSIM: " << quality[1] << endl;
156 cout << "----------------------" << endl;
157
158 //BICUBIC
159 Mat bicubic;
160 resize(img_downscaled, bicubic, Size(), scale, scale, INTER_CUBIC);
161 quality = getQualityValues(cropped, bicubic);
162
163 psnrValues.push_back(quality[0]);
164 ssimValues.push_back(quality[1]);
165
166 cout << "Bicubic " << endl;
167 cout << "PSNR: " << quality[0] << " SSIM: " << quality[1] << endl;
168 cout << "----------------------" << endl;
169
170 //NEAREST NEIGHBOR
171 Mat nearest;
172 resize(img_downscaled, nearest, Size(), scale, scale, INTER_NEAREST);
173 quality = getQualityValues(cropped, nearest);
174
175 psnrValues.push_back(quality[0]);
176 ssimValues.push_back(quality[1]);
177
178 cout << "Nearest neighbor" << endl;
179 cout << "PSNR: " << quality[0] << " SSIM: " << quality[1] << endl;
180 cout << "----------------------" << endl;
181
182 //LANCZOS
183 Mat lanczos;
184 resize(img_downscaled, lanczos, Size(), scale, scale, INTER_LANCZOS4);
185 quality = getQualityValues(cropped, lanczos);
186
187 psnrValues.push_back(quality[0]);
188 ssimValues.push_back(quality[1]);
189
190 cout << "Lanczos" << endl;
191 cout << "PSNR: " << quality[0] << " SSIM: " << quality[1] << endl;
192 cout << "-----------------------------------------------" << endl;
193
194 vector <Mat> imgs{img_new, bicubic, nearest, lanczos};
195 vector <String> titles{sr.getAlgorithm(), "Bicubic", "Nearest neighbor", "Lanczos"};
196 showBenchmark(imgs, "Quality benchmark", Size(bicubic.cols, bicubic.rows), titles, psnrValues, ssimValues);
197
198 waitKey(0);
199
200 return 0;
201 }
202 #else
203 int main()
204 {
205 std::cout << "This sample requires the OpenCV Quality module." << std::endl;
206 return 0;
207 }
208 #endif
cv::Scalar
Scalar_< double > Scalar
Definition: types.hpp:669
cv::mean
Scalar mean(InputArray src, InputArray mask=noArray())
Calculates an average (mean) of array elements.
cv::INTER_CUBIC
Definition: imgproc.hpp:250
cv::Mat::rows
int rows
the number of rows and columns or (-1, -1) when the matrix has more than 2 dimensions ...
Definition: mat.hpp:2086
cv::Mat::copyTo
void copyTo(OutputArray m) const
Copies the matrix to another one.
cv::imread
Mat imread(const String &filename, int flags=IMREAD_COLOR)
Loads an image from a file.
cv::Size_::width
_Tp width
the width
Definition: types.hpp:339
std
STL namespace.
highgui.hpp
cv::imshow
void imshow(const String &winname, InputArray mat)
Displays an image in the specified window.
cv
"black box" representation of the file storage associated with a file on disk.
Definition: affine.hpp:51
cv::FONT_HERSHEY_COMPLEX_SMALL
smaller version of FONT_HERSHEY_COMPLEX
Definition: imgproc.hpp:820
cv::PSNR
double PSNR(InputArray src1, InputArray src2, double R=255.)
Computes the Peak Signal-to-Noise Ratio (PSNR) image quality metric.
cv::Size_
Template class for specifying the size of an image or rectangle.
Definition: types.hpp:315
cv::quality::quality_utils::scale
void scale(cv::Mat &mat, const cv::Mat &range, const T min, const T max)
Definition: quality_utils.hpp:90
cv::Vec< double, 2 >
cv::noArray
InputOutputArray noArray()
cv::Mat::cols
int cols
Definition: mat.hpp:2086
cv::Rect
Rect2i Rect
Definition: types.hpp:462
cv::Rect_
Template class for 2D rectangles.
Definition: types.hpp:420
cv::INTER_LANCZOS4
Definition: imgproc.hpp:256
quality.hpp
cv::namedWindow
void namedWindow(const String &winname, int flags=WINDOW_AUTOSIZE)
Creates a window.
cv::Size_::height
_Tp height
the height
Definition: types.hpp:340
cv::Size
Size2i Size
Definition: types.hpp:347
dnn_superres.hpp
cv::Vec2d
Vec< double, 2 > Vec2d
Definition: matx.hpp:423
cv::hal::resize
void resize(int src_type, const uchar *src_data, size_t src_step, int src_width, int src_height, uchar *dst_data, size_t dst_step, int dst_width, int dst_height, double inv_scale_x, double inv_scale_y, int interpolation)
cv::putText
void putText(InputOutputArray img, const String &text, Point org, int fontFace, double fontScale, Scalar color, int thickness=1, int lineType=LINE_8, bool bottomLeftOrigin=false)
Draws a text string.
cv::INTER_NEAREST
Definition: imgproc.hpp:246
imgproc.hpp
cv::Vec3d
Vec< double, 3 > Vec3d
Definition: matx.hpp:424
cv::Scalar_< double >
cv::Mat
n-dimensional dense array class
Definition: mat.hpp:791
cv::Mat::empty
bool empty() const
Returns true if the array has no elements.
cv::Point
Point2i Point
Definition: types.hpp:194
cv::waitKey
int waitKey(int delay=0)
Waits for a pressed key.

## Explanation

1. Read and downscale the image

   int width = img.cols - (img.cols % scale);
   int height = img.rows - (img.rows % scale);
   Mat cropped = img(Rect(0, 0, width, height));
   Mat img_downscaled;
   cv::resize(cropped, img_downscaled, cv::Size(), 1.0 / scale, 1.0 / scale);

   Resize the image by the scaling factor. Before that a cropping is necessary, so the images will align.

2. Set the model

   DnnSuperResImpl sr;
   sr.readModel(path);
   sr.setModel(algorithm, scale);
   sr.upsample(img_downscaled, img_new);

   Instantiate a dnn super-resolution object. Read and set the algorithm and scaling factor.

3. Perform benchmarking

   double psnr = PSNR(img_new, cropped);
   Scalar q = cv::quality::QualitySSIM::compute(img_new, cropped, cv::noArray());
   double ssim = mean(cv::Vec3f(q[0], q[1], q[2]))[0];

   Calculate PSNR and SSIM. Use OpenCVs PSNR (core opencv) and SSIM (contrib) functions to compare the images. Repeat it with other upscaling algorithms, such as other DL models or interpolation methods (eg. bicubic, nearest neighbor).

## Benchmarking results

## Dataset benchmarking

### General100 dataset

##### 2x scaling factor

| Avg inference time in sec (CPU) | Avg PSNR | Avg SSIM
--- | --- | --- | ---
ESPCN | 0.008795 | 32.7059 | 0.9276
EDSR | 5.923450 | 34.1300 | 0.9447
FSRCNN | 0.021741 | 32.8886 | 0.9301
LapSRN | 0.114812 | 32.2681 | 0.9248
Bicubic | 0.000208 | 32.1638 | 0.9305
Nearest neighbor | 0.000114 | 29.1665 | 0.9049
Lanczos | 0.001094 | 32.4687 | 0.9327

##### 3x scaling factor

| Avg inference time in sec (CPU) | Avg PSNR | Avg SSIM
--- | --- | --- | ---
ESPCN | 0.005495 | 28.4229 | 0.8474
EDSR | 2.455510 | 29.9828 | 0.8801
FSRCNN | 0.008807 | 28.3068 | 0.8429
LapSRN | 0.282575 | 26.7330 | 0.8862
Bicubic | 0.000311 | 26.0635 | 0.8754
Nearest neighbor | 0.000148 | 23.5628 | 0.8174
Lanczos | 0.001012 | 25.9115 | 0.8706

##### 4x scaling factor

| Avg inference time in sec (CPU) | Avg PSNR | Avg SSIM
--- | --- | --- | ---
ESPCN | 0.004311 | 26.6870 | 0.7891
EDSR | 1.607570 | 28.1552 | 0.8317
FSRCNN | 0.005302 | 26.6088 | 0.7863
LapSRN | 0.121229 | 26.7383 | 0.7896
Bicubic | 0.000311 | 26.0635 | 0.8754
Nearest neighbor | 0.000148 | 23.5628 | 0.8174
Lanczos | 0.001012 | 25.9115 | 0.8706

## Images

#### 2x scaling factor

Set5: butterfly.png | size: 256x256 |  |
--- | --- | --- | ---
Original | Bicubic interpolation | Nearest neighbor interpolation | Lanczos interpolation
PSRN / SSIM / Speed (CPU) | 26.6645 / 0.9048 / 0.000201 | 23.6854 / 0.8698 / 0.000075 | 26.9476 / 0.9075 / 0.001039
ESPCN | FSRCNN | LapSRN | EDSR
29.0341 / 0.9354 / 0.004157 | 29.0077 / 0.9345 / 0.006325 | 27.8212 / 0.9230 / 0.037937 | 30.0347 / 0.9453 / 2.077280

#### 3x scaling factor

Urban100: img_001.png | size: 1024x644 |  |
--- | --- | --- | ---
Original | Bicubic interpolation | Nearest neighbor interpolation | Lanczos interpolation
PSRN / SSIM / Speed (CPU) | 27.0474 / 0.8484 / 0.000391 | 26.0842 / 0.8353 / 0.000236 | 27.0704 / 0.8483 / 0.002234
ESPCN | FSRCNN | LapSRN is not trained for 3x because of its architecture | EDSR
28.0118 / 0.8588 / 0.030748 | 28.0184 / 0.8597 / 0.094173 |  | 30.5671 / 0.9019 / 9.517580

#### 4x scaling factor

Set14: comic.png | size: 250x361 |  |
--- | --- | --- | ---
Original | Bicubic interpolation | Nearest neighbor interpolation | Lanczos interpolation
PSRN / SSIM / Speed (CPU) | 19.6766 / 0.6413 / 0.000262 | 18.5106 / 0.5879 / 0.000085 | 19.4948 / 0.6317 / 0.001098
ESPCN | FSRCNN | LapSRN | EDSR
20.0417 / 0.6302 / 0.001894 | 20.0885 / 0.6384 / 0.002103 | 20.0676 / 0.6339 / 0.061640 | 20.5233 / 0.6901 / 0.665876

#### 8x scaling factor

Div2K: 0006.png | size: 1356x2040 |
--- | --- | ---
Original | Bicubic interpolation | Nearest neighbor interpolation
PSRN / SSIM / Speed (CPU) | 26.3139 / 0.8033 / 0.001107 | 23.8291 / 0.7340 / 0.000611
Lanczos interpolation | LapSRN |
26.1565 / 0.7962 / 0.004782 | 26.7046 / 0.7987 / 2.274290 |