3 * Copyright (C) 2011 Robert Jobbagy <jobbagy.robert@gmail.com>
4 * Copyright (C) 2011 - 2018 Nicola Murino <nicola.murino@gmail.com>
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
22 * DEALINGS IN THE SOFTWARE.
24 * Alternatively, the contents of this file may be used under the
25 * GNU Lesser General Public License Version 2.1 (the "LGPL"), in
26 * which case the following provisions apply instead of the ones
29 * This library is free software; you can redistribute it and/or
30 * modify it under the terms of the GNU Library General Public
31 * License as published by the Free Software Foundation; either
32 * version 2 of the License, or (at your option) any later version.
34 * This library is distributed in the hope that it will be useful,
35 * but WITHOUT ANY WARRANTY; without even the implied warranty of
36 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
37 * Library General Public License for more details.
39 * You should have received a copy of the GNU Library General Public
40 * License along with this library; if not, write to the
41 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
42 * Boston, MA 02110-1301, USA.
50 #include "MotionCells.h"
51 #include <opencv2/imgproc.hpp>
53 MotionCells::MotionCells ()
56 m_motioncells_idx_count = 0;
57 m_motioncellsidxcstr = NULL;
58 m_saveInDatafile = false;
60 m_initdatafilefailed = new char[BUSMSGLEN];
61 m_savedatafilefailed = new char[BUSMSGLEN];
74 m_changed_datafile = false;
76 memset (&m_header, 0, sizeof (MotionCellHeader));
77 m_header.headersize = GINT32_TO_BE (MC_HEADER);
78 m_header.type = GINT32_TO_BE (MC_TYPE);
79 m_header.version = GINT32_TO_BE (MC_VERSION);
80 m_header.itemsize = 0;
83 m_header.starttime = 0;
86 MotionCells::~MotionCells ()
92 delete[]m_initdatafilefailed;
93 delete[]m_savedatafilefailed;
94 if (m_motioncellsidxcstr)
95 delete[]m_motioncellsidxcstr;
97 m_pcurFrame.release ();
98 m_pprevFrame.release ();
99 transparencyimg.release ();
100 m_pdifferenceImage.release ();
101 m_pbwImage.release ();
105 MotionCells::performDetectionMotionCells (cv::Mat p_frame,
106 double p_sensitivity, double p_framerate, int p_gridx, int p_gridy,
107 gint64 timestamp_millisec, bool p_isVisible, bool p_useAlpha,
108 int motionmaskcoord_count, motionmaskcoordrect * motionmaskcoords,
109 int motionmaskcells_count, motioncellidx * motionmaskcellsidx,
110 cellscolor motioncellscolor, int motioncells_count,
111 motioncellidx * motioncellsidx, gint64 starttime, char *p_datafile,
112 bool p_changed_datafile, int p_thickness)
119 p_framerate >= 1 ? p_framerate <= 5 ? sumframecnt = 1
120 : p_framerate <= 10 ? sumframecnt = 2
121 : p_framerate <= 15 ? sumframecnt = 3
122 : p_framerate <= 20 ? sumframecnt = 4
123 : p_framerate <= 25 ? sumframecnt = 5
124 : p_framerate <= 30 ? sumframecnt = 6 : sumframecnt = 0 : sumframecnt = 0;
127 m_changed_datafile = p_changed_datafile;
128 if (m_framecnt >= sumframecnt) {
129 m_useAlpha = p_useAlpha;
132 if (m_changed_datafile) {
133 ret = initDataFile (p_datafile, starttime);
138 frameSize = p_frame.size ();
139 frameSize.width /= 2;
140 frameSize.height /= 2;
141 setMotionCells (frameSize.width, frameSize.height);
142 m_sensitivity = 1 - p_sensitivity;
143 m_isVisible = p_isVisible;
144 m_pcurFrame = p_frame.clone ();
145 cv::Mat m_pcurgreyImage = cv::Mat (frameSize, CV_8UC1);
146 cv::Mat m_pprevgreyImage = cv::Mat (frameSize, CV_8UC1);
147 cv::Mat m_pgreyImage = cv::Mat (frameSize, CV_8UC1);
148 cv::Mat m_pcurDown = cv::Mat (frameSize, m_pcurFrame.type ());
149 cv::Mat m_pprevDown = cv::Mat (frameSize, m_pprevFrame.type ());
150 m_pbwImage.create (frameSize, CV_8UC1);
151 pyrDown (m_pprevFrame, m_pprevDown);
152 cvtColor (m_pprevDown, m_pprevgreyImage, cv::COLOR_RGB2GRAY);
153 pyrDown (m_pcurFrame, m_pcurDown);
154 cvtColor (m_pcurDown, m_pcurgreyImage, cv::COLOR_RGB2GRAY);
155 m_pdifferenceImage = m_pcurgreyImage.clone ();
156 //cvSmooth(m_pcurgreyImage, m_pcurgreyImage, CV_GAUSSIAN, 3, 0);//TODO camera noise reduce,something smoothing, and rethink runningavg weights
158 //Minus the current gray frame from the 8U moving average.
159 cv::absdiff (m_pprevgreyImage, m_pcurgreyImage, m_pdifferenceImage);
161 //Convert the image to black and white.
162 cv::adaptiveThreshold (m_pdifferenceImage, m_pbwImage, 255,
163 cv::ADAPTIVE_THRESH_GAUSSIAN_C, cv::THRESH_BINARY_INV, 7, 5);
165 // Dilate and erode to get object blobs
166 cv::dilate (m_pbwImage, m_pbwImage, cv::Mat (), cv::Point (-1, -1), 2);
167 cv::erode (m_pbwImage, m_pbwImage, cv::Mat (), cv::Point (-1, -1), 2);
169 //mask-out the overlay on difference image
170 if (motionmaskcoord_count > 0)
171 performMotionMaskCoords (motionmaskcoords, motionmaskcoord_count);
172 if (motionmaskcells_count > 0)
173 performMotionMask (motionmaskcellsidx, motionmaskcells_count);
174 if (getIsNonZero (m_pbwImage)) { //detect Motion
175 if (m_MotionCells.size () > 0) //it contains previous motioncells what we used when frames dropped
176 m_MotionCells.clear ();
177 (motioncells_count > 0) ?
178 calculateMotionPercentInMotionCells (motioncellsidx,
180 : calculateMotionPercentInMotionCells (motionmaskcellsidx, 0);
182 transparencyimg = cv::Mat::zeros (p_frame.size (), p_frame.type ());
183 if (m_motioncellsidxcstr)
184 delete[]m_motioncellsidxcstr;
185 m_motioncells_idx_count = m_MotionCells.size () * MSGLEN; //one motion cell idx: (lin idx : col idx,) it's up to 6 character except last motion cell idx
186 m_motioncellsidxcstr = new char[m_motioncells_idx_count];
187 char *tmpstr = new char[MSGLEN + 1];
189 for (unsigned int i = 0; i < m_MotionCells.size (); i++) {
191 pt1.x = m_MotionCells.at (i).cell_pt1.x * 2;
192 pt1.y = m_MotionCells.at (i).cell_pt1.y * 2;
193 pt2.x = m_MotionCells.at (i).cell_pt2.x * 2;
194 pt2.y = m_MotionCells.at (i).cell_pt2.y * 2;
195 if (m_useAlpha && m_isVisible) {
196 cv::rectangle (transparencyimg,
198 CV_RGB (motioncellscolor.B_channel_value,
199 motioncellscolor.G_channel_value,
200 motioncellscolor.R_channel_value), cv::FILLED);
201 } else if (m_isVisible) {
202 cv::rectangle (p_frame,
204 CV_RGB (motioncellscolor.B_channel_value,
205 motioncellscolor.G_channel_value,
206 motioncellscolor.R_channel_value), p_thickness);
209 if (i < m_MotionCells.size () - 1) {
210 snprintf (tmpstr, MSGLEN + 1, "%d:%d,", m_MotionCells.at (i).lineidx,
211 m_MotionCells.at (i).colidx);
213 snprintf (tmpstr, MSGLEN + 1, "%d:%d", m_MotionCells.at (i).lineidx,
214 m_MotionCells.at (i).colidx);
217 strncpy (m_motioncellsidxcstr, tmpstr, m_motioncells_idx_count);
219 strcat (m_motioncellsidxcstr, tmpstr);
221 if (m_MotionCells.size () == 0)
222 strncpy (m_motioncellsidxcstr, " ", m_motioncells_idx_count);
224 if (m_useAlpha && m_isVisible) {
225 if (m_MotionCells.size () > 0)
226 blendImages (p_frame, transparencyimg, m_alpha, m_beta);
231 if (mc_savefile && m_saveInDatafile) {
232 ret = saveMotionCells (timestamp_millisec);
237 m_motioncells_idx_count = 0;
238 if (m_MotionCells.size () > 0)
239 m_MotionCells.clear ();
242 m_pprevFrame = m_pcurFrame.clone ();
245 for (int i = 0; i < m_gridy; ++i) {
251 if (p_framerate <= 5) {
252 if (m_MotionCells.size () > 0)
253 m_MotionCells.clear ();
255 } else { //we do frame drop
256 m_motioncells_idx_count = 0;
258 for (unsigned int i = 0; i < m_MotionCells.size (); i++) {
260 pt1.x = m_MotionCells.at (i).cell_pt1.x * 2;
261 pt1.y = m_MotionCells.at (i).cell_pt1.y * 2;
262 pt2.x = m_MotionCells.at (i).cell_pt2.x * 2;
263 pt2.y = m_MotionCells.at (i).cell_pt2.y * 2;
264 if (m_useAlpha && m_isVisible) {
265 cv::rectangle (transparencyimg,
268 CV_RGB (motioncellscolor.B_channel_value,
269 motioncellscolor.G_channel_value,
270 motioncellscolor.R_channel_value), cv::FILLED);
271 } else if (m_isVisible) {
272 cv::rectangle (p_frame,
275 CV_RGB (motioncellscolor.B_channel_value,
276 motioncellscolor.G_channel_value,
277 motioncellscolor.R_channel_value), p_thickness);
281 if (m_useAlpha && m_isVisible) {
282 if (m_MotionCells.size () > 0)
283 blendImages (p_frame, transparencyimg, m_alpha, m_beta);
290 MotionCells::initDataFile (char *p_datafile, gint64 starttime) //p_date is increased with difference between current and previous buffer ts
293 if (strncmp (p_datafile, " ", 1)) {
294 mc_savefile = fopen (p_datafile, "w");
295 if (mc_savefile == NULL) {
296 //fprintf(stderr, "%s %d:initDataFile:fopen:%d (%s)\n", __FILE__, __LINE__, errno,
298 strncpy (m_initdatafilefailed, strerror (errno), BUSMSGLEN - 1);
299 m_initerrorcode = errno;
302 m_saveInDatafile = true;
307 //it needs these bytes
309 GINT32_TO_BE ((int) ceil (ceil (m_gridx * m_gridy / 8.0) / 4.0) * 4 +
310 sizeof (mcd.timestamp));
311 m_header.gridx = GINT32_TO_BE (m_gridx);
312 m_header.gridy = GINT32_TO_BE (m_gridy);
313 m_header.starttime = GINT64_TO_BE (starttime);
315 snprintf (m_header.name, sizeof (m_header.name), "%s %dx%d", MC_VERSIONTEXT,
316 GINT32_FROM_BE (m_header.gridx), GINT32_FROM_BE (m_header.gridy));
317 m_changed_datafile = false;
322 MotionCells::saveMotionCells (gint64 timestamp_millisec)
325 MotionCellData mc_data;
326 mc_data.timestamp = GINT32_TO_BE (timestamp_millisec);
328 //There is no datafile
329 if (mc_savefile == NULL)
332 if (ftello (mc_savefile) == 0) {
333 //cerr << "Writing out file header"<< m_header.headersize <<":" << sizeof(MotionCellHeader) << " itemsize:"
334 //<< m_header.itemsize << endl;
335 if (fwrite (&m_header, sizeof (MotionCellHeader), 1, mc_savefile) != 1) {
336 //fprintf(stderr, "%s %d:saveMotionCells:fwrite:%d (%s)\n", __FILE__, __LINE__, errno,
338 strncpy (m_savedatafilefailed, strerror (errno), BUSMSGLEN - 1);
339 m_saveerrorcode = errno;
346 GINT32_FROM_BE (m_header.itemsize) - sizeof (mc_data.timestamp));
347 if (mc_data.data == NULL) {
348 //fprintf(stderr, "%s %d:saveMotionCells:calloc:%d (%s)\n", __FILE__, __LINE__, errno,
350 strncpy (m_savedatafilefailed, strerror (errno), BUSMSGLEN - 1);
351 m_saveerrorcode = errno;
355 for (unsigned int i = 0; i < m_MotionCells.size (); i++) {
357 m_MotionCells.at (i).lineidx * GINT32_FROM_BE (m_header.gridx) +
358 m_MotionCells.at (i).colidx;
359 int bytenum = (int) floor (bitnum / 8.0);
360 int shift = bitnum - bytenum * 8;
361 mc_data.data[bytenum] = mc_data.data[bytenum] | (1 << shift);
362 //cerr << "Motion Detected " << "line:" << m_MotionCells.at(i).lineidx << " col:" << m_MotionCells.at(i).colidx;
363 //cerr << " bitnum " << bitnum << " bytenum " << bytenum << " shift " << shift << " value " << (int)mc_data.data[bytenum] << endl;
366 if (fwrite (&mc_data.timestamp, sizeof (mc_data.timestamp), 1,
368 //fprintf(stderr, "%s %d:saveMotionCells:fwrite:%d (%s)\n", __FILE__, __LINE__, errno,
370 strncpy (m_savedatafilefailed, strerror (errno), BUSMSGLEN - 1);
371 m_saveerrorcode = errno;
375 if (fwrite (mc_data.data,
376 GINT32_FROM_BE (m_header.itemsize) - sizeof (mc_data.timestamp), 1,
378 //fprintf(stderr, "%s %d:saveMotionCells:fwrite:%d (%s)\n", __FILE__, __LINE__, errno,
380 strncpy (m_savedatafilefailed, strerror (errno), BUSMSGLEN - 1);
381 m_saveerrorcode = errno;
390 MotionCells::calculateMotionPercentInCell (int p_row, int p_col,
391 double *p_cellarea, double *p_motionarea)
393 double cntpixelsnum = 0;
394 double cntmotionpixelnum = 0;
396 int ybegin = floor ((double) p_row * m_cellheight);
397 int yend = floor ((double) (p_row + 1) * m_cellheight);
398 int xbegin = floor ((double) (p_col) * m_cellwidth);
399 int xend = floor ((double) (p_col + 1) * m_cellwidth);
400 int cellw = xend - xbegin;
401 int cellh = yend - ybegin;
402 int cellarea = cellw * cellh;
403 *p_cellarea = cellarea;
404 int thresholdmotionpixelnum = floor ((double) cellarea * m_sensitivity);
406 for (int i = ybegin; i < yend; i++) {
407 for (int j = xbegin; j < xend; j++) {
409 if ((((uchar *) (m_pbwImage.data + m_pbwImage.step[0] * i))[j]) > 0) {
411 if (cntmotionpixelnum >= thresholdmotionpixelnum) { //we dont needs calculate anymore
412 *p_motionarea = cntmotionpixelnum;
413 return (cntmotionpixelnum / cntpixelsnum);
416 int remainingpixelsnum = cellarea - cntpixelsnum;
417 if ((cntmotionpixelnum + remainingpixelsnum) < thresholdmotionpixelnum) { //moving pixels number will be less than threshold
424 return (cntmotionpixelnum / cntpixelsnum);
428 MotionCells::calculateMotionPercentInMotionCells (motioncellidx *
429 p_motioncellsidx, int p_motioncells_count)
431 if (p_motioncells_count == 0) {
432 for (int i = 0; i < m_gridy; i++) {
433 for (int j = 0; j < m_gridx; j++) {
434 m_pCells[i][j].MotionPercent = calculateMotionPercentInCell (i, j,
435 &m_pCells[i][j].CellArea, &m_pCells[i][j].MotionArea);
436 m_pCells[i][j].hasMotion =
437 m_sensitivity < m_pCells[i][j].MotionPercent ? true : false;
438 if (m_pCells[i][j].hasMotion) {
442 mci.cell_pt1.x = floor ((double) j * m_cellwidth);
443 mci.cell_pt1.y = floor ((double) i * m_cellheight);
444 mci.cell_pt2.x = floor ((double) (j + 1) * m_cellwidth);
445 mci.cell_pt2.y = floor ((double) (i + 1) * m_cellheight);
446 int w = mci.cell_pt2.x - mci.cell_pt1.x;
447 int h = mci.cell_pt2.y - mci.cell_pt1.y;
448 mci.motioncell = cv::Rect (mci.cell_pt1.x, mci.cell_pt1.y, w, h);
449 m_MotionCells.push_back (mci);
454 for (int k = 0; k < p_motioncells_count; ++k) {
456 int i = p_motioncellsidx[k].lineidx;
457 int j = p_motioncellsidx[k].columnidx;
458 m_pCells[i][j].MotionPercent =
459 calculateMotionPercentInCell (i, j,
460 &m_pCells[i][j].CellArea, &m_pCells[i][j].MotionArea);
461 m_pCells[i][j].hasMotion =
462 m_pCells[i][j].MotionPercent > m_sensitivity ? true : false;
463 if (m_pCells[i][j].hasMotion) {
465 mci.lineidx = p_motioncellsidx[k].lineidx;
466 mci.colidx = p_motioncellsidx[k].columnidx;
467 mci.cell_pt1.x = floor ((double) j * m_cellwidth);
468 mci.cell_pt1.y = floor ((double) i * m_cellheight);
469 mci.cell_pt2.x = floor ((double) (j + 1) * m_cellwidth);
470 mci.cell_pt2.y = floor ((double) (i + 1) * m_cellheight);
471 int w = mci.cell_pt2.x - mci.cell_pt1.x;
472 int h = mci.cell_pt2.y - mci.cell_pt1.y;
473 mci.motioncell = cv::Rect (mci.cell_pt1.x, mci.cell_pt1.y, w, h);
474 m_MotionCells.push_back (mci);
481 MotionCells::performMotionMaskCoords (motionmaskcoordrect * p_motionmaskcoords,
482 int p_motionmaskcoords_count)
487 cv::Point lowerright;
490 for (int i = 0; i < p_motionmaskcoords_count; i++) {
491 upperleft.x = p_motionmaskcoords[i].upper_left_x;
492 upperleft.y = p_motionmaskcoords[i].upper_left_y;
493 lowerright.x = p_motionmaskcoords[i].lower_right_x;
494 lowerright.y = p_motionmaskcoords[i].lower_right_y;
495 cv::rectangle (m_pbwImage, upperleft, lowerright, CV_RGB (0, 0, 0),
501 MotionCells::performMotionMask (motioncellidx * p_motionmaskcellsidx,
502 int p_motionmaskcells_count)
504 for (int k = 0; k < p_motionmaskcells_count; k++) {
505 int beginy = p_motionmaskcellsidx[k].lineidx * m_cellheight;
506 int beginx = p_motionmaskcellsidx[k].columnidx * m_cellwidth;
508 (double) p_motionmaskcellsidx[k].columnidx * m_cellwidth + m_cellwidth;
510 (double) p_motionmaskcellsidx[k].lineidx * m_cellheight + m_cellheight;
511 for (int i = beginy; i < endy; i++)
512 for (int j = beginx; j < endx; j++) {
513 ((uchar *) (m_pbwImage.data + m_pbwImage.step[0] * i))[j] = 0;
518 ///BGR if we use only OpenCV
519 //RGB if we use gst+OpenCV
521 MotionCells::blendImages (cv::Mat p_actFrame, cv::Mat p_cellsFrame,
522 float p_alpha, float p_beta)
525 int height = p_actFrame.size ().height;
526 int width = p_actFrame.size ().width;
527 int step = p_actFrame.step[0] / sizeof (uchar);
528 int channels = p_actFrame.channels ();
529 int cellstep = p_cellsFrame.step[0] / sizeof (uchar);
530 uchar *curImageData = (uchar *) p_actFrame.data;
531 uchar *cellImageData = (uchar *) p_cellsFrame.data;
533 for (int i = 0; i < height; i++)
534 for (int j = 0; j < width; j++)
535 for (int k = 0; k < channels; k++)
536 if (cellImageData[i * cellstep + j * channels + k] > 0) {
537 curImageData[i * step + j * channels + k] =
538 round ((double) curImageData[i * step + j * channels +
539 k] * p_alpha + ((double) cellImageData[i * cellstep +
540 j * channels + k] * p_beta));