src/my-place/place/PlacesDetector.cpp

   1 /*
   2  * Copyright (c) 2016 Samsung Electronics Co., Ltd.
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  * http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16
  17 #include <sstream>
  18 #include <Types.h>
  19 #include <Json.h>
  20 #include <Similarity.h>
  21 #include "PlacesDetector.h"
  22 #include "PlaceCateger.h"
  23 #include <Median.h>
  24 #ifdef DEBUG_MODE
  25 #include <Gmap.h>
  26 #endif /* DEBUG_MODE */
  27 #include <UserPlacesTypes.h>
  28 #include <fstream>
  29 #include <algorithm>
  30 #include <UserPlacesParams.h>
  31 #include <DebugUtils.h>
  32 #include <DatabaseManager.h>
  33
  34 #define __DELETE_PLACES_QUERY "DELETE FROM " PLACE_TABLE
  35
  36 #ifdef DEBUG_MODE
  37 #define __USER_PLACES_FILE "/tmp/user_places.txt" // TODO: Only for debug purposes -> Remove in final solution
  38 #endif /* DEBUG_MODE */
  39
  40 #define __GET_VISITS_QUERY "SELECT "\
  41         VISIT_COLUMN_END_TIME ", "\
  42         VISIT_COLUMN_START_TIME ", "\
  43         VISIT_COLUMN_WIFI_APS ", "\
  44         VISIT_COLUMN_LOCATION_VALID ", "\
  45         VISIT_COLUMN_LOCATION_LATITUDE ", "\
  46         VISIT_COLUMN_LOCATION_LONGITUDE ", "\
  47         VISIT_COLUMN_LOCATION_ACCURACY ", "\
  48         VISIT_COLUMN_CATEG_HOME ", "\
  49         VISIT_COLUMN_CATEG_WORK ", "\
  50         VISIT_COLUMN_CATEG_OTHER \
  51         " FROM " VISIT_TABLE
  52
  53 #define __PLACE_TABLE_COLUMNS \
  54         PLACE_COLUMN_CATEG_ID " INTEGER, "\
  55         PLACE_COLUMN_CATEG_CONFIDENCE " REAL, "\
  56         PLACE_COLUMN_NAME " TEXT, "\
  57         PLACE_COLUMN_LOCATION_VALID " INTEGER, "\
  58         PLACE_COLUMN_LOCATION_LATITUDE " REAL, "\
  59         PLACE_COLUMN_LOCATION_LONGITUDE " REAL, "\
  60         PLACE_COLUMN_LOCATION_ACCURACY " REAL, "\
  61         PLACE_COLUMN_WIFI_APS " STRING, "\
  62         PLACE_COLUMN_CREATE_DATE " timestamp"
  63
  64 void ctx::PlacesDetector::detectPlaces()
  65 {
  66         _D("");
  67         __dbCreateTable();
  68         __dbDeletePlaces();
  69         __dbDeleteOldEntries();
  70         std::vector<Json> records = __dbGetVisits();
  71         Visits visits = __visitsFromJsons(records);
  72         __processVisits(visits);
  73 }
  74
  75 std::vector<ctx::Json> ctx::PlacesDetector::__dbGetVisits()
  76 {
  77         std::vector<Json> records;
  78         DatabaseManager dbManager;
  79         bool ret = dbManager.executeSync(__GET_VISITS_QUERY, &records);
  80         _D("load visits execute query result: %s", ret ? "SUCCESS" : "FAIL");
  81         return records;
  82 }
  83
  84 double ctx::PlacesDetector::__doubleValueFromJson(Json &row, const char* key)
  85 {
  86         double value;
  87         row.get(NULL, key, &value);
  88         _D("key: %s, value: %lf", key, value);
  89         return value;
  90 }
  91
  92 ctx::Categs ctx::PlacesDetector::__visitCategsFromJson(Json &row)
  93 {
  94         Categs categs;
  95         categs[PLACE_CATEG_ID_HOME] = __doubleValueFromJson(row, VISIT_COLUMN_CATEG_HOME);
  96         categs[PLACE_CATEG_ID_WORK] = __doubleValueFromJson(row, VISIT_COLUMN_CATEG_WORK);
  97         categs[PLACE_CATEG_ID_OTHER] = __doubleValueFromJson(row, VISIT_COLUMN_CATEG_OTHER);
  98         return categs;
  99 }
 100
 101 void ctx::PlacesDetector::__visitLocationFromJson(Json &row, ctx::Visit &visit)
 102 {
 103         int locationValidInt;
 104         row.get(NULL, VISIT_COLUMN_LOCATION_VALID, &locationValidInt);
 105         visit.locationValid = (bool) locationValidInt;
 106         row.get(NULL, VISIT_COLUMN_LOCATION_LATITUDE, &(visit.location.latitude));
 107         row.get(NULL, VISIT_COLUMN_LOCATION_LONGITUDE, &(visit.location.longitude));
 108         row.get(NULL, VISIT_COLUMN_LOCATION_ACCURACY, &(visit.location.accuracy));
 109 }
 110
 111 ctx::Visit ctx::PlacesDetector::__visitFromJson(Json &row)
 112 {
 113         int startTime;
 114         int endTime;
 115         std::string str;
 116         row.get(NULL, VISIT_COLUMN_START_TIME, &startTime);
 117         row.get(NULL, VISIT_COLUMN_END_TIME, &endTime);
 118         row.get(NULL, VISIT_COLUMN_WIFI_APS, &str);
 119
 120         std::stringstream ss;
 121         ss << str;
 122         std::shared_ptr<MacSet> macSet = std::make_shared<MacSet>();
 123         ss >> *macSet;
 124
 125         Interval interval(startTime, endTime);
 126         Categs categs = __visitCategsFromJson(row);
 127
 128         Visit visit(interval, macSet, categs);
 129
 130         __visitLocationFromJson(row, visit);
 131
 132         return visit;
 133 }
 134
 135 ctx::Visits ctx::PlacesDetector::__visitsFromJsons(std::vector<Json>& records)
 136 {
 137         Visits visits;
 138         _D("db_result: number of all visits: %d", records.size());
 139
 140         for (Json &row : records) {
 141                 Visit visit = __visitFromJson(row);
 142                 visits.push_back(visit);
 143         }
 144         _D("number of all visits in vector: %d", visits.size());
 145         return visits;
 146 }
 147
 148
 149 void ctx::PlacesDetector::__reduceOutliers(ctx::Visits &visits)
 150 {
 151         int size = visits.size();
 152         visits.erase(std::remove_if(
 153                                         visits.begin(),
 154                                         visits.end(),
 155                                         [](Visit v)->bool {
 156                                                 int minutes = (v.interval.end - v.interval.start) / 60;
 157                                                 return (minutes < PLACES_DETECTOR_MIN_VISIT_DURATION_MINUTES)
 158                                                                 || (minutes > PLACES_DETECTOR_MAX_VISIT_DURATION_MINUTES);
 159                                         }),
 160                                 visits.end());
 161         int newSize = visits.size();
 162         if (size != newSize)
 163                 _D("Visits number from %d to %d (to short and to long reduction)", size, newSize);
 164 }
 165
 166 std::vector<std::shared_ptr<ctx::Place>> ctx::PlacesDetector::__processVisits(ctx::Visits &visits, bool testMode)
 167 {
 168 #ifdef DEBUG_MODE
 169         std::vector<Visits> placesVisits; // TODO: remove from final solution.
 170 #endif /* DEBUG_MODE */
 171
 172         std::vector<std::shared_ptr<Place>> newDetectedPlaces;
 173
 174         __reduceOutliers(visits);
 175         auto components = __mergeVisits(visits);
 176
 177         for (std::shared_ptr<graph::Component> component : *components) {
 178                 // Small places outliers reduction
 179                 if (!testMode && component->size() < PLACES_DETECTOR_MIN_VISITS_PER_BIG_PLACE)
 180                         continue;
 181                 std::shared_ptr<Visits> merged = std::make_shared<Visits>();
 182                 for (graph::Node i : *component) {
 183                         merged->push_back(visits[i]);
 184                 }
 185                 std::shared_ptr<Place> place = __placeFromMergedVisits(*merged);
 186                 if (place->categId == PLACE_CATEG_ID_NONE)
 187                         continue;
 188                 newDetectedPlaces.push_back(place);
 189                 if (!testMode)
 190                         __dbInsertPlace(*place);
 191
 192 #ifdef DEBUG_MODE
 193                 { // TODO: Only for debug -> remove in final solution
 194                         Visits placeVisits;
 195                         for (graph::Node i : *component) {
 196                                 placeVisits.push_back(visits[i]);
 197                         }
 198                         placesVisits.push_back(placeVisits);
 199                 }
 200 #endif /* DEBUG_MODE */
 201         }
 202
 203 #ifdef DEBUG_MODE
 204         { // Print to file TODO: Only for debug -> remove in final solution
 205                 std::ofstream out(__USER_PLACES_FILE);
 206                 for (size_t i = 0; i < newDetectedPlaces.size(); i++) {
 207                         DebugUtils::printPlace2Stream(*newDetectedPlaces[i], out);
 208                         Visits placeVisits = placesVisits[i];
 209                         for (Visit visit : placeVisits) {
 210                                 visit.printShort2Stream(out);
 211                         }
 212                 }
 213                 out.close();
 214                 Gmap::writeMap(newDetectedPlaces);
 215         }
 216 #endif /* DEBUG_MODE */
 217
 218 return newDetectedPlaces;
 219 }
 220
 221 void ctx::PlacesDetector::__mergeLocation(const Visits &visits, Place &place)
 222 {
 223         std::vector<double> latitudes;
 224         std::vector<double> longitudes;
 225         std::vector<double> accuracy;
 226         place.locationValid = false;
 227         for (const Visit& visit : visits) {
 228                 if (visit.locationValid) {
 229                         latitudes.push_back(visit.location.latitude);
 230                         longitudes.push_back(visit.location.longitude);
 231                         accuracy.push_back(visit.location.accuracy);
 232                         place.locationValid = true;
 233                 }
 234         }
 235         if (place.locationValid) {
 236                 place.location = medianLocation(latitudes, longitudes, accuracy);
 237                 _D("place location set: lat=%.8f, lon=%.8f, acc=%.8f",
 238                                 place.location.latitude,
 239                                 place.location.longitude,
 240                                 place.location.accuracy);
 241         } else {
 242                 _D("place location not set");
 243         }
 244 }
 245
 246 std::shared_ptr<ctx::Place> ctx::PlacesDetector::__placeFromMergedVisits(Visits &mergedVisits)
 247 {
 248         std::shared_ptr<Place> place = std::make_shared<Place>();
 249         place->createDate = std::time(nullptr);
 250         std::vector<std::shared_ptr<MacSet>> macSets;
 251         for (const Visit &visit : mergedVisits) {
 252                 macSets.push_back(visit.macSet);
 253         }
 254         std::shared_ptr<MacSet> allMacs = macSetsUnion(macSets);
 255         __mergeLocation(mergedVisits, *place);
 256         PlaceCateger::categorize(mergedVisits, *place);
 257
 258         /*
 259          * At this stage getting network names associated with Wifi AP MAC addresses is not needed (by looking
 260          * to Wifi APs MAC addresses to network names map in database). This additional information for place
 261          * is needed only when place is returned for external clients (getting straight from the database).
 262          */
 263         for (ctx::Mac mac : *allMacs) {
 264                 place->wifiAps.insert(std::pair<std::string, std::string>(mac, ""));
 265         }
 266
 267         return place;
 268 }
 269
 270 void ctx::PlacesDetector::__reduceOutliers(std::shared_ptr<ctx::graph::Components> &cc)
 271 {
 272         int size = cc->size();
 273         cc->erase(std::remove_if(cc->begin(),
 274                                                          cc->end(),
 275                                                          [](std::shared_ptr<graph::Component> &c)->bool {
 276                                                                  return c->size() < PLACES_DETECTOR_MIN_VISITS_PER_PLACE;
 277                                                          }),
 278                           cc->end());
 279         int newSize = cc->size();
 280         if (size != newSize)
 281                 _D("Connected components from %d to %d (min visit per place)", size, newSize);
 282 }
 283
 284 std::shared_ptr<ctx::graph::Components> ctx::PlacesDetector::__mergeVisits(const std::vector<Visit> &visits)
 285 {
 286         auto graph = __graphFromVisits(visits);
 287         auto cc = graph::connectedComponents(*graph);
 288         __reduceOutliers(cc);
 289         return cc;
 290 }
 291
 292 std::shared_ptr<ctx::graph::Graph> ctx::PlacesDetector::__graphFromVisits(const std::vector<Visit> &visits)
 293 {
 294         std::shared_ptr<graph::Graph> graph = std::make_shared<graph::Graph>();
 295         graph->resize(visits.size());
 296         for (size_t i = 0; i < visits.size(); i++) {
 297                 (*graph)[i] = std::make_shared<graph::NeighbourNodes>();
 298                 for (size_t j = 0; j < i; j++) {
 299                         if (similarity::isJoint(*visits[i].macSet, *visits[j].macSet)) {
 300                                 (*graph)[i]->insert(j);
 301                                 (*graph)[j]->insert(i);
 302                         }
 303                 }
 304         }
 305         return graph;
 306 }
 307
 308 void ctx::PlacesDetector::__dbDeletePlaces()
 309 {
 310         std::vector<Json> records;
 311         DatabaseManager dbManager;
 312         bool ret = dbManager.executeSync(__DELETE_PLACES_QUERY, &records);
 313         _D("delete places execute query result: %s", ret ? "SUCCESS" : "FAIL");
 314 }
 315
 316 void ctx::PlacesDetector::__dbDeleteOldEntries()
 317 {
 318         time_t currentTime;
 319         time(&currentTime);
 320         time_t thresholdTime = currentTime - PLACES_DETECTOR_RETENTION_SECONDS;
 321         __dbDeleteOlderVisitsThan(thresholdTime);
 322         __dbDeleteOlderWifiAPsThan(thresholdTime);
 323 }
 324
 325 void ctx::PlacesDetector::__dbDeleteOlderVisitsThan(time_t threshold)
 326 {
 327         std::stringstream query;
 328         query << "DELETE FROM " << VISIT_TABLE;
 329         query << " WHERE " << VISIT_COLUMN_END_TIME << " < " << threshold;
 330         // query << " AND 0"; // XXX: Always false condition. Uncomment it for not deleting any visit during development.
 331         std::vector<Json> records;
 332         DatabaseManager dbManager;
 333         bool ret = dbManager.executeSync(query.str().c_str(), &records);
 334         _D("deleting visits older than: %d, result: %s", threshold, ret ? "SUCCESS" : "FAIL");
 335 }
 336
 337 void ctx::PlacesDetector::__dbDeleteOlderWifiAPsThan(time_t threshold)
 338 {
 339         std::stringstream query;
 340         query << "DELETE FROM " << WIFI_APS_MAP_TABLE;
 341         query << " WHERE " << WIFI_APS_MAP_COLUMN_INSERT_TIME << " < " << threshold;
 342         std::vector<Json> records;
 343         DatabaseManager dbManager;
 344         bool ret = dbManager.executeSync(query.str().c_str(), &records);
 345         _D("deleting Wifi APs older than: %d, result: %s", threshold, ret ? "SUCCESS" : "FAIL");
 346 }
 347
 348 void ctx::PlacesDetector::__dbCreateTable()
 349 {
 350         DatabaseManager dbManager;
 351         bool ret = dbManager.createTableSync(PLACE_TABLE, __PLACE_TABLE_COLUMNS);
 352         _D("db: place Table Creation Result: %s", ret ? "SUCCESS" : "FAIL");
 353 }
 354
 355 void ctx::PlacesDetector::__dbInsertPlace(const Place &place)
 356 {
 357         Json data;
 358         data.set(NULL, PLACE_COLUMN_CATEG_ID, place.categId);
 359         data.set(NULL, PLACE_COLUMN_CATEG_CONFIDENCE, place.categConfidence);
 360         data.set(NULL, PLACE_COLUMN_NAME, place.name);
 361
 362         data.set(NULL, PLACE_COLUMN_LOCATION_VALID, place.locationValid);
 363         data.set(NULL, PLACE_COLUMN_LOCATION_LATITUDE, place.location.latitude);
 364         data.set(NULL, PLACE_COLUMN_LOCATION_LONGITUDE, place.location.longitude);
 365         data.set(NULL, PLACE_COLUMN_LOCATION_ACCURACY, place.location.accuracy);
 366         std::string wifiAps;
 367         for (std::pair<std::string, std::string> ap : place.wifiAps) {
 368                 wifiAps.append(ap.first);
 369                 wifiAps.append(",");
 370         }
 371         wifiAps = wifiAps.substr(0, wifiAps.size()-1);
 372         data.set(NULL, PLACE_COLUMN_WIFI_APS, wifiAps);
 373         data.set(NULL, PLACE_COLUMN_CREATE_DATE, static_cast<int>(place.createDate));
 374
 375         int64_t rowId;
 376         DatabaseManager dbManager;
 377         bool ret = dbManager.insertSync(PLACE_TABLE, data, &rowId);
 378         _D("insert place execute query result: %s", ret ? "SUCCESS" : "FAIL");
 379 }
 380
 381 extern "C" SO_EXPORT void detectPlaces()
 382 {
 383         ctx::PlacesDetector::detectPlaces();
 384 }