plugins/database/databasesink.cpp

   1 #include "databasesink.h"
   2 #include "abstractroutingengine.h"
   3 #include "listplusplus.h"
   4
   5 int bufferLength = 100;
   6
   7 extern "C" AbstractSinkManager * create(AbstractRoutingEngine* routingengine, map<string, string> config)
   8 {
   9         return new DatabaseSinkManager(routingengine, config);
  10 }
  11
  12 void * cbFunc(gpointer data)
  13 {
  14         Shared *shared = static_cast<Shared*>(data);
  15
  16         if(!shared)
  17         {
  18                 throw std::runtime_error("Could not cast shared object.");
  19         }
  20
  21         vector<DictionaryList<string> > insertList;
  22
  23         while(1)
  24         {
  25                 DBObject* obj = shared->queue.pop();
  26
  27                 if( obj->quit )
  28                 {
  29                         delete obj;
  30                         break;
  31                 }
  32
  33                 DictionaryList<string> dict;
  34
  35                 NameValuePair<string> one("key", obj->key);
  36                 NameValuePair<string> two("value", obj->value);
  37                 NameValuePair<string> three("source", obj->source);
  38                 NameValuePair<string> four("time", boost::lexical_cast<string>(obj->time));
  39                 NameValuePair<string> five("sequence", boost::lexical_cast<string>(obj->sequence));
  40
  41                 dict.push_back(one);
  42                 dict.push_back(two);
  43                 dict.push_back(three);
  44                 dict.push_back(four);
  45                 dict.push_back(five);
  46
  47                 insertList.push_back(dict);
  48
  49                 if(insertList.size() > bufferLength)
  50                 {
  51                         shared->db->exec("BEGIN IMMEDIATE TRANSACTION");
  52                         for(int i=0; i< insertList.size(); i++)
  53                         {
  54                                 DictionaryList<string> d = insertList[i];
  55                                 shared->db->insert(d);
  56                         }
  57                         shared->db->exec("END TRANSACTION");
  58                         insertList.clear();
  59                 }
  60                 delete obj;
  61         }
  62
  63         /// final flush of whatever is still in the queue:
  64
  65         shared->db->exec("BEGIN IMMEDIATE TRANSACTION");
  66         for(int i=0; i< insertList.size(); i++)
  67         {
  68                 DictionaryList<string> d = insertList[i];
  69                 shared->db->insert(d);
  70         }
  71         shared->db->exec("END TRANSACTION");
  72
  73         return NULL;
  74 }
  75
  76 int getNextEvent(gpointer data)
  77 {
  78         PlaybackShared* pbshared = static_cast<PlaybackShared*>(data);
  79
  80         if(!pbshared)
  81                 throw std::runtime_error("failed to cast PlaybackShared object");
  82
  83         if(pbshared->stop)
  84                 return 0;
  85
  86         auto itr = pbshared->playbackQueue.begin();
  87
  88         if(itr == pbshared->playbackQueue.end())
  89         {
  90                 return 0;
  91         }
  92
  93         DBObject* obj = *itr;
  94
  95         AbstractPropertyType* value = VehicleProperty::getPropertyTypeForPropertyNameValue(obj->key,obj->value);
  96
  97         if(value)
  98         {
  99                 pbshared->routingEngine->updateProperty(obj->key, value, pbshared->uuid);
 100                 value->timestamp = obj->time;
 101                 //value->sequence = obj->sequence;
 102         }
 103
 104         if(++itr != pbshared->playbackQueue.end())
 105         {
 106                 DBObject *o2 = *itr;
 107                 double t = o2->time - obj->time;
 108
 109                 if(t > 0)
 110                         g_timeout_add((t*1000) / pbshared->playBackMultiplier, getNextEvent, pbshared);
 111                 else
 112                         g_timeout_add(1, getNextEvent, pbshared);
 113         }
 114
 115         pbshared->playbackQueue.remove(obj);
 116         DebugOut()<<"playback Queue size: "<<pbshared->playbackQueue.size()<<endl;
 117         delete obj;
 118
 119         return 0;
 120 }
 121
 122 DatabaseSink::DatabaseSink(AbstractRoutingEngine *engine, map<std::string, std::string> config)
 123         :AbstractSource(engine,config),thread(NULL),shared(NULL),playback(false),playbackShared(NULL), playbackMultiplier(1)
 124 {
 125         databaseName = "storage";
 126         tablename = "data";
 127         tablecreate = "CREATE TABLE IF NOT EXISTS data (key TEXT, value BLOB, source TEXT, time REAL, sequence REAL)";
 128
 129         if(config.find("databaseFile") != config.end())
 130         {
 131                 setDatabaseFileName(config["databaseFile"]);
 132         }
 133
 134         if(config.find("bufferLength") != config.end())
 135         {
 136                 bufferLength = atoi(config["bufferLength"].c_str());
 137         }
 138
 139         if(config.find("properties") != config.end())
 140         {
 141                 parseConfig();
 142         }
 143
 144         for(auto itr=propertiesToSubscribeTo.begin();itr!=propertiesToSubscribeTo.end();itr++)
 145         {
 146                 engine->subscribeToProperty(*itr,this);
 147         }
 148
 149         mSupported.push_back(DatabaseFile);
 150         mSupported.push_back(DatabaseLogging);
 151         mSupported.push_back(DatabasePlayback);
 152
 153         routingEngine->setSupported(supported(), this);
 154
 155         if(config.find("startOnLoad")!= config.end())
 156         {
 157                 setLogging(true);
 158         }
 159
 160         if(config.find("playbackMultiplier")!= config.end())
 161         {
 162                 playbackMultiplier = boost::lexical_cast<uint>(config["playbackMultiplier"]);
 163         }
 164
 165         if(config.find("playbackOnLoad")!= config.end())
 166         {
 167                 setPlayback(true);
 168         }
 169
 170
 171 }
 172
 173 DatabaseSink::~DatabaseSink()
 174 {
 175         if(shared)
 176         {
 177                 DBObject* obj = new DBObject();
 178                 obj->quit = true;
 179
 180                 shared->queue.append(obj);
 181
 182                 g_thread_join(thread);
 183 //              g_thread_unref(thread);
 184                 delete shared;
 185         }
 186
 187         if(playbackShared)
 188         {
 189                 delete playbackShared;
 190         }
 191 }
 192
 193
 194 void DatabaseSink::supportedChanged(PropertyList supportedProperties)
 195 {
 196
 197 }
 198
 199 PropertyList DatabaseSink::supported()
 200 {
 201         return mSupported;
 202 }
 203
 204 PropertyInfo DatabaseSink::getPropertyInfo(VehicleProperty::Property property)
 205 {
 206         /// TODO: Compute update frequency for properties in the database
 207         return PropertyInfo::invalid();
 208 }
 209
 210 void DatabaseSink::parseConfig()
 211 {
 212         json_object *rootobject;
 213         json_tokener *tokener = json_tokener_new();
 214         enum json_tokener_error err;
 215         do
 216         {
 217                 rootobject = json_tokener_parse_ex(tokener, configuration["properties"].c_str(),configuration["properties"].size());
 218         } while ((err = json_tokener_get_error(tokener)) == json_tokener_continue);
 219         if (err != json_tokener_success)
 220         {
 221                 fprintf(stderr, "Error: %s\n", json_tokener_error_desc(err));
 222         }
 223         if (tokener->char_offset < configuration["properties"].size()) // XXX shouldn't access internal fields
 224         {
 225                 //Should handle the extra data here sometime...
 226         }
 227
 228         json_object *propobject = json_object_object_get(rootobject,"properties");
 229
 230         g_assert(json_object_get_type(propobject) == json_type_array);
 231
 232         array_list *proplist = json_object_get_array(propobject);
 233
 234         for(int i=0; i < array_list_length(proplist); i++)
 235         {
 236                 json_object *idxobj = (json_object*)array_list_get_idx(proplist,i);
 237                 std::string prop = json_object_get_string(idxobj);
 238                 propertiesToSubscribeTo.push_back(prop);
 239
 240                 DebugOut()<<"DatabaseSink logging: "<<prop<<endl;
 241         }
 242
 243         //json_object_put(propobject);
 244         json_object_put(rootobject);
 245 }
 246
 247 void DatabaseSink::stopDb()
 248 {
 249         if(!shared)
 250                 return;
 251
 252         DBObject *obj = new DBObject();
 253         obj->quit = true;
 254         shared->queue.append(obj);
 255
 256         g_thread_join(thread);
 257
 258         delete shared;
 259         shared = NULL;
 260 }
 261
 262 void DatabaseSink::startDb()
 263 {
 264         if(playback)
 265         {
 266                 DebugOut(0)<<"ERROR: tried to start logging during playback.  Only logging or playback can be used at one time"<<endl;
 267                 return;
 268         }
 269
 270         if(shared)
 271         {
 272                 DebugOut(0)<<"WARNING: logging already started.  doing nothing."<<endl;
 273                 return;
 274         }
 275
 276         initDb();
 277
 278         thread = g_thread_new("dbthread", cbFunc, shared);
 279 }
 280
 281 void DatabaseSink::startPlayback()
 282 {
 283         if(playback)
 284                 return;
 285
 286         playback = true;
 287
 288         initDb();
 289
 290         /// populate playback queue:
 291
 292         vector<vector<string> > results = shared->db->select("SELECT * FROM "+tablename);
 293
 294         /// we are done with shared.  clean up:
 295         delete shared;
 296         shared = NULL;
 297
 298         if(playbackShared)
 299         {
 300                 delete playbackShared;
 301         }
 302
 303         playbackShared = new PlaybackShared(routingEngine, uuid(), playbackMultiplier);
 304
 305         for(int i=0;i<results.size();i++)
 306         {
 307                 if(results[i].size() < 5)
 308                 {
 309                         throw std::runtime_error("column mismatch in query");
 310                 }
 311
 312                 DBObject* obj = new DBObject();
 313
 314                 obj->key = results[i][0];
 315                 obj->value = results[i][1];
 316                 obj->source = results[i][2];
 317                 obj->time = boost::lexical_cast<double>(results[i][3]);
 318
 319                 /// TODO: figure out why sequence is broken:
 320
 321 //              obj->sequence = boost::lexical_cast<int>(results[i][4]);
 322
 323                 playbackShared->playbackQueue.push_back(obj);
 324         }
 325
 326         g_timeout_add(0,getNextEvent,playbackShared);
 327 }
 328
 329 void DatabaseSink::initDb()
 330 {
 331         if(shared) delete shared;
 332
 333         shared = new Shared;
 334         shared->db->init(databaseName, tablename, tablecreate);
 335 }
 336
 337 void DatabaseSink::setPlayback(bool v)
 338 {
 339         AsyncSetPropertyRequest request;
 340         request.property = DatabasePlayback;
 341         request.value = new DatabasePlaybackType(v);
 342
 343         setProperty(request);
 344 }
 345
 346 void DatabaseSink::setLogging(bool b)
 347 {
 348         AsyncSetPropertyRequest request;
 349         request.property = DatabaseLogging;
 350         request.value = new DatabaseLoggingType(b);
 351
 352         setProperty(request);
 353
 354         delete request.value;
 355 }
 356
 357 void DatabaseSink::setDatabaseFileName(string filename)
 358 {
 359         databaseName = filename;
 360
 361         initDb();
 362
 363         vector<vector<string> > supportedStr = shared->db->select("SELECT DISTINCT key FROM "+tablename);
 364
 365         for(int i=0; i < supportedStr.size(); i++)
 366         {
 367                 if(!ListPlusPlus<VehicleProperty::Property>(&mSupported).contains(supportedStr[i][0]))
 368                         mSupported.push_back(supportedStr[i][0]);
 369         }
 370
 371         delete shared;
 372         shared = NULL;
 373
 374         routingEngine->setSupported(mSupported, this);
 375 }
 376
 377 void DatabaseSink::propertyChanged(AbstractPropertyType *value)
 378 {
 379         VehicleProperty::Property property = value->name;
 380
 381         if(!shared)
 382                 return;
 383
 384         if(!ListPlusPlus<VehicleProperty::Property>(&mSupported).contains(property))
 385         {
 386                 mSupported.push_back(property);
 387                 routingEngine->setSupported(mSupported, this);
 388         }
 389
 390         DBObject* obj = new DBObject;
 391         obj->key = property;
 392         obj->value = value->toString();
 393         obj->source = value->sourceUuid;
 394         obj->time = value->timestamp;
 395         obj->sequence = value->sequence;
 396
 397         shared->queue.append(obj);
 398 }
 399
 400
 401 const string DatabaseSink::uuid()
 402 {
 403         return "9f88156e-cb92-4472-8775-9c08addf50d3";
 404 }
 405
 406 void DatabaseSink::getPropertyAsync(AsyncPropertyReply *reply)
 407 {
 408         reply->success = false;
 409
 410         if(reply->property == DatabaseFile)
 411         {
 412                 DatabaseFileType temp(databaseName);
 413                 reply->value = &temp;
 414
 415                 reply->success = true;
 416                 reply->completed(reply);
 417
 418                 return;
 419         }
 420         else if(reply->property == DatabaseLogging)
 421         {
 422                 DatabaseLoggingType temp = shared;
 423
 424                 reply->value = &temp;
 425                 reply->success = true;
 426                 reply->completed(reply);
 427
 428                 return;
 429         }
 430
 431         else if(reply->property == DatabasePlayback)
 432         {
 433                 DatabasePlaybackType temp = playback;
 434                 reply->value = &temp;
 435                 reply->success = true;
 436                 reply->completed(reply);
 437
 438                 return;
 439         }
 440
 441         reply->completed(reply);
 442 }
 443
 444 void DatabaseSink::getRangePropertyAsync(AsyncRangePropertyReply *reply)
 445 {
 446         BaseDB * db = new BaseDB();
 447         db->init(databaseName, tablename, tablecreate);
 448
 449         ostringstream query;
 450         query.precision(15);
 451
 452         query<<"SELECT * from "<<tablename<<" WHERE (";
 453
 454         for(auto itr = reply->properties.begin(); itr != reply->properties.end(); itr++)
 455         {
 456                 if(itr != reply->properties.begin())
 457                         query<<" OR ";
 458
 459                 query<<"key='"<<(*itr)<<"'";
 460         }
 461
 462         query<<") AND";
 463
 464         if(reply->timeBegin && reply->timeEnd)
 465         {
 466                 query<<" time BETWEEN "<<reply->timeBegin<<" AND "<<reply->timeEnd;
 467         }
 468
 469         if(reply->sequenceBegin >= 0 && reply->sequenceEnd >=0)
 470         {
 471                 query<<" AND sequence BETWEEN "<<reply->sequenceBegin<<" AND "<<reply->sequenceEnd;
 472         }
 473
 474         std::vector<std::vector<string>> data = db->select(query.str());
 475
 476         DebugOut()<<"Dataset size "<<data.size()<<endl;
 477
 478         for(auto i=0;i<data.size();i++)
 479         {
 480                 if(data[i].size() != 5)
 481                         continue;
 482
 483                 DBObject dbobj;
 484                 dbobj.key = data[i][0];
 485                 dbobj.value = data[i][1];
 486                 dbobj.source = data[i][2];
 487                 dbobj.time = boost::lexical_cast<double>(data[i][3]);
 488                 dbobj.sequence = boost::lexical_cast<double>(data[i][4]);
 489
 490                 AbstractPropertyType* property = VehicleProperty::getPropertyTypeForPropertyNameValue(dbobj.key, dbobj.value);
 491                 if(property)
 492                 {
 493                         property->timestamp = dbobj.time;
 494                         property->sequence = dbobj.sequence;
 495
 496                         reply->values.push_back(property);
 497                 }
 498         }
 499
 500         reply->success = true;
 501         reply->completed(reply);
 502
 503         delete db;
 504 }
 505
 506 AsyncPropertyReply *DatabaseSink::setProperty(AsyncSetPropertyRequest request)
 507 {
 508         AsyncPropertyReply* reply = new AsyncPropertyReply(request);
 509         reply->success = false;
 510
 511         if(request.property == DatabaseLogging)
 512         {
 513                 if(request.value->value<bool>())
 514                 {
 515                         setPlayback(false);
 516                         startDb();
 517                         reply->success = true;
 518                         DatabaseLoggingType temp(true);
 519                         routingEngine->updateProperty(&temp,uuid());
 520                 }
 521                 else
 522                 {
 523                         stopDb();
 524                         reply->success = true;
 525                         DatabaseLoggingType temp(false);
 526                         routingEngine->updateProperty(&temp,uuid());
 527                 }
 528         }
 529
 530         else if(request.property == DatabaseFile)
 531         {
 532                 std::string fname = request.value->toString();
 533
 534                 databaseName = fname;
 535
 536                 DatabaseFileType temp(databaseName);
 537
 538                 routingEngine->updateProperty(DatabaseFile,&temp,uuid());
 539
 540                 reply->success = true;
 541         }
 542         else if( request.property == DatabasePlayback)
 543         {
 544                 if(request.value->value<bool>())
 545                 {
 546                         setLogging(false);
 547                         startPlayback();
 548
 549                         DatabasePlaybackType temp(playback);
 550
 551                         routingEngine->updateProperty(&temp,uuid());
 552                 }
 553                 else
 554                 {
 555                         if(playbackShared)
 556                                 playbackShared->stop = true;
 557
 558                         playback = false;
 559
 560                         DatabasePlaybackType temp(playback);
 561
 562                         routingEngine->updateProperty(&temp,uuid());
 563                 }
 564
 565                 reply->success = true;
 566         }
 567
 568         return reply;
 569 }
 570
 571 void DatabaseSink::subscribeToPropertyChanges(VehicleProperty::Property )
 572 {
 573
 574 }
 575
 576 void DatabaseSink::unsubscribeToPropertyChanges(VehicleProperty::Property )
 577 {
 578 }