plugins/database/databasesink.cpp

   1 #include "databasesink.h"
   2 #include "abstractroutingengine.h"
   3 #include "listplusplus.h"
   4 #include "superptr.hpp"
   5 #include "uuidhelper.h"
   6 #include "ambplugin.h"
   7
   8 #include <thread>
   9
  10 int bufferLength = 100;
  11 int timeout=1000;
  12
  13 extern "C" void create(AbstractRoutingEngine* routingengine, map<string, string> config)
  14 {
  15         auto plugin = new AmbPlugin<DatabaseSink>(routingengine, config);
  16         plugin->init();
  17 }
  18
  19 static void * cbFunc(Shared* shared)
  20 {
  21         if(!shared)
  22         {
  23                 throw std::runtime_error("Could not cast shared object.");
  24         }
  25
  26         ///new tripID:
  27         shared->tripId = amb::createUuid();
  28
  29         vector<DictionaryList<string> > insertList;
  30
  31         while(1)
  32         {
  33                 usleep(timeout*1000);
  34
  35                 DBObject obj = shared->queue.pop();
  36
  37                 if( obj.quit )
  38                 {
  39                         break;
  40                 }
  41
  42                 DictionaryList<string> dict;
  43
  44                 NameValuePair<string> one("key", obj.key);
  45                 NameValuePair<string> two("value", obj.value);
  46                 NameValuePair<string> three("source", obj.source);
  47                 NameValuePair<string> zone("zone", boost::lexical_cast<string>(obj.zone));
  48                 NameValuePair<string> four("time", boost::lexical_cast<string>(obj.time));
  49                 NameValuePair<string> five("sequence", boost::lexical_cast<string>(obj.sequence));
  50                 NameValuePair<string> six("tripId", boost::lexical_cast<string>(shared->tripId));
  51
  52                 dict.push_back(one);
  53                 dict.push_back(two);
  54                 dict.push_back(three);
  55                 dict.push_back(zone);
  56                 dict.push_back(four);
  57                 dict.push_back(five);
  58                 dict.push_back(six);
  59
  60                 insertList.push_back(dict);
  61
  62                 if(insertList.size() > bufferLength)
  63                 {
  64                         shared->db->exec("BEGIN IMMEDIATE TRANSACTION");
  65                         for(int i=0; i< insertList.size(); i++)
  66                         {
  67                                 DictionaryList<string> d = insertList[i];
  68                                 shared->db->insert(d);
  69                         }
  70                         shared->db->exec("END TRANSACTION");
  71                         insertList.clear();
  72                 }
  73                 //delete obj;
  74         }
  75
  76         /// final flush of whatever is still in the queue:
  77
  78         shared->db->exec("BEGIN IMMEDIATE TRANSACTION");
  79         for(int i=0; i< insertList.size(); i++)
  80         {
  81                 DictionaryList<string> d = insertList[i];
  82                 shared->db->insert(d);
  83         }
  84         shared->db->exec("END TRANSACTION");
  85
  86         return NULL;
  87 }
  88
  89 int getNextEvent(gpointer data)
  90 {
  91         PlaybackShared* pbshared = static_cast<PlaybackShared*>(data);
  92
  93         if(!pbshared)
  94                 throw std::runtime_error("failed to cast PlaybackShared object");
  95
  96         if(pbshared->stop)
  97                 return 0;
  98
  99         auto itr = pbshared->playbackQueue.begin();
 100
 101         if(itr == pbshared->playbackQueue.end())
 102         {
 103                 return 0;
 104         }
 105
 106         DBObject obj = *itr;
 107
 108         auto value = amb::make_unique(VehicleProperty::getPropertyTypeForPropertyNameValue(obj.key, obj.value));
 109
 110         if(value)
 111         {
 112                 value->priority = AbstractPropertyType::Instant;
 113                 value->timestamp = obj.time;
 114                 value->sequence = obj.sequence;
 115                 value->sourceUuid = obj.source;
 116                 value->zone = obj.zone;
 117                 pbshared->routingEngine->updateProperty(value.get(), pbshared->uuid);
 118         }
 119
 120         if(++itr != pbshared->playbackQueue.end())
 121         {
 122                 DBObject o2 = *itr;
 123                 double t = o2.time - obj.time;
 124
 125                 if(t > 0)
 126                         g_timeout_add((t*1000) / pbshared->playBackMultiplier, getNextEvent, pbshared);
 127                 else
 128                         g_timeout_add(1, getNextEvent, pbshared);
 129         }
 130
 131         pbshared->playbackQueue.remove(obj);
 132         DebugOut()<<"playback Queue size: "<<pbshared->playbackQueue.size()<<endl;
 133
 134         return 0;
 135 }
 136
 137 DatabaseSink::DatabaseSink(AbstractRoutingEngine *engine, map<std::string, std::string> config, AbstractSource &parent)
 138         :AmbPluginImpl(engine, config, parent), shared(nullptr), playback(false), playbackShared(nullptr), playbackMultiplier(1)
 139 {
 140         tablename = "data";
 141         tablecreate = database_table_create;
 142
 143         if(config.find("bufferLength") != config.end())
 144         {
 145                 bufferLength = boost::lexical_cast<int>(config["bufferLength"]);
 146         }
 147
 148         if(config.find("frequency") != config.end())
 149         {
 150                 try
 151                 {
 152                         int t = boost::lexical_cast<int>(config["frequency"]);
 153                         timeout = 1000 / t;
 154                 }catch(...)
 155                 {
 156                         DebugOut(DebugOut::Error)<<"Failed to parse frequency: Invalid value "<<config["frequency"]<<endl;
 157                 }
 158         }
 159
 160         if(config.find("properties") != config.end())
 161         {
 162                 parseConfig();
 163         }
 164
 165         for(auto itr : propertiesToSubscribeTo)
 166         {
 167                 engine->subscribeToProperty(itr, &parent);
 168         }
 169
 170         databaseName = addPropertySupport(Zone::None, [](){ return new DatabaseFileType("storage"); });
 171         playback = addPropertySupport(Zone::None, [](){ return new DatabasePlaybackType(false); });
 172         databaseLogging = addPropertySupport(Zone::None, [](){ return new DatabaseLoggingType(false); });
 173
 174         if(config.find("startOnLoad")!= config.end())
 175         {
 176                 databaseLogging->setValue(config["startOnLoad"] == "true");
 177         }
 178
 179         if(config.find("playbackMultiplier")!= config.end())
 180         {
 181                 playbackMultiplier = boost::lexical_cast<uint>(config["playbackMultiplier"]);
 182         }
 183
 184         if(config.find("playbackOnLoad")!= config.end())
 185         {
 186                 playback->setValue(config["playbackOnLoad"] == "true");
 187         }
 188 }
 189
 190 DatabaseSink::~DatabaseSink()
 191 {
 192         if(shared)
 193         {
 194                 stopDb();
 195         }
 196
 197         if(playbackShared)
 198         {
 199                 delete playbackShared;
 200         }
 201 }
 202
 203
 204 void DatabaseSink::supportedChanged(const PropertyList &supportedProperties)
 205 {
 206
 207 }
 208
 209 void DatabaseSink::parseConfig()
 210 {
 211         json_object *rootobject;
 212         json_tokener *tokener = json_tokener_new();
 213         enum json_tokener_error err;
 214         do
 215         {
 216                 rootobject = json_tokener_parse_ex(tokener, configuration["properties"].c_str(),configuration["properties"].size());
 217         } while ((err = json_tokener_get_error(tokener)) == json_tokener_continue);
 218         if (err != json_tokener_success)
 219         {
 220                 fprintf(stderr, "Error: %s\n", json_tokener_error_desc(err));
 221         }
 222         if (tokener->char_offset < configuration["properties"].size()) // XXX shouldn't access internal fields
 223         {
 224                 //Should handle the extra data here sometime...
 225         }
 226
 227         json_object *propobject = json_object_object_get(rootobject,"properties");
 228
 229         g_assert(json_object_get_type(propobject) == json_type_array);
 230
 231         array_list *proplist = json_object_get_array(propobject);
 232
 233         for(int i=0; i < array_list_length(proplist); i++)
 234         {
 235                 json_object *idxobj = (json_object*)array_list_get_idx(proplist,i);
 236                 std::string prop = json_object_get_string(idxobj);
 237                 propertiesToSubscribeTo.push_back(prop);
 238
 239                 DebugOut()<<"DatabaseSink logging: "<<prop<<endl;
 240         }
 241
 242         //json_object_put(propobject);
 243         json_object_put(rootobject);
 244 }
 245
 246 void DatabaseSink::stopDb()
 247 {
 248         if(!shared)
 249                 return;
 250
 251         DBObject obj;
 252         obj.quit = true;
 253         shared->queue.append(obj);
 254
 255         if(thread && thread->joinable())
 256                 thread->join();
 257
 258         delete shared;
 259         shared = NULL;
 260 }
 261
 262 void DatabaseSink::startDb()
 263 {
 264         if(playback->value<bool>())
 265         {
 266                 DebugOut(0)<<"ERROR: tried to start logging during playback.  Only logging or playback can be used at one time"<<endl;
 267                 return;
 268         }
 269
 270         if(shared)
 271         {
 272                 DebugOut(0)<<"WARNING: logging already started.  doing nothing."<<endl;
 273                 return;
 274         }
 275
 276         initDb();
 277
 278         if(thread && thread->joinable())
 279                 thread->detach();
 280
 281         thread = amb::make_unique(new std::thread(cbFunc, shared));
 282 }
 283
 284 void DatabaseSink::startPlayback()
 285 {
 286         if(playback->value<bool>())
 287                 return;
 288
 289         playback->setValue(true);
 290
 291         initDb();
 292
 293         /// populate playback queue:
 294
 295         vector<vector<string> > results = shared->db->select("SELECT * FROM "+tablename);
 296
 297         /// we are done with shared.  clean up:
 298         delete shared;
 299         shared = NULL;
 300
 301         if(playbackShared)
 302         {
 303                 delete playbackShared;
 304         }
 305
 306         playbackShared = new PlaybackShared(routingEngine, uuid(), playbackMultiplier);
 307
 308         for(int i=0;i<results.size();i++)
 309         {
 310                 if(results[i].size() < 5)
 311                 {
 312                         throw std::runtime_error("column mismatch in query");
 313                 }
 314
 315                 DBObject obj;
 316
 317                 obj.key = results[i][0];
 318                 obj.value = results[i][1];
 319                 obj.source = results[i][2];
 320                 obj.zone = boost::lexical_cast<int>(results[i][3]);
 321                 obj.time = boost::lexical_cast<double>(results[i][4]);
 322                 obj.sequence = boost::lexical_cast<double>(results[i][5]);
 323
 324                 playbackShared->playbackQueue.push_back(obj);
 325         }
 326
 327         g_timeout_add(0, getNextEvent, playbackShared);
 328 }
 329
 330 void DatabaseSink::initDb()
 331 {
 332         if(shared) delete shared;
 333
 334         shared = new Shared;
 335         shared->db->init(databaseName->value<std::string>(), tablename, tablecreate);
 336 }
 337
 338 void DatabaseSink::setDatabaseFileName(string filename)
 339 {
 340         initDb();
 341
 342         vector<vector<string> > supportedStr = shared->db->select("SELECT DISTINCT key, zone, source FROM " + tablename);
 343
 344         for(int i=0; i < supportedStr.size(); i++)
 345         {
 346                 std::string name = supportedStr[i][0];
 347
 348                 if(!contains(supported(), name))
 349                 {
 350                         std::string zoneStr = supportedStr[i][1];
 351                         std::string sourceStr = supportedStr[i][2];
 352
 353                         DebugOut() << "adding property " << name << " in zone: " << zoneStr << "for source: " << sourceStr << endl;
 354
 355                         Zone::Type zone = boost::lexical_cast<Zone::Type>(zoneStr);
 356                         auto property = addPropertySupport(zone, [name]() { return VehicleProperty::getPropertyTypeForPropertyNameValue(name); });
 357                         property->sourceUuid = sourceStr;
 358                 }
 359         }
 360
 361         delete shared;
 362         shared = NULL;
 363
 364         routingEngine->updateSupported(supported(), PropertyList(), &source);
 365 }
 366
 367 void DatabaseSink::propertyChanged(AbstractPropertyType *value)
 368 {
 369         VehicleProperty::Property property = value->name;
 370
 371         if(!shared)
 372                 return;
 373
 374         if(!contains(supported(), property))
 375         {
 376                 addPropertySupport(value->zone, [property]() { return VehicleProperty::getPropertyTypeForPropertyNameValue(property);});
 377                 routingEngine->updateSupported(supported(), PropertyList(), &source);
 378         }
 379
 380         DBObject obj;
 381         obj.key = property;
 382         obj.value = value->toString();
 383         obj.source = value->sourceUuid;
 384         obj.time = value->timestamp;
 385         obj.sequence = value->sequence;
 386         obj.zone = value->zone;
 387
 388         shared->queue.append(obj);
 389 }
 390
 391
 392 const std::string DatabaseSink::uuid() const
 393 {
 394         return "9f88156e-cb92-4472-8775-9c08addf50d3";
 395 }
 396
 397 void DatabaseSink::init()
 398 {
 399         if(configuration.find("databaseFile") != configuration.end())
 400         {
 401                 databaseName->setValue(configuration["databaseFile"]);
 402                 setDatabaseFileName(configuration["databaseFile"]);
 403         }
 404
 405         routingEngine->updateSupported(supported(), PropertyList(), &source);
 406 }
 407
 408 void DatabaseSink::getRangePropertyAsync(AsyncRangePropertyReply *reply)
 409 {
 410         BaseDB * db = new BaseDB();
 411         db->init(databaseName->value<std::string>(), tablename, tablecreate);
 412
 413         ostringstream query;
 414         query.precision(15);
 415
 416         query<<"SELECT * from "<<tablename<<" WHERE (";
 417
 418         for(auto itr = reply->properties.begin(); itr != reply->properties.end(); itr++)
 419         {
 420                 DebugOut() << "prop: " << (*itr) << endl;
 421                 if(itr != reply->properties.begin())
 422                         query<<" OR ";
 423
 424                 query<<"key='"<<(*itr)<<"'";
 425         }
 426
 427         query<<")";
 428
 429         if(reply->timeEnd)
 430         {
 431                 query<<" AND time BETWEEN "<<reply->timeBegin<<" AND "<<reply->timeEnd;
 432         }
 433
 434         if(reply->sequenceBegin >= 0 && reply->sequenceEnd >=0)
 435         {
 436                 query<<" AND sequence BETWEEN "<<reply->sequenceBegin<<" AND "<<reply->sequenceEnd;
 437         }
 438
 439         if(reply->sourceUuid != "")
 440                 query<<" AND source='"<<reply->sourceUuid<<"'";
 441
 442         query<<" AND zone="<<reply->zone;
 443
 444         std::vector<std::vector<string>> data = db->select(query.str());
 445
 446         DebugOut()<<"Dataset size "<<data.size()<<endl;
 447
 448         if(!data.size())
 449         {
 450                 reply->success = false;
 451                 reply->error = AsyncPropertyReply::InvalidOperation;
 452         }
 453         else
 454         {
 455                 reply->success = true;
 456         }
 457
 458         for(auto i=0; i<data.size(); i++)
 459         {
 460                 if(data[i].size() != 7)
 461                         continue;
 462
 463                 DBObject dbobj;
 464                 dbobj.key = data[i][0];
 465                 dbobj.value = data[i][1];
 466                 dbobj.source = data[i][2];
 467                 dbobj.zone = boost::lexical_cast<double>(data[i][3]);
 468                 dbobj.time = boost::lexical_cast<double>(data[i][4]);
 469                 dbobj.sequence = boost::lexical_cast<double>(data[i][5]);
 470                 dbobj.tripId = data[i][6];
 471
 472                 AbstractPropertyType* property = VehicleProperty::getPropertyTypeForPropertyNameValue(dbobj.key, dbobj.value);
 473                 if(property)
 474                 {
 475                         property->timestamp = dbobj.time;
 476                         property->sequence = dbobj.sequence;
 477
 478                         reply->values.push_back(property);
 479                 }
 480         }
 481
 482
 483         reply->completed(reply);
 484
 485         delete db;
 486 }
 487
 488 AsyncPropertyReply *DatabaseSink::setProperty(AsyncSetPropertyRequest request)
 489 {
 490         AsyncPropertyReply* reply = AmbPluginImpl::setProperty(request);
 491
 492         if(request.property == DatabaseLogging)
 493         {
 494                 if(request.value->value<bool>())
 495                 {
 496                         startDb();
 497                 }
 498                 else
 499                 {
 500                         stopDb();
 501                 }
 502         }
 503         else if(request.property == DatabaseFile)
 504         {
 505                 setDatabaseFileName(databaseName->value<std::string>());
 506         }
 507         else if( request.property == DatabasePlayback)
 508         {
 509                 if(request.value->value<bool>())
 510                 {
 511                         startPlayback();
 512                 }
 513                 else
 514                 {
 515                         if(playbackShared)
 516                                 playbackShared->stop = true;
 517
 518                         playback = false;
 519                 }
 520         }
 521
 522         return reply;
 523 }
 524
 525 void DatabaseSink::subscribeToPropertyChanges(VehicleProperty::Property )
 526 {
 527
 528 }
 529
 530 void DatabaseSink::unsubscribeToPropertyChanges(VehicleProperty::Property )
 531 {
 532 }
 533