plugins/database/databasesink.cpp

   1 #include "databasesink.h"
   2 #include "abstractroutingengine.h"
   3 #include "listplusplus.h"
   4 #include "superptr.hpp"
   5 #include "uuidhelper.h"
   6 #include "ambplugin.h"
   7
   8 #include <thread>
   9
  10 int bufferLength = 100;
  11 int timeout=1000;
  12
  13 extern "C" void create(AbstractRoutingEngine* routingengine, map<string, string> config)
  14 {
  15         auto plugin = new AmbPlugin<DatabaseSink>(routingengine, config);
  16         plugin->init();
  17 }
  18
  19 static void * cbFunc(Shared* shared)
  20 {
  21         if(!shared)
  22         {
  23                 throw std::runtime_error("Could not get shared object.");
  24         }
  25
  26         ///new tripID:
  27         shared->tripId = amb::createUuid();
  28
  29         vector<DictionaryList<string> > insertList;
  30
  31         double startTime = amb::currentTime();
  32
  33         while(1)
  34         {
  35                 DBObject obj = shared->queue.pop();
  36
  37                 if( obj.quit )
  38                 {
  39                         break;
  40                 }
  41
  42                 DictionaryList<string> dict;
  43
  44                 NameValuePair<string> one("key", obj.key);
  45                 NameValuePair<string> two("value", obj.value);
  46                 NameValuePair<string> three("source", obj.source);
  47                 NameValuePair<string> zone("zone", boost::lexical_cast<string>(obj.zone));
  48                 NameValuePair<string> four("time", boost::lexical_cast<string>(amb::Timestamp::instance()->epochTime(obj.time)));
  49                 NameValuePair<string> five("sequence", boost::lexical_cast<string>(obj.sequence));
  50                 NameValuePair<string> six("tripId", shared->tripId);
  51
  52                 dict.push_back(one);
  53                 dict.push_back(two);
  54                 dict.push_back(three);
  55                 dict.push_back(zone);
  56                 dict.push_back(four);
  57                 dict.push_back(five);
  58                 dict.push_back(six);
  59
  60                 insertList.push_back(dict);
  61
  62                 if(insertList.size() >= bufferLength && amb::currentTime() - startTime >= timeout / 1000)
  63                 {
  64                         startTime = amb::currentTime();
  65
  66                         shared->db->exec("BEGIN IMMEDIATE TRANSACTION");
  67                         for(int i=0; i< insertList.size(); i++)
  68                         {
  69                                 DictionaryList<string> d = insertList[i];
  70                                 shared->db->insert(d);
  71                         }
  72                         shared->db->exec("END TRANSACTION");
  73                         insertList.clear();
  74                 }
  75                 //delete obj;
  76         }
  77
  78         /// final flush of whatever is still in the queue:
  79
  80         shared->db->exec("BEGIN IMMEDIATE TRANSACTION");
  81         for(auto d : insertList)
  82         {
  83                 shared->db->insert(d);
  84         }
  85         shared->db->exec("END TRANSACTION");
  86
  87         return NULL;
  88 }
  89
  90 int getNextEvent(gpointer data)
  91 {
  92         PlaybackShared* pbshared = static_cast<PlaybackShared*>(data);
  93
  94         if(!pbshared)
  95                 throw std::runtime_error("failed to cast PlaybackShared object");
  96
  97         if(pbshared->stop)
  98                 return 0;
  99
 100         auto itr = pbshared->playbackQueue.begin();
 101
 102         if(itr == pbshared->playbackQueue.end())
 103         {
 104                 return 0;
 105         }
 106
 107         DBObject obj = *itr;
 108
 109         auto value = amb::make_unique(VehicleProperty::getPropertyTypeForPropertyNameValue(obj.key, obj.value));
 110
 111         if(value)
 112         {
 113                 value->priority = AbstractPropertyType::Instant;
 114                 value->timestamp = obj.time;
 115                 value->sequence = obj.sequence;
 116                 value->sourceUuid = obj.source;
 117                 value->zone = obj.zone;
 118                 pbshared->routingEngine->updateProperty(value.get(), pbshared->uuid);
 119         }
 120
 121         if(++itr != pbshared->playbackQueue.end())
 122         {
 123                 DBObject o2 = *itr;
 124                 double t = o2.time - obj.time;
 125
 126                 if(t > 0)
 127                         g_timeout_add((t*1000) / pbshared->playBackMultiplier, getNextEvent, pbshared);
 128                 else
 129                         g_timeout_add(1, getNextEvent, pbshared);
 130         }
 131
 132         pbshared->playbackQueue.remove(obj);
 133         DebugOut()<<"playback Queue size: "<<pbshared->playbackQueue.size()<<endl;
 134
 135         return 0;
 136 }
 137
 138 DatabaseSink::DatabaseSink(AbstractRoutingEngine *engine, map<std::string, std::string> config, AbstractSource &parent)
 139         :AmbPluginImpl(engine, config, parent), shared(nullptr), playback(false), playbackShared(nullptr), playbackMultiplier(1)
 140 {
 141         tablename = "data";
 142         tablecreate = database_table_create;
 143
 144         if(config.find("bufferLength") != config.end())
 145         {
 146                 bufferLength = boost::lexical_cast<int>(config["bufferLength"]);
 147         }
 148
 149         if(config.find("frequency") != config.end())
 150         {
 151                 try
 152                 {
 153                         int t = boost::lexical_cast<int>(config["frequency"]);
 154                         timeout = 1000 / t;
 155                 }catch(...)
 156                 {
 157                         DebugOut(DebugOut::Error)<<"Failed to parse frequency: Invalid value "<<config["frequency"]<<endl;
 158                 }
 159         }
 160
 161         if(config.find("properties") != config.end())
 162         {
 163                 parseConfig();
 164         }
 165
 166         for(auto itr : propertiesToSubscribeTo)
 167         {
 168                 engine->subscribeToProperty(itr, &parent);
 169         }
 170
 171         databaseName = addPropertySupport(Zone::None, [](){ return new DatabaseFileType("storage"); });
 172         playback = addPropertySupport(Zone::None, [](){ return new DatabasePlaybackType(false); });
 173         databaseLogging = addPropertySupport(Zone::None, [](){ return new DatabaseLoggingType(false); });
 174
 175         if(config.find("startOnLoad")!= config.end())
 176         {
 177                 DebugOut() << "start on load? " << config["startOnLoad"] << endl;
 178                 databaseLogging->setValue(config["startOnLoad"] == "true");
 179         }
 180
 181         if(config.find("playbackMultiplier")!= config.end())
 182         {
 183                 playbackMultiplier = boost::lexical_cast<uint>(config["playbackMultiplier"]);
 184         }
 185
 186         if(config.find("playbackOnLoad")!= config.end())
 187         {
 188                 playback->setValue(config["playbackOnLoad"] == "true");
 189         }
 190 }
 191
 192 DatabaseSink::~DatabaseSink()
 193 {
 194         if(shared)
 195         {
 196                 stopDb();
 197         }
 198
 199         if(playbackShared)
 200         {
 201                 delete playbackShared;
 202         }
 203 }
 204
 205
 206 void DatabaseSink::supportedChanged(const PropertyList &supportedProperties)
 207 {
 208
 209 }
 210
 211 void DatabaseSink::parseConfig()
 212 {
 213         json_object *rootobject;
 214         json_tokener *tokener = json_tokener_new();
 215         enum json_tokener_error err;
 216         do
 217         {
 218                 rootobject = json_tokener_parse_ex(tokener, configuration["properties"].c_str(),configuration["properties"].size());
 219         } while ((err = json_tokener_get_error(tokener)) == json_tokener_continue);
 220         if (err != json_tokener_success)
 221         {
 222                 fprintf(stderr, "Error: %s\n", json_tokener_error_desc(err));
 223         }
 224         if (tokener->char_offset < configuration["properties"].size()) // XXX shouldn't access internal fields
 225         {
 226                 //Should handle the extra data here sometime...
 227         }
 228
 229         json_object *propobject = json_object_object_get(rootobject,"properties");
 230
 231         g_assert(json_object_get_type(propobject) == json_type_array);
 232
 233         array_list *proplist = json_object_get_array(propobject);
 234
 235         for(int i=0; i < array_list_length(proplist); i++)
 236         {
 237                 json_object *idxobj = (json_object*)array_list_get_idx(proplist,i);
 238                 std::string prop = json_object_get_string(idxobj);
 239                 propertiesToSubscribeTo.push_back(prop);
 240
 241                 DebugOut()<<"DatabaseSink logging: "<<prop<<endl;
 242         }
 243
 244         //json_object_put(propobject);
 245         json_object_put(rootobject);
 246 }
 247
 248 void DatabaseSink::stopDb()
 249 {
 250         databaseLogging->setValue(false);
 251
 252         if(!shared)
 253                 return;
 254
 255         DBObject obj;
 256         obj.quit = true;
 257         shared->queue.append(obj);
 258
 259         if(thread && thread->joinable())
 260                 thread->join();
 261
 262         delete shared;
 263         shared = NULL;
 264
 265         routingEngine->updateProperty(databaseLogging.get(), source.uuid());
 266 }
 267
 268 void DatabaseSink::startDb()
 269 {
 270         if(playback->value<bool>())
 271         {
 272                 DebugOut(DebugOut::Error)<<"ERROR: tried to start logging during playback.  Only logging or playback can be used at one time"<<endl;
 273                 return;
 274         }
 275
 276         if(shared)
 277         {
 278                 DebugOut(DebugOut::Warning)<<"WARNING: logging already started.  doing nothing."<<endl;
 279                 return;
 280         }
 281
 282         initDb();
 283
 284         if(thread && thread->joinable())
 285                 thread->detach();
 286
 287         thread = amb::make_unique(new std::thread(cbFunc, shared));
 288
 289         databaseLogging->setValue(true);
 290         routingEngine->updateProperty(databaseLogging.get(), source.uuid());
 291 }
 292
 293 void DatabaseSink::startPlayback()
 294 {
 295         if(playback->value<bool>())
 296                 return;
 297
 298         playback->setValue(true);
 299
 300         initDb();
 301
 302         /// populate playback queue:
 303
 304         vector<vector<string> > results = shared->db->select("SELECT * FROM "+tablename);
 305
 306         stopDb();
 307
 308         if(playbackShared)
 309         {
 310                 delete playbackShared;
 311         }
 312
 313         playbackShared = new PlaybackShared(routingEngine, uuid(), playbackMultiplier);
 314
 315         for(int i=0;i<results.size();i++)
 316         {
 317                 if(results[i].size() < 5)
 318                 {
 319                         throw std::runtime_error("column mismatch in query");
 320                 }
 321
 322                 DBObject obj;
 323
 324                 obj.key = results[i][0];
 325                 obj.value = results[i][1];
 326                 obj.source = results[i][2];
 327                 obj.zone = boost::lexical_cast<int>(results[i][3]);
 328                 obj.time = boost::lexical_cast<double>(results[i][4]);
 329                 obj.sequence = boost::lexical_cast<double>(results[i][5]);
 330
 331                 playbackShared->playbackQueue.push_back(obj);
 332         }
 333
 334         g_timeout_add(0, getNextEvent, playbackShared);
 335 }
 336
 337 void DatabaseSink::initDb()
 338 {
 339         if(shared) delete shared;
 340
 341         shared = new Shared;
 342         shared->db->init(databaseName->value<std::string>(), tablename, tablecreate);
 343 }
 344
 345 void DatabaseSink::setDatabaseFileName(string filename)
 346 {
 347         bool isLogging = databaseLogging->value<bool>();
 348
 349         stopDb();
 350         initDb();
 351
 352         vector<vector<string> > supportedStr = shared->db->select("SELECT DISTINCT key, zone, source FROM " + tablename);
 353
 354         for(int i=0; i < supportedStr.size(); i++)
 355         {
 356                 std::string name = supportedStr[i][0];
 357
 358                 if(!contains(supported(), name))
 359                 {
 360                         std::string zoneStr = supportedStr[i][1];
 361                         std::string sourceStr = supportedStr[i][2];
 362
 363                         DebugOut() << "adding property " << name << " in zone: " << zoneStr << "for source: " << sourceStr << endl;
 364
 365                         Zone::Type zone = boost::lexical_cast<Zone::Type>(zoneStr);
 366                         auto property = addPropertySupport(zone, [name]() { return VehicleProperty::getPropertyTypeForPropertyNameValue(name); });
 367                         property->sourceUuid = sourceStr;
 368                 }
 369         }
 370
 371         if(isLogging)
 372         {
 373                 stopDb();
 374                 startDb();
 375         }
 376
 377         routingEngine->updateSupported(supported(), PropertyList(), &source);
 378 }
 379
 380 void DatabaseSink::propertyChanged(AbstractPropertyType *value)
 381 {
 382         VehicleProperty::Property property = value->name;
 383
 384         DebugOut() << "Received property change for " << property << endl;
 385
 386         if(!shared)
 387                 return;
 388
 389         if(!contains(supported(), property))
 390         {
 391                 addPropertySupport(value->zone, [property]() { return VehicleProperty::getPropertyTypeForPropertyNameValue(property);});
 392                 routingEngine->updateSupported(supported(), PropertyList(), &source);
 393         }
 394
 395         DBObject obj;
 396         obj.key = property;
 397         obj.value = value->toString();
 398         obj.source = value->sourceUuid;
 399         obj.time = value->timestamp;
 400         obj.sequence = value->sequence;
 401         obj.zone = value->zone;
 402
 403         shared->queue.append(obj);
 404 }
 405
 406
 407 const std::string DatabaseSink::uuid() const
 408 {
 409         return "9f88156e-cb92-4472-8775-9c08addf50d3";
 410 }
 411
 412 void DatabaseSink::init()
 413 {
 414         if(configuration.find("databaseFile") != configuration.end())
 415         {
 416                 databaseName->setValue(configuration["databaseFile"]);
 417                 setDatabaseFileName(configuration["databaseFile"]);
 418         }
 419
 420         DebugOut() << "databaseLogging: " << databaseLogging->value<bool>() << endl;
 421
 422         routingEngine->updateSupported(supported(), PropertyList(), &source);
 423 }
 424
 425 void DatabaseSink::getRangePropertyAsync(AsyncRangePropertyReply *reply)
 426 {
 427         BaseDB * db = new BaseDB();
 428         db->init(databaseName->value<std::string>(), tablename, tablecreate);
 429
 430         ostringstream query;
 431         query.precision(15);
 432
 433         query<<"SELECT * from "<<tablename<<" WHERE (";
 434
 435         for(auto itr = reply->properties.begin(); itr != reply->properties.end(); itr++)
 436         {
 437                 DebugOut() << "prop: " << (*itr) << endl;
 438                 if(itr != reply->properties.begin())
 439                         query<<" OR ";
 440
 441                 query<<"key='"<<(*itr)<<"'";
 442         }
 443
 444         query<<")";
 445
 446         if(reply->timeEnd)
 447         {
 448                 query<<" AND time BETWEEN "<<reply->timeBegin<<" AND "<<reply->timeEnd;
 449         }
 450
 451         if(reply->sequenceBegin >= 0 && reply->sequenceEnd >=0)
 452         {
 453                 query<<" AND sequence BETWEEN "<<reply->sequenceBegin<<" AND "<<reply->sequenceEnd;
 454         }
 455
 456         if(reply->sourceUuid != "")
 457                 query<<" AND source='"<<reply->sourceUuid<<"'";
 458
 459         query<<" AND zone="<<reply->zone;
 460
 461         std::vector<std::vector<string>> data = db->select(query.str());
 462
 463         DebugOut()<<"Dataset size "<<data.size()<<endl;
 464
 465         if(!data.size())
 466         {
 467                 reply->success = false;
 468                 reply->error = AsyncPropertyReply::InvalidOperation;
 469         }
 470         else
 471         {
 472                 reply->success = true;
 473         }
 474
 475         for(auto i=0; i<data.size(); i++)
 476         {
 477                 if(data[i].size() != 7)
 478                         continue;
 479
 480                 DBObject dbobj;
 481                 dbobj.key = data[i][0];
 482                 dbobj.value = data[i][1];
 483                 dbobj.source = data[i][2];
 484                 dbobj.zone = boost::lexical_cast<double>(data[i][3]);
 485                 dbobj.time = boost::lexical_cast<double>(data[i][4]);
 486                 dbobj.sequence = boost::lexical_cast<double>(data[i][5]);
 487                 dbobj.tripId = data[i][6];
 488
 489                 AbstractPropertyType* property = VehicleProperty::getPropertyTypeForPropertyNameValue(dbobj.key, dbobj.value);
 490                 if(property)
 491                 {
 492                         property->timestamp = dbobj.time;
 493                         property->sequence = dbobj.sequence;
 494
 495                         reply->values.push_back(property);
 496                 }
 497         }
 498
 499
 500         reply->completed(reply);
 501
 502         delete db;
 503 }
 504
 505 AsyncPropertyReply *DatabaseSink::setProperty(const AsyncSetPropertyRequest &request)
 506 {
 507         AsyncPropertyReply* reply = AmbPluginImpl::setProperty(request);
 508
 509         if(request.property == DatabaseLogging)
 510         {
 511                 if(databaseLogging->value<bool>())
 512                 {
 513                         startDb();
 514                 }
 515                 else
 516                 {
 517                         stopDb();
 518                 }
 519         }
 520         else if(request.property == DatabaseFile)
 521         {
 522                 setDatabaseFileName(databaseName->value<std::string>());
 523         }
 524         else if( request.property == DatabasePlayback)
 525         {
 526                 if(playback->value<bool>())
 527                 {
 528                         startPlayback();
 529                 }
 530                 else
 531                 {
 532                         if(playbackShared)
 533                                 playbackShared->stop = true;
 534
 535                         playback = false;
 536                 }
 537         }
 538
 539         return reply;
 540 }
 541
 542 void DatabaseSink::subscribeToPropertyChanges(VehicleProperty::Property )
 543 {
 544
 545 }
 546
 547 void DatabaseSink::unsubscribeToPropertyChanges(VehicleProperty::Property )
 548 {
 549 }
 550