src/backends/webdav/CalDAVSource.cpp

   1 /*
   2  * Copyright (C) 2010 Intel Corporation
   3  */
   4
   5 #include "config.h"
   6
   7 #ifdef ENABLE_DAV
   8
   9 // include first, it sets HANDLE_LIBICAL_MEMORY for us
  10 #include <syncevo/icalstrdup.h>
  11
  12 #include "CalDAVSource.h"
  13
  14 #include <boost/bind.hpp>
  15 #include <boost/algorithm/string/replace.hpp>
  16
  17 #include <syncevo/declarations.h>
  18 SE_BEGIN_CXX
  19
  20 /**
  21  * @return "<master>" if subid is empty, otherwise subid
  22  */
  23 static std::string SubIDName(const std::string &subid)
  24 {
  25     return subid.empty() ? "<master>" : subid;
  26 }
  27
  28 /** remove X-SYNCEVOLUTION-EXDATE-DETACHED from VEVENT */
  29 static void removeSyncEvolutionExdateDetached(icalcomponent *parent)
  30 {
  31     icalproperty *prop = icalcomponent_get_first_property(parent, ICAL_ANY_PROPERTY);
  32     while (prop) {
  33         icalproperty *next = icalcomponent_get_next_property(parent, ICAL_ANY_PROPERTY);
  34         const char *xname = icalproperty_get_x_name(prop);
  35         if (xname &&
  36             !strcmp(xname, "X-SYNCEVOLUTION-EXDATE-DETACHED")) {
  37             icalcomponent_remove_property(parent, prop);
  38             icalproperty_free(prop);
  39         }
  40         prop = next;
  41     }
  42 }
  43
  44 CalDAVSource::CalDAVSource(const SyncSourceParams &params,
  45                            const boost::shared_ptr<Neon::Settings> &settings) :
  46     WebDAVSource(params, settings)
  47 {
  48     SyncSourceLogging::init(InitList<std::string>("SUMMARY") + "LOCATION",
  49                             ", ",
  50                             m_operations);
  51     // override default backup/restore from base class with our own
  52     // version
  53     m_operations.m_backupData = boost::bind(&CalDAVSource::backupData,
  54                                             this, _1, _2, _3);
  55     m_operations.m_restoreData = boost::bind(&CalDAVSource::restoreData,
  56                                              this, _1, _2, _3);
  57 }
  58
  59 void CalDAVSource::listAllSubItems(SubRevisionMap_t &revisions)
  60 {
  61     revisions.clear();
  62
  63     const std::string query =
  64         "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n"
  65         "<C:calendar-query xmlns:D=\"DAV:\"\n"
  66         "xmlns:C=\"urn:ietf:params:xml:ns:caldav\">\n"
  67         "<D:prop>\n"
  68         "<D:getetag/>\n"
  69
  70         // In practice, peers always return the full data dump
  71         // even if asked to return only a subset. Therefore we use this
  72         // REPORT to populate our m_cache instead of sending lots of GET
  73         // requests later on: faster sync, albeit with higher
  74         // memory consumption.
  75         //
  76         // Because incremental syncs typically don't use listAllSubItems(),
  77         // this looks like a good trade-off.
  78 #ifdef SHORT_ALL_SUB_ITEMS_DATA
  79         "<C:calendar-data>\n"
  80         "<C:comp name=\"VCALENDAR\">\n"
  81         "<C:prop name=\"VERSION\"/>\n"
  82         "<C:comp name=\"VEVENT\">\n"
  83         "<C:prop name=\"SUMMARY\"/>\n"
  84         "<C:prop name=\"UID\"/>\n"
  85         "<C:prop name=\"RECURRENCE-ID\"/>\n"
  86         "<C:prop name=\"SEQUENCE\"/>\n"
  87         "</C:comp>\n"
  88         "<C:comp name=\"VTIMEZONE\"/>\n"
  89         "</C:comp>\n"
  90         "</C:calendar-data>\n"
  91 #else
  92         "<C:calendar-data/>\n"
  93 #endif
  94
  95         "</D:prop>\n"
  96         // filter expected by Yahoo! Calendar
  97         "<C:filter>\n"
  98         "<C:comp-filter name=\"VCALENDAR\">\n"
  99         "<C:comp-filter name=\"VEVENT\">\n"
 100         "</C:comp-filter>\n"
 101         "</C:comp-filter>\n"
 102         "</C:filter>\n"
 103         "</C:calendar-query>\n";
 104     Timespec deadline = createDeadline();
 105     getSession()->startOperation("REPORT 'meta data'", deadline);
 106     while (true) {
 107         string data;
 108         Neon::XMLParser parser;
 109         parser.initReportParser(boost::bind(&CalDAVSource::appendItem, this,
 110                                             boost::ref(revisions),
 111                                             _1, _2, boost::ref(data)));
 112         m_cache.clear();
 113         m_cache.m_initialized = false;
 114         parser.pushHandler(boost::bind(Neon::XMLParser::accept, "urn:ietf:params:xml:ns:caldav", "calendar-data", _2, _3),
 115                            boost::bind(Neon::XMLParser::append, boost::ref(data), _2, _3));
 116         Neon::Request report(*getSession(), "REPORT", getCalendar().m_path, query, parser);
 117         report.addHeader("Depth", "1");
 118         report.addHeader("Content-Type", "application/xml; charset=\"utf-8\"");
 119         if (report.run()) {
 120             break;
 121         }
 122     }
 123
 124     m_cache.m_initialized = true;
 125 }
 126
 127 void CalDAVSource::addResource(StringMap &items,
 128                                const std::string &href,
 129                                const std::string &etag)
 130 {
 131     std::string davLUID = path2luid(Neon::URI::parse(href).m_path);
 132     items[davLUID] = ETag2Rev(etag);
 133 }
 134
 135 void CalDAVSource::updateAllSubItems(SubRevisionMap_t &revisions)
 136 {
 137     // list items to identify new, updated and removed ones
 138     const std::string query =
 139         "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n"
 140         "<C:calendar-query xmlns:D=\"DAV:\"\n"
 141         "xmlns:C=\"urn:ietf:params:xml:ns:caldav\">\n"
 142         "<D:prop>\n"
 143         "<D:getetag/>\n"
 144         "</D:prop>\n"
 145         // filter expected by Yahoo! Calendar
 146         "<C:filter>\n"
 147         "<C:comp-filter name=\"VCALENDAR\">\n"
 148         "<C:comp-filter name=\"VEVENT\">\n"
 149         "</C:comp-filter>\n"
 150         "</C:comp-filter>\n"
 151         "</C:filter>\n"
 152         "</C:calendar-query>\n";
 153     Timespec deadline = createDeadline();
 154     StringMap items;
 155     getSession()->startOperation("updateAllSubItems REPORT 'list items'", deadline);
 156     while (true) {
 157         string data;
 158         Neon::XMLParser parser;
 159         items.clear();
 160         parser.initReportParser(boost::bind(&CalDAVSource::addResource,
 161                                             this, boost::ref(items), _1, _2));
 162         Neon::Request report(*getSession(), "REPORT", getCalendar().m_path, query, parser);
 163         report.addHeader("Depth", "1");
 164         report.addHeader("Content-Type", "application/xml; charset=\"utf-8\"");
 165         if (report.run()) {
 166             break;
 167         }
 168     }
 169
 170     // remove obsolete entries
 171     SubRevisionMap_t::iterator it = revisions.begin();
 172     while (it != revisions.end()) {
 173         SubRevisionMap_t::iterator next = it;
 174         ++next;
 175         if (items.find(it->first) == items.end()) {
 176             revisions.erase(it);
 177         }
 178         it = next;
 179     }
 180
 181     // build list of new or updated entries,
 182     // copy others to cache
 183     m_cache.clear();
 184     m_cache.m_initialized = false;
 185     std::list<std::string> mustRead;
 186     BOOST_FOREACH(const StringPair &item, items) {
 187         SubRevisionMap_t::iterator it = revisions.find(item.first);
 188         if (it == revisions.end() ||
 189             it->second.m_revision != item.second) {
 190             // read current information below
 191             SE_LOG_DEBUG(NULL, NULL, "updateAllSubItems(): read new or modified item %s", item.first.c_str());
 192             mustRead.push_back(item.first);
 193             // The server told us that the item exists. We still need
 194             // to deal with the situation that the server might fail
 195             // to deliver the item data when we ask for it below.
 196             //
 197             // There are two reasons when this can happen: either an
 198             // item was removed in the meantime or the server is
 199             // confused.  The latter started to happen reliably with
 200             // the Google Calendar server sometime in January/February
 201             // 2012.
 202             //
 203             // In both cases, let's assume that the item is really gone
 204             // (and not just unreadable due to that other Google Calendar
 205             // bug, see loadItem()+REPORT workaround), and therefore let's
 206             // remove the entry from the revisions.
 207             if (it != revisions.end()) {
 208                 revisions.erase(it);
 209             }
 210             m_cache.erase(item.first);
 211         } else {
 212             // copy still relevant information
 213             SE_LOG_DEBUG(NULL, NULL, "updateAllSubItems(): unmodified item %s", it->first.c_str());
 214             addSubItem(it->first, it->second);
 215         }
 216     }
 217
 218     // request dump of these items, add to cache and revisions
 219     //
 220     // Failures to find or read certain items will be
 221     // ignored. appendItem() will only be called for actually
 222     // retrieved items. This is partly intentional: Google is known to
 223     // have problems with providing all of its data via GET or the
 224     // multiget REPORT below. It returns a 404 error for items that a
 225     // calendar-query includes (see loadItem()). Such items are
 226     // ignored and thus will be silently skipped. This is not
 227     // perfect, but better than failing the sync.
 228     //
 229     // Unfortunately there are other servers (Radicale, I'm looking at
 230     // you) which simply return neither data nor errors for the
 231     // requested hrefs. To handle that we try the multiget first,
 232     // record retrieved or failed responses, then follow up with
 233     // individual requests for anything that wasn't mentioned.
 234     if (!mustRead.empty()) {
 235         std::stringstream buffer;
 236         buffer << "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n"
 237             "<C:calendar-multiget xmlns:D=\"DAV:\"\n"
 238             "   xmlns:C=\"urn:ietf:params:xml:ns:caldav\">\n"
 239             "<D:prop>\n"
 240             "   <D:getetag/>\n"
 241             "   <C:calendar-data/>\n"
 242             "</D:prop>\n";
 243         BOOST_FOREACH(const std::string &luid, mustRead) {
 244             buffer << "<D:href>" << luid2path(luid) << "</D:href>\n";
 245         }
 246         buffer << "</C:calendar-multiget>";
 247         std::string query = buffer.str();
 248         std::set<std::string> results; // LUIDs of all hrefs returned by report
 249         getSession()->startOperation("updateAllSubItems REPORT 'multiget new/updated items'", deadline);
 250         while (true) {
 251             string data;
 252             Neon::XMLParser parser;
 253             parser.initReportParser(boost::bind(&CalDAVSource::appendMultigetResult, this,
 254                                                 boost::ref(revisions),
 255                                                 boost::ref(results),
 256                                                 _1, _2, boost::ref(data)));
 257             parser.pushHandler(boost::bind(Neon::XMLParser::accept, "urn:ietf:params:xml:ns:caldav", "calendar-data", _2, _3),
 258                                boost::bind(Neon::XMLParser::append, boost::ref(data), _2, _3));
 259             Neon::Request report(*getSession(), "REPORT", getCalendar().m_path,
 260                                  query, parser);
 261             report.addHeader("Depth", "1");
 262             report.addHeader("Content-Type", "application/xml; charset=\"utf-8\"");
 263             if (report.run()) {
 264                 break;
 265             }
 266         }
 267         // Workaround for Radicale 0.6.4: it simply returns nothing (no error, no data).
 268         // Fall back to GET of items with no response.
 269         BOOST_FOREACH(const std::string &luid, mustRead) {
 270             if (results.find(luid) == results.end()) {
 271                 getSession()->startOperation(StringPrintf("GET item %s not returned by 'multiget new/updated items'", luid.c_str()),
 272                                              deadline);
 273                 std::string path = luid2path(luid);
 274                 std::string data;
 275                 std::string etag;
 276                 while (true) {
 277                     data.clear();
 278                     Neon::Request req(*getSession(), "GET", path,
 279                                       "", data);
 280                     req.addHeader("Accept", contentType());
 281                     if (req.run()) {
 282                         etag = getETag(req);
 283                         break;
 284                     }
 285                 }
 286                 appendItem(revisions, path, etag, data);
 287             }
 288         }
 289     }
 290 }
 291
 292 int CalDAVSource::appendMultigetResult(SubRevisionMap_t &revisions,
 293                                        std::set<std::string> &luids,
 294                                        const std::string &href,
 295                                        const std::string &etag,
 296                                        std::string &data)
 297 {
 298     // record which items were seen in the response...
 299     luids.insert(path2luid(href));
 300     // and store information about them
 301     return appendItem(revisions, href, etag, data);
 302 }
 303
 304 int CalDAVSource::appendItem(SubRevisionMap_t &revisions,
 305                              const std::string &href,
 306                              const std::string &etag,
 307                              std::string &data)
 308 {
 309     // Ignore responses with no data: this is not perfect (should better
 310     // try to figure out why there is no data), but better than
 311     // failing.
 312     //
 313     // One situation is the response for the collection itself,
 314     // which comes with a 404 status and no data with Google Calendar.
 315     if (data.empty()) {
 316         return 0;
 317     }
 318
 319     Event::unescapeRecurrenceID(data);
 320     eptr<icalcomponent> calendar(icalcomponent_new_from_string((char *)data.c_str()), // cast is a hack for broken definition in old libical
 321                                  "iCalendar 2.0");
 322     Event::fixIncomingCalendar(calendar.get());
 323     std::string davLUID = path2luid(Neon::URI::parse(href).m_path);
 324     SubRevisionEntry &entry = revisions[davLUID];
 325     entry.m_revision = ETag2Rev(etag);
 326     long maxSequence = 0;
 327     std::string uid;
 328     entry.m_subids.clear();
 329     for (icalcomponent *comp = icalcomponent_get_first_component(calendar, ICAL_VEVENT_COMPONENT);
 330          comp;
 331          comp = icalcomponent_get_next_component(calendar, ICAL_VEVENT_COMPONENT)) {
 332         std::string subid = Event::getSubID(comp);
 333         uid = Event::getUID(comp);
 334         long sequence = Event::getSequence(comp);
 335         if (sequence > maxSequence) {
 336             maxSequence = sequence;
 337         }
 338         entry.m_subids.insert(subid);
 339     }
 340     entry.m_uid = uid;
 341
 342     // Ignore items which contain no VEVENT. Happens with Google Calendar
 343     // after using it for a while. Deleting them via DELETE doesn't seem
 344     // to have an effect either, so all we really can do is ignore them.
 345     if (entry.m_subids.empty()) {
 346         SE_LOG_DEBUG(NULL, NULL, "ignoring broken item %s (is empty)", davLUID.c_str());
 347         revisions.erase(davLUID);
 348         m_cache.erase(davLUID);
 349         data.clear();
 350         return 0;
 351     }
 352
 353     if (!m_cache.m_initialized) {
 354         boost::shared_ptr<Event> event(new Event);
 355         event->m_DAVluid = davLUID;
 356         event->m_UID = uid;
 357         event->m_etag = entry.m_revision;
 358         event->m_subids = entry.m_subids;
 359         event->m_sequence = maxSequence;
 360 #ifndef SHORT_ALL_SUB_ITEMS_DATA
 361         // we got a full data dump, use it
 362         for (icalcomponent *comp = icalcomponent_get_first_component(calendar, ICAL_VEVENT_COMPONENT);
 363              comp;
 364              comp = icalcomponent_get_next_component(calendar, ICAL_VEVENT_COMPONENT)) {
 365         }
 366         event->m_calendar = calendar;
 367 #endif
 368         m_cache.insert(make_pair(davLUID, event));
 369     }
 370
 371     // reset data for next item
 372     data.clear();
 373     return 0;
 374 }
 375
 376 void CalDAVSource::addSubItem(const std::string &luid,
 377                               const SubRevisionEntry &entry)
 378 {
 379     boost::shared_ptr<Event> &event = m_cache[luid];
 380     event.reset(new Event);
 381     event->m_DAVluid = luid;
 382     event->m_etag = entry.m_revision;
 383     event->m_UID = entry.m_uid;
 384     // We don't know sequence and last-modified. This
 385     // information will have to be filled in by loadItem()
 386     // when some operation on this event needs it.
 387     event->m_subids = entry.m_subids;
 388 }
 389
 390 void CalDAVSource::setAllSubItems(const SubRevisionMap_t &revisions)
 391 {
 392     if (!m_cache.m_initialized) {
 393         // populate our cache (without data) from the information cached
 394         // for us
 395         BOOST_FOREACH(const SubRevisionMap_t::value_type &subentry,
 396                       revisions) {
 397             addSubItem(subentry.first,
 398                        subentry.second);
 399         }
 400         m_cache.m_initialized = true;
 401     }
 402 }
 403
 404 SubSyncSource::SubItemResult CalDAVSource::insertSubItem(const std::string &luid, const std::string &callerSubID,
 405                                                          const std::string &item)
 406 {
 407     SubItemResult subres;
 408
 409     // parse new event
 410     boost::shared_ptr<Event> newEvent(new Event);
 411     newEvent->m_calendar.set(icalcomponent_new_from_string((char *)item.c_str()), // hack for old libical
 412                              "parsing iCalendar 2.0");
 413     struct icaltimetype lastmodtime = icaltime_null_time();
 414     icalcomponent *firstcomp = NULL;
 415     for (icalcomponent *comp = firstcomp = icalcomponent_get_first_component(newEvent->m_calendar, ICAL_VEVENT_COMPONENT);
 416          comp;
 417          comp = icalcomponent_get_next_component(newEvent->m_calendar, ICAL_VEVENT_COMPONENT)) {
 418         std::string subid = Event::getSubID(comp);
 419         EventCache::iterator it;
 420         // remove X-SYNCEVOLUTION-EXDATE-DETACHED, could be added by
 421         // the engine's read/modify/write cycle when resolving a
 422         // conflict
 423         removeSyncEvolutionExdateDetached(comp);
 424         if (!luid.empty() &&
 425             (it = m_cache.find(luid)) != m_cache.end()) {
 426             // Additional sanity check: ensure that the expected UID is set.
 427             // Necessary if the peer we synchronize with (aka the local
 428             // data storage) doesn't support foreign UIDs. Maemo 5 calendar
 429             // backend is one example.
 430             Event::setUID(comp, it->second->m_UID);
 431             newEvent->m_UID = it->second->m_UID;
 432         } else {
 433             newEvent->m_UID = Event::getUID(comp);
 434             if (newEvent->m_UID.empty()) {
 435                 // create new UID
 436                 newEvent->m_UID = UUID();
 437                 Event::setUID(comp, newEvent->m_UID);
 438             }
 439         }
 440         newEvent->m_sequence = Event::getSequence(comp);
 441         newEvent->m_subids.insert(subid);
 442
 443         // set DTSTAMP to LAST-MODIFIED in replacement
 444         //
 445         // Needed because Google insists on replacing the original
 446         // DTSTAMP and checks it (409, "Can only store an event with
 447         // a newer DTSTAMP").
 448         //
 449         // According to RFC 2445, the property is set once when the
 450         // event is created for the first time. RFC 5545 extends this
 451         // and states that without a METHOD property (the case with
 452         // CalDAV), DTSTAMP is identical to LAST-MODIFIED, so Google
 453         // is right.
 454         icalproperty *dtstamp = icalcomponent_get_first_property(comp, ICAL_DTSTAMP_PROPERTY);
 455         if (dtstamp) {
 456             icalproperty *lastmod = icalcomponent_get_first_property(comp, ICAL_LASTMODIFIED_PROPERTY);
 457             if (lastmod) {
 458                 lastmodtime = icalproperty_get_lastmodified(lastmod);
 459                 icalproperty_set_dtstamp(dtstamp, lastmodtime);
 460             }
 461         }
 462     }
 463     if (newEvent->m_subids.size() != 1) {
 464         SE_THROW("new CalDAV item did not contain exactly one VEVENT");
 465     }
 466     std::string subid = *newEvent->m_subids.begin();
 467
 468     // Determine whether we already know the merged item even though
 469     // our caller didn't.
 470     std::string davLUID = luid;
 471     std::string knownSubID = callerSubID;
 472     if (davLUID.empty()) {
 473         EventCache::iterator it = m_cache.findByUID(newEvent->m_UID);
 474         if (it != m_cache.end()) {
 475             davLUID = it->first;
 476             knownSubID = subid;
 477         }
 478     }
 479
 480     if (davLUID.empty()) {
 481         // New VEVENT; should not be part of an existing merged item
 482         // ("meeting series").
 483         //
 484         // If another app created a resource with the same UID,
 485         // then two things can happen:
 486         // 1. server merges the data (Google)
 487         // 2. adding the item is rejected (standard compliant CalDAV server)
 488         //
 489         // If the UID is truely new, then
 490         // 3. the server may rename the item
 491         //
 492         // The following code deals with case 3 and also covers case
 493         // 1, but our usual Google workarounds (for example, no
 494         // patching of SEQUENCE) were not applied and thus sending the
 495         // item might fail.
 496         //
 497         // Case 2 is not currently handled and causes the sync to fail.
 498         // This is in line with the current design ("concurrency detected,
 499         // causes error, fixed by trying again in slow sync").
 500         InsertItemResult res;
 501         // Yahoo expects resource names to match UID + ".ics".
 502         std::string name = newEvent->m_UID + ".ics";
 503         std::string buffer;
 504         const std::string *data;
 505         if (!settings().googleChildHack() || subid.empty()) {
 506             // avoid re-encoding item data
 507             data = &item;
 508         } else {
 509             // sanitize item first: when adding child event without parent,
 510             // then the RECURRENCE-ID confuses Google
 511             eptr<char> icalstr(ical_strdup(icalcomponent_as_ical_string(newEvent->m_calendar)));
 512             buffer = icalstr.get();
 513             Event::escapeRecurrenceID(buffer);
 514             data = &buffer;
 515         }
 516         SE_LOG_DEBUG(this, NULL, "inserting new VEVENT");
 517         res = insertItem(name, *data, true);
 518         subres.m_mainid = res.m_luid;
 519         subres.m_uid = newEvent->m_UID;
 520         subres.m_subid = subid;
 521         subres.m_revision = res.m_revision;
 522
 523         EventCache::iterator it = m_cache.find(res.m_luid);
 524         if (it != m_cache.end()) {
 525             // merge into existing Event
 526             Event &event = loadItem(*it->second);
 527             event.m_etag = res.m_revision;
 528             if (event.m_subids.find(subid) != event.m_subids.end()) {
 529                 // was already in that item but caller didn't seem to know,
 530                 // and now we replaced the data on the CalDAV server
 531                 subres.m_state = ITEM_REPLACED;
 532             } else {
 533                 // add to merged item
 534                 event.m_subids.insert(subid);
 535             }
 536             icalcomponent_merge_component(event.m_calendar,
 537                                           newEvent->m_calendar.release()); // function destroys merged calendar
 538         } else {
 539             // Google Calendar adds a default alarm each time a VEVENT is added
 540             // anew. Avoid that by resending our data if necessary (= no alarm set).
 541             if (settings().googleAlarmHack() &&
 542                 !icalcomponent_get_first_component(firstcomp, ICAL_VALARM_COMPONENT)) {
 543                 // add to cache, then update it
 544                 newEvent->m_DAVluid = res.m_luid;
 545                 newEvent->m_etag = res.m_revision;
 546                 m_cache[newEvent->m_DAVluid] = newEvent;
 547
 548                 // potentially need to know sequence and mod time on server:
 549                 // keep pointer (clears pointer in newEvent),
 550                 // then get and parse new copy from server
 551                 eptr<icalcomponent> calendar = newEvent->m_calendar;
 552
 553                 if (settings().googleUpdateHack()) {
 554                     loadItem(*newEvent);
 555
 556                     // increment in original data
 557                     newEvent->m_sequence++;
 558                     newEvent->m_lastmodtime++;
 559                     Event::setSequence(firstcomp, newEvent->m_sequence);
 560                     icalproperty *lastmod = icalcomponent_get_first_property(firstcomp, ICAL_LASTMODIFIED_PROPERTY);
 561                     if (lastmod) {
 562                         lastmodtime = icaltime_from_timet(newEvent->m_lastmodtime, false);
 563                         lastmodtime.is_utc = 1;
 564                         icalproperty_set_lastmodified(lastmod, lastmodtime);
 565                     }
 566                     icalproperty *dtstamp = icalcomponent_get_first_property(firstcomp, ICAL_DTSTAMP_PROPERTY);
 567                     if (dtstamp) {
 568                         icalproperty_set_dtstamp(dtstamp, lastmodtime);
 569                     }
 570                     // re-encode below
 571                     data = &buffer;
 572                 }
 573                 bool mangleRecurrenceID = settings().googleChildHack() && !subid.empty();
 574                 if (data == &buffer || mangleRecurrenceID) {
 575                     eptr<char> icalstr(ical_strdup(icalcomponent_as_ical_string(calendar)));
 576                     buffer = icalstr.get();
 577                 }
 578                 if (mangleRecurrenceID) {
 579                     Event::escapeRecurrenceID(buffer);
 580                 }
 581                 SE_LOG_DEBUG(NULL, NULL, "resending VEVENT to get rid of VALARM");
 582                 res = insertItem(name, *data, true);
 583                 newEvent->m_etag =
 584                     subres.m_revision = res.m_revision;
 585                 newEvent->m_calendar = calendar;
 586             } else {
 587                 // add to cache without further changes
 588                 newEvent->m_DAVluid = res.m_luid;
 589                 newEvent->m_etag = res.m_revision;
 590                 m_cache[newEvent->m_DAVluid] = newEvent;
 591             }
 592         }
 593     } else {
 594         if (!subid.empty() && subid != knownSubID) {
 595             SE_THROW(StringPrintf("new CalDAV item does not have right RECURRENCE-ID: item %s != expected %s",
 596                                   subid.c_str(), knownSubID.c_str()));
 597         }
 598         Event &event = loadItem(davLUID);
 599
 600         if (subid.empty() && subid != knownSubID) {
 601             // fix incomplete iCalendar 2.0 item: should have had a RECURRENCE-ID
 602             icalcomponent *newcomp =
 603                 icalcomponent_get_first_component(newEvent->m_calendar, ICAL_VEVENT_COMPONENT);
 604             icalproperty *prop = icalcomponent_get_first_property(newcomp, ICAL_RECURRENCEID_PROPERTY);
 605             if (prop) {
 606                 icalcomponent_remove_property(newcomp, prop);
 607                 icalproperty_free(prop);
 608             }
 609
 610             // reconstruct RECURRENCE-ID with known value and TZID from start time of
 611             // the parent event or (if not found) the current event
 612             eptr<icalproperty> rid(icalproperty_new_recurrenceid(icaltime_from_string(knownSubID.c_str())),
 613                                    "new rid");
 614             icalproperty *dtstart = NULL;
 615             icalcomponent *comp;
 616             // look for parent first
 617             for (comp = icalcomponent_get_first_component(event.m_calendar, ICAL_VEVENT_COMPONENT);
 618                  comp && !dtstart;
 619                  comp = icalcomponent_get_next_component(event.m_calendar, ICAL_VEVENT_COMPONENT)) {
 620                 if (!icalcomponent_get_first_property(comp, ICAL_RECURRENCEID_PROPERTY)) {
 621                     dtstart = icalcomponent_get_first_property(comp, ICAL_DTSTART_PROPERTY);
 622                 }
 623             }
 624             // fall back to current event
 625             if (!dtstart) {
 626                 dtstart = icalcomponent_get_first_property(newcomp, ICAL_DTSTART_PROPERTY);
 627             }
 628             // ignore missing TZID
 629             if (dtstart) {
 630                 icalparameter *tzid = icalproperty_get_first_parameter(dtstart, ICAL_TZID_PARAMETER);
 631                 if (tzid) {
 632                     icalproperty_set_parameter(rid, icalparameter_new_clone(tzid));
 633                 }
 634             }
 635
 636             // finally add RECURRENCE-ID and fix newEvent's meta information
 637             icalcomponent_add_property(newcomp, rid.release());
 638             subid = knownSubID;
 639             newEvent->m_subids.erase("");
 640             newEvent->m_subids.insert(subid);
 641         }
 642
 643         // no changes expected yet, copy previous attributes
 644         subres.m_mainid = davLUID;
 645         subres.m_uid = event.m_UID;
 646         subres.m_subid = subid;
 647         subres.m_revision = event.m_etag;
 648
 649         // Google hack: increase sequence number if smaller or equal to
 650         // sequence on server. Server rejects update otherwise.
 651         // See http://code.google.com/p/google-caldav-issues/issues/detail?id=26
 652         if (settings().googleUpdateHack()) {
 653             // always bump SEQ by one before PUT
 654             event.m_sequence++;
 655             if (newEvent->m_sequence < event.m_sequence) {
 656                 // override in new event, existing ones will be updated below
 657                 Event::setSequence(firstcomp, event.m_sequence);
 658             } else {
 659                 // new event sequence is equal or higher, use that
 660                 event.m_sequence = newEvent->m_sequence;
 661             }
 662         }
 663
 664         // update cache: find old VEVENT and remove it before adding new one,
 665         // update last modified time of all other components
 666         icalcomponent *removeme = NULL;
 667         for (icalcomponent *comp = icalcomponent_get_first_component(event.m_calendar, ICAL_VEVENT_COMPONENT);
 668              comp;
 669              comp = icalcomponent_get_next_component(event.m_calendar, ICAL_VEVENT_COMPONENT)) {
 670             if (Event::getSubID(comp) == subid) {
 671                 removeme = comp;
 672             } else if (settings().googleUpdateHack()) {
 673                 // increase modification time stamps to that of the new item,
 674                 // Google rejects the whole update otherwise
 675                 if (!icaltime_is_null_time(lastmodtime)) {
 676                     icalproperty *dtstamp = icalcomponent_get_first_property(comp, ICAL_DTSTAMP_PROPERTY);
 677                     if (dtstamp) {
 678                         icalproperty_set_dtstamp(dtstamp, lastmodtime);
 679                     }
 680                     icalproperty *lastmod = icalcomponent_get_first_property(comp, ICAL_LASTMODIFIED_PROPERTY);
 681                     if (lastmod) {
 682                         icalproperty_set_lastmodified(lastmod, lastmodtime);
 683                     }
 684                 }
 685                 // set SEQ to the one increased above
 686                 Event::setSequence(comp, event.m_sequence);
 687             }
 688         }
 689         if (davLUID != luid) {
 690             // caller didn't know final UID: if found, then tell him to
 691             // merge the data and try again
 692             if (removeme) {
 693                 subres.m_state = ITEM_NEEDS_MERGE;
 694                 goto done;
 695             } else {
 696                 event.m_subids.insert(subid);
 697             }
 698         } else {
 699             if (removeme) {
 700                 // this is what we expect when the caller mentions the DAV LUID
 701                 icalcomponent_remove_component(event.m_calendar, removeme);
 702                 icalcomponent_free(removeme);
 703             } else {
 704                 // caller confused?!
 705                 SE_THROW("event not found");
 706             }
 707         }
 708
 709         icalcomponent_merge_component(event.m_calendar,
 710                                       newEvent->m_calendar.release()); // function destroys merged calendar
 711         eptr<char> icalstr(ical_strdup(icalcomponent_as_ical_string(event.m_calendar)));
 712         std::string data = icalstr.get();
 713
 714         // Google gets confused when adding a child without parent,
 715         // replace in that case.
 716         bool haveParent = event.m_subids.find("") != event.m_subids.end();
 717         if (settings().googleChildHack() && !haveParent) {
 718             Event::escapeRecurrenceID(data);
 719         }
 720
 721         // TODO: avoid updating item on server immediately?
 722         try {
 723             SE_LOG_DEBUG(this, NULL, "updating VEVENT");
 724             InsertItemResult res = insertItem(event.m_DAVluid, data, true);
 725             if (res.m_state != ITEM_OKAY ||
 726                 res.m_luid != event.m_DAVluid) {
 727                 // should not merge with anything, if so, our cache was invalid
 728                 SE_THROW("CalDAV item not updated as expected");
 729             }
 730             event.m_etag = res.m_revision;
 731             subres.m_revision = event.m_etag;
 732         } catch (const TransportStatusException &ex) {
 733             if (ex.syncMLStatus() == 403 &&
 734                 strstr(ex.what(), "You don't have access to change that event")) {
 735                 // Google Calendar sometimes refuses writes for specific items,
 736                 // typically meetings organized by someone else.
 737 #if 1
 738                 // Treat like a temporary, per item error to avoid aborting the
 739                 // whole sync session. Doesn't really solve the problem (client
 740                 // and server remain out of sync and will run into this again and
 741                 // again), but better than giving up on all items or ignoring the
 742                 // problem.
 743                 SE_THROW_EXCEPTION_STATUS(StatusException,
 744                                           "CalDAV peer rejected updated with 403, keep trying",
 745                                           SyncMLStatus(417));
 746 #else
 747                 // Assume that the item hasn't changed and mark it as "merged".
 748                 // This is incorrect. The 403 error has been seen in cases where
 749                 // a detached recurrence had to be added to an existing meeting
 750                 // series. Ignoring the problem means would keep the detached
 751                 // recurrence out of the server permanently.
 752                 SE_LOG_INFO(this, NULL, "%s: not updated because CalDAV server refused write access for it",
 753                             getSubDescription(event, subid).c_str());
 754                 subres.m_merged = true;
 755                 subres.m_revision = event.m_etag;
 756 #endif
 757             } else if (ex.syncMLStatus() == 409 &&
 758                        strstr(ex.what(), "Can only store an event with a newer DTSTAMP")) {
 759                 SE_LOG_DEBUG(NULL, NULL, "resending VEVENT with updated SEQUENCE/LAST-MODIFIED/DTSTAMP to work around 409");
 760
 761                 // Sometimes a PUT of two linked events updates one of them on the server
 762                 // (visible in modified SEQUENCE and LAST-MODIFIED values) and then
 763                 // fails with 409 because, presumably, the other item now has
 764                 // too low SEQUENCE/LAST-MODIFIED/DTSTAMP values.
 765                 //
 766                 // An attempt with splitting the PUT in advance worked for some cases,
 767                 // but then it still happened for others. So let's use brute force and
 768                 // try again once more after reading the updated event anew.
 769                 eptr<icalcomponent> fullcal = event.m_calendar;
 770                 loadItem(event);
 771                 event.m_sequence++;
 772                 lastmodtime = icaltime_from_timet(event.m_lastmodtime, false);
 773                 lastmodtime.is_utc = 1;
 774                 event.m_calendar = fullcal;
 775                 for (icalcomponent *comp = icalcomponent_get_first_component(event.m_calendar, ICAL_VEVENT_COMPONENT);
 776                      comp;
 777                      comp = icalcomponent_get_next_component(event.m_calendar, ICAL_VEVENT_COMPONENT)) {
 778                     if (!icaltime_is_null_time(lastmodtime)) {
 779                         icalproperty *dtstamp = icalcomponent_get_first_property(comp, ICAL_DTSTAMP_PROPERTY);
 780                         if (dtstamp) {
 781                             icalproperty_set_dtstamp(dtstamp, lastmodtime);
 782                         }
 783                         icalproperty *lastmod = icalcomponent_get_first_property(comp, ICAL_LASTMODIFIED_PROPERTY);
 784                         if (lastmod) {
 785                             icalproperty_set_lastmodified(lastmod, lastmodtime);
 786                         }
 787                     }
 788                     Event::setSequence(comp, event.m_sequence);
 789                 }
 790                 eptr<char> icalstr(ical_strdup(icalcomponent_as_ical_string(event.m_calendar)));
 791                 std::string data = icalstr.get();
 792                 InsertItemResult res = insertItem(event.m_DAVluid, data, true);
 793                 if (res.m_state != ITEM_OKAY ||
 794                     res.m_luid != event.m_DAVluid) {
 795                     // should not merge with anything, if so, our cache was invalid
 796                     SE_THROW("CalDAV item not updated as expected");
 797                 }
 798                 event.m_etag = res.m_revision;
 799                 subres.m_revision = event.m_etag;
 800             } else {
 801                 throw;
 802             }
 803         }
 804     }
 805
 806  done:
 807     return subres;
 808 }
 809
 810 void CalDAVSource::readSubItem(const std::string &davLUID, const std::string &subid, std::string &item)
 811 {
 812     Event &event = loadItem(davLUID);
 813     if (event.m_subids.size() == 1) {
 814         // simple case: convert existing VCALENDAR
 815         if (*event.m_subids.begin() == subid) {
 816             eptr<char> icalstr(ical_strdup(icalcomponent_as_ical_string(event.m_calendar)));
 817             item = icalstr.get();
 818         } else {
 819             SE_THROW("event not found");
 820         }
 821     } else {
 822         // complex case: create VCALENDAR with just the VTIMEZONE definition(s)
 823         // and the one event, then convert that
 824         eptr<icalcomponent> calendar(icalcomponent_new(ICAL_VCALENDAR_COMPONENT), "VCALENDAR");
 825         for (icalcomponent *tz = icalcomponent_get_first_component(event.m_calendar, ICAL_VTIMEZONE_COMPONENT);
 826              tz;
 827              tz = icalcomponent_get_next_component(event.m_calendar, ICAL_VTIMEZONE_COMPONENT)) {
 828             eptr<icalcomponent> clone(icalcomponent_new_clone(tz), "VTIMEZONE");
 829             icalcomponent_add_component(calendar, clone.release());
 830         }
 831         bool found = false;
 832         icalcomponent *parent = NULL;
 833         for (icalcomponent *comp = icalcomponent_get_first_component(event.m_calendar, ICAL_VEVENT_COMPONENT);
 834              comp;
 835              comp = icalcomponent_get_next_component(event.m_calendar, ICAL_VEVENT_COMPONENT)) {
 836             if (Event::getSubID(comp) == subid) {
 837                 eptr<icalcomponent> clone(icalcomponent_new_clone(comp), "VEVENT");
 838                 if (subid.empty()) {
 839                     parent = clone.get();
 840                 }
 841                 icalcomponent_add_component(calendar, clone.release());
 842                 found = true;
 843                 break;
 844             }
 845         }
 846
 847         if (!found) {
 848             SE_THROW("event not found");
 849         }
 850
 851         // tell engine and peers about EXDATEs implied by
 852         // RECURRENCE-IDs in detached recurrences by creating
 853         // X-SYNCEVOLUTION-EXDATE-DETACHED in the parent
 854         if (parent && event.m_subids.size() > 1) {
 855             // remove all old X-SYNCEVOLUTION-EXDATE-DETACHED (just in case)
 856             removeSyncEvolutionExdateDetached(parent);
 857
 858             // now populate with RECURRENCE-IDs of detached recurrences
 859             for (icalcomponent *comp = icalcomponent_get_first_component(event.m_calendar, ICAL_VEVENT_COMPONENT);
 860                  comp;
 861                  comp = icalcomponent_get_next_component(event.m_calendar, ICAL_VEVENT_COMPONENT)) {
 862                 icalproperty *prop = icalcomponent_get_first_property(comp, ICAL_RECURRENCEID_PROPERTY);
 863                 if (prop) {
 864                     eptr<char> rid(ical_strdup(icalproperty_get_value_as_string(prop)));
 865                     icalproperty *exdate = icalproperty_new_from_string(StringPrintf("X-SYNCEVOLUTION-EXDATE-DETACHED:%s", rid.get()).c_str());
 866                     if (exdate) {
 867                         icalparameter *tzid = icalproperty_get_first_parameter(prop, ICAL_TZID_PARAMETER);
 868                         if (tzid) {
 869                             icalproperty_add_parameter(exdate, icalparameter_new_clone(tzid));
 870                         }
 871 #if 0
 872                         // not needed
 873                         if (icalproperty_get_recurrenceid(exdate).is_date) {
 874                             icalproperty_add_parameter(exdate, icalparameter_new_value(ICAL_VALUE_DATE));
 875                         }
 876 #endif
 877                         icalcomponent_add_property(parent, exdate);
 878                     }
 879                 }
 880             }
 881         }
 882
 883         eptr<char> icalstr(ical_strdup(icalcomponent_as_ical_string(calendar)));
 884         item = icalstr.get();
 885     }
 886 }
 887
 888 void CalDAVSource::Event::escapeRecurrenceID(std::string &data)
 889 {
 890     boost::replace_all(data,
 891                        "\nRECURRENCE-ID",
 892                        "\nX-SYNCEVOLUTION-RECURRENCE-ID");
 893 }
 894
 895 void CalDAVSource::Event::unescapeRecurrenceID(std::string &data)
 896 {
 897     boost::replace_all(data,
 898                        "\nX-SYNCEVOLUTION-RECURRENCE-ID",
 899                        "\nRECURRENCE-ID");
 900 }
 901
 902 std::string CalDAVSource::removeSubItem(const string &davLUID, const std::string &subid)
 903 {
 904     EventCache::iterator it = m_cache.find(davLUID);
 905     if (it == m_cache.end()) {
 906         // gone already
 907         throwError(STATUS_NOT_FOUND, "deleting item: " + davLUID);
 908         return "";
 909     }
 910     // use item as it is, load only if it is not going to be removed entirely
 911     Event &event = *it->second;
 912
 913     if (event.m_subids.size() == 1) {
 914         // remove entire merged item, nothing will be left after removal
 915         if (*event.m_subids.begin() != subid) {
 916             SE_LOG_DEBUG(this, NULL, "%s: request to remove the %s recurrence: only the %s recurrence exists",
 917                          davLUID.c_str(),
 918                          SubIDName(subid).c_str(),
 919                          SubIDName(*event.m_subids.begin()).c_str());
 920             throwError(STATUS_NOT_FOUND, "remove sub-item: " + SubIDName(subid) + " in " + davLUID);
 921             return event.m_etag;
 922         } else {
 923             try {
 924                 removeItem(event.m_DAVluid);
 925             } catch (const TransportStatusException &ex) {
 926                 if (ex.syncMLStatus() == 409 &&
 927                     strstr(ex.what(), "Can't delete a recurring event")) {
 928                     // Google CalDAV:
 929                     // HTTP/1.1 409 Can't delete a recurring event except on its organizer's calendar
 930                     //
 931                     // Workaround: remove RRULE and EXDATE before deleting
 932                     bool updated = false;
 933                     icalcomponent *comp = icalcomponent_get_first_component(event.m_calendar, ICAL_VEVENT_COMPONENT);
 934                     if (comp) {
 935                         icalproperty *prop;
 936                         while ((prop = icalcomponent_get_first_property(comp, ICAL_RRULE_PROPERTY)) != NULL) {
 937                             icalcomponent_remove_property(comp, prop);
 938                             icalproperty_free(prop);
 939                             updated = true;
 940                         }
 941                         while ((prop = icalcomponent_get_first_property(comp, ICAL_EXDATE_PROPERTY)) != NULL) {
 942                             icalcomponent_remove_property(comp, prop);
 943                             icalproperty_free(prop);
 944                             updated = true;
 945                         }
 946                     }
 947                     if (updated) {
 948                         SE_LOG_DEBUG(this, NULL, "Google recurring event delete hack: remove RRULE before deleting");
 949                         eptr<char> icalstr(ical_strdup(icalcomponent_as_ical_string(event.m_calendar)));
 950                         insertSubItem(davLUID, subid, icalstr.get());
 951                         // It has been observed that trying the DELETE immediately
 952                         // failed again with the same "Can't delete a recurring event"
 953                         // error although the event no longer has an RRULE. Seems
 954                         // that the Google server sometimes need a bit of time until
 955                         // changes really trickle through all databases. Let's
 956                         // try a few times before giving up.
 957                         for (int retry = 0; retry < 5; retry++) {
 958                             try {
 959                                 SE_LOG_DEBUG(this, NULL, "Google recurring event delete hack: remove event, attempt #%d", retry);
 960                                 removeSubItem(davLUID, subid);
 961                                 break;
 962                             } catch (const TransportStatusException &ex2) {
 963                                 if (ex2.syncMLStatus() == 409 &&
 964                                     strstr(ex2.what(), "Can't delete a recurring event")) {
 965                                     SE_LOG_DEBUG(this, NULL, "Google recurring event delete hack: try again in a second");
 966                                     Sleep(1);
 967                                 } else {
 968                                     throw;
 969                                 }
 970                             }
 971                         }
 972                     } else {
 973                         SE_LOG_DEBUG(this, NULL, "Google recurring event delete hack not applicable, giving up");
 974                         throw;
 975                     }
 976                 } else {
 977                     throw;
 978                 }
 979             }
 980         }
 981         m_cache.erase(davLUID);
 982         return "";
 983     } else {
 984         loadItem(event);
 985         bool found = false;
 986         bool parentRemoved = false;
 987         for (icalcomponent *comp = icalcomponent_get_first_component(event.m_calendar, ICAL_VEVENT_COMPONENT);
 988              comp;
 989              comp = icalcomponent_get_next_component(event.m_calendar, ICAL_VEVENT_COMPONENT)) {
 990             if (Event::getSubID(comp) == subid) {
 991                 icalcomponent_remove_component(event.m_calendar, comp);
 992                 icalcomponent_free(comp);
 993                 found = true;
 994                 if (subid.empty()) {
 995                     parentRemoved = true;
 996                 }
 997             }
 998         }
 999         if (!found) {
1000             throwError(STATUS_NOT_FOUND, "remove sub-item: " + SubIDName(subid) + " in " + davLUID);
1001             return event.m_etag;
1002         }
1003         event.m_subids.erase(subid);
1004         // TODO: avoid updating the item immediately
1005         eptr<char> icalstr(ical_strdup(icalcomponent_as_ical_string(event.m_calendar)));
1006         InsertItemResult res;
1007         if (parentRemoved && settings().googleChildHack()) {
1008             // Must avoid VEVENTs with RECURRENCE-ID in
1009             // event.m_calendar and the PUT request.  Brute-force
1010             // approach here is to encode as string, escape, and parse
1011             // again.
1012             string item = icalstr.get();
1013             Event::escapeRecurrenceID(item);
1014             event.m_calendar.set(icalcomponent_new_from_string((char *)item.c_str()), // hack for old libical
1015                                  "parsing iCalendar 2.0");
1016             res = insertItem(davLUID, item, true);
1017         } else {
1018             res = insertItem(davLUID, icalstr.get(), true);
1019         }
1020         if (res.m_state != ITEM_OKAY ||
1021             res.m_luid != davLUID) {
1022             SE_THROW("unexpected result of removing sub event");
1023         }
1024         event.m_etag = res.m_revision;
1025         return event.m_etag;
1026     }
1027 }
1028
1029 void CalDAVSource::removeMergedItem(const std::string &davLUID)
1030 {
1031     EventCache::iterator it = m_cache.find(davLUID);
1032     if (it == m_cache.end()) {
1033         // gone already, no need to do anything
1034         SE_LOG_DEBUG(this, NULL, "%s: ignoring request to delete non-existent item",
1035                      davLUID.c_str());
1036         return;
1037     }
1038     // use item as it is, load only if it is not going to be removed entirely
1039     Event &event = *it->second;
1040
1041     // remove entire merged item, nothing will be left after removal
1042     try {
1043         removeItem(event.m_DAVluid);
1044     } catch (const TransportStatusException &ex) {
1045         if (ex.syncMLStatus() == 409 &&
1046             strstr(ex.what(), "Can't delete a recurring event")) {
1047             // Google CalDAV:
1048             // HTTP/1.1 409 Can't delete a recurring event except on its organizer's calendar
1049             //
1050             // Workaround: use the workarounds from removeSubItem()
1051             std::set<std::string> subids = event.m_subids;
1052             for (std::set<std::string>::reverse_iterator it = subids.rbegin();
1053                  it != subids.rend();
1054                  ++it) {
1055                 removeSubItem(davLUID, *it);
1056             }
1057         } else {
1058             throw;
1059         }
1060     }
1061
1062     m_cache.erase(davLUID);
1063 }
1064
1065 void CalDAVSource::flushItem(const string &davLUID)
1066 {
1067     // TODO: currently we always flush immediately, so no need to send data here
1068     EventCache::iterator it = m_cache.find(davLUID);
1069     if (it != m_cache.end()) {
1070         it->second->m_calendar.set(NULL);
1071     }
1072 }
1073
1074 std::string CalDAVSource::getSubDescription(const string &davLUID, const string &subid)
1075 {
1076     EventCache::iterator it = m_cache.find(davLUID);
1077     if (it == m_cache.end()) {
1078         // unknown item, return empty string for fallback
1079         return "";
1080     } else {
1081         return getSubDescription(*it->second, subid);
1082     }
1083 }
1084
1085 std::string CalDAVSource::getSubDescription(Event &event, const string &subid)
1086 {
1087     if (!event.m_calendar) {
1088         // Don't load (expensive!) only to provide the description.
1089         // Returning an empty string will trigger the fallback (logging the ID).
1090         return "";
1091     }
1092     for (icalcomponent *comp = icalcomponent_get_first_component(event.m_calendar, ICAL_VEVENT_COMPONENT);
1093          comp;
1094          comp = icalcomponent_get_next_component(event.m_calendar, ICAL_VEVENT_COMPONENT)) {
1095         if (Event::getSubID(comp) == subid) {
1096             std::string descr;
1097
1098             const char *summary = icalcomponent_get_summary(comp);
1099             if (summary && summary[0]) {
1100                 descr += summary;
1101             }
1102
1103             if (true /* is event */) {
1104                 const char *location = icalcomponent_get_location(comp);
1105                 if (location && location[0]) {
1106                     if (!descr.empty()) {
1107                         descr += ", ";
1108                     }
1109                     descr += location;
1110                 }
1111             }
1112             // TODO: other item types
1113             return descr;
1114         }
1115     }
1116     return "";
1117 }
1118
1119 std::string CalDAVSource::getDescription(const string &luid)
1120 {
1121     StringPair ids = MapSyncSource::splitLUID(luid);
1122     return getSubDescription(ids.first, ids.second);
1123 }
1124
1125 CalDAVSource::Event &CalDAVSource::findItem(const std::string &davLUID)
1126 {
1127     EventCache::iterator it = m_cache.find(davLUID);
1128     if (it == m_cache.end()) {
1129         throwError(STATUS_NOT_FOUND, "finding item: " + davLUID);
1130     }
1131     return *it->second;
1132 }
1133
1134 CalDAVSource::Event &CalDAVSource::loadItem(const std::string &davLUID)
1135 {
1136     Event &event = findItem(davLUID);
1137     return loadItem(event);
1138 }
1139
1140 int CalDAVSource::storeItem(const std::string &wantedLuid,
1141                             std::string &item,
1142                             std::string &data,
1143                             const std::string &href)
1144 {
1145     std::string luid = path2luid(Neon::URI::parse(href).m_path);
1146     if (luid == wantedLuid) {
1147         SE_LOG_DEBUG(NULL, NULL, "got item %s via REPORT fallback", luid.c_str());
1148         item = data;
1149     }
1150     data.clear();
1151     return 0;
1152 }
1153
1154 CalDAVSource::Event &CalDAVSource::loadItem(Event &event)
1155 {
1156     if (!event.m_calendar) {
1157         std::string item;
1158         try {
1159             readItem(event.m_DAVluid, item, true);
1160         } catch (const TransportStatusException &ex) {
1161             if (ex.syncMLStatus() == 404) {
1162                 // Someone must have created a detached recurrence on
1163                 // the server without the master event. We avoid that
1164                 // with the "Google Child Hack", but have no control
1165                 // over other clients. So let's deal with this problem
1166                 // after logging it.
1167                 Exception::log();
1168
1169                 // We know about the event because it showed up in a REPORT.
1170                 // So let's use such a REPORT to retrieve the desired item.
1171                 // Not as efficient as a GET (and thus not the default), but
1172                 // so be it.
1173 #if 0
1174                 // This would be fairly efficient, but runs into the same 404 error as a GET.
1175                 std::string query =
1176                     StringPrintf("<?xml version=\"1.0\" encoding=\"utf-8\"?>\n"
1177                                  "<C:calendar-multiget xmlns:D=\"DAV:\"\n"
1178                                  "   xmlns:C=\"urn:ietf:params:xml:ns:caldav\">\n"
1179                                  "<D:prop>\n"
1180                                  "   <C:calendar-data/>\n"
1181                                  "</D:prop>\n"
1182                                  "<D:href><[CDATA[%s]]></D:href>\n"
1183                                  "</C:calendar-multiget>",
1184                                  event.m_DAVluid.c_str());
1185                 Neon::XMLParser parser;
1186                 std::string href, etag;
1187                 item = "";
1188                 parser.initReportParser(href, etag);
1189                 parser.pushHandler(boost::bind(Neon::XMLParser::accept, "urn:ietf:params:xml:ns:caldav", "calendar-data", _2, _3),
1190                                    boost::bind(Neon::XMLParser::append, boost::ref(item), _2, _3));
1191                 Neon::Request report(*getSession(), "REPORT", getCalendar().m_path, query, parser);
1192                 report.addHeader("Content-Type", "application/xml; charset=\"utf-8\"");
1193                 report.run();
1194 #else
1195                 std::string query =
1196                     StringPrintf("<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n"
1197                                  "<C:calendar-query xmlns:D=\"DAV:\"\n"
1198                                  "xmlns:C=\"urn:ietf:params:xml:ns:caldav\">\n"
1199                                  "<D:prop>\n"
1200                                  "<D:getetag/>\n"
1201                                  "<C:calendar-data/>\n"
1202                                  "</D:prop>\n"
1203                                  // filter expected by Yahoo! Calendar
1204                                  "<C:filter>\n"
1205                                  "<C:comp-filter name=\"VCALENDAR\">\n"
1206                                  "<C:comp-filter name=\"VEVENT\">\n"
1207                                  "<C:prop-filter name=\"UID\">\n"
1208                                  "<C:text-match collation=\"i;octet\"><![CDATA[%s]]></C:text-match>\n"
1209                                  "</C:prop-filter>\n"
1210                                  "</C:comp-filter>\n"
1211                                  "</C:comp-filter>\n"
1212                                  "</C:filter>\n"
1213                                  "</C:calendar-query>\n",
1214                                  event.m_UID.c_str());
1215                 Timespec deadline = createDeadline();
1216                 getSession()->startOperation("REPORT 'single item'", deadline);
1217                 while (true) {
1218                     Neon::XMLParser parser;
1219                     std::string data;
1220                     parser.initReportParser(boost::bind(&CalDAVSource::storeItem,
1221                                                         this,
1222                                                         boost::ref(event.m_DAVluid),
1223                                                         boost::ref(item),
1224                                                         boost::ref(data),
1225                                                         _1));
1226                     parser.pushHandler(boost::bind(Neon::XMLParser::accept, "urn:ietf:params:xml:ns:caldav", "calendar-data", _2, _3),
1227                                        boost::bind(Neon::XMLParser::append, boost::ref(data), _2, _3));
1228                     Neon::Request report(*getSession(), "REPORT", getCalendar().m_path, query, parser);
1229                     report.addHeader("Depth", "1");
1230                     report.addHeader("Content-Type", "application/xml; charset=\"utf-8\"");
1231                     if (report.run()) {
1232                         break;
1233                     }
1234                 }
1235 #endif
1236             } else {
1237                 throw;
1238             }
1239         }
1240         Event::unescapeRecurrenceID(item);
1241         event.m_calendar.set(icalcomponent_new_from_string((char *)item.c_str()), // hack for old libical
1242                              "parsing iCalendar 2.0");
1243         Event::fixIncomingCalendar(event.m_calendar.get());
1244
1245         // Sequence number/last-modified might have been increased by last save.
1246         // Or the cache was populated by setAllSubItems(), which doesn't give
1247         // us the information. In that case, UID might also still be unknown.
1248         // Either way, check it again.
1249         for (icalcomponent *comp = icalcomponent_get_first_component(event.m_calendar, ICAL_VEVENT_COMPONENT);
1250              comp;
1251              comp = icalcomponent_get_next_component(event.m_calendar, ICAL_VEVENT_COMPONENT)) {
1252             if (event.m_UID.empty()) {
1253                 event.m_UID = Event::getUID(comp);
1254             }
1255             long sequence = Event::getSequence(comp);
1256             if (sequence > event.m_sequence) {
1257                 event.m_sequence = sequence;
1258             }
1259             icalproperty *lastmod = icalcomponent_get_first_property(comp, ICAL_LASTMODIFIED_PROPERTY);
1260             if (lastmod) {
1261                 icaltimetype lastmodtime = icalproperty_get_lastmodified(lastmod);
1262                 time_t mod = icaltime_as_timet(lastmodtime);
1263                 if (mod > event.m_lastmodtime) {
1264                     event.m_lastmodtime = mod;
1265                 }
1266             }
1267         }
1268     }
1269     return event;
1270 }
1271
1272 void CalDAVSource::Event::fixIncomingCalendar(icalcomponent *calendar)
1273 {
1274     // Evolution has a problem when the parent event uses a time
1275     // zone and the RECURRENCE-ID uses UTC (can happen in Exchange
1276     // meeting invitations): then Evolution and/or libical do not
1277     // recognize that the detached recurrence overrides the
1278     // regular recurrence and display both.
1279     //
1280     // As a workaround, remember time zone of DTSTART in parent event
1281     // in the first loop iteration. Then below transform the RECURRENCE-ID
1282     // time.
1283     bool ridInUTC = false;
1284     const icaltimezone *zone = NULL;
1285
1286     for (icalcomponent *comp = icalcomponent_get_first_component(calendar, ICAL_VEVENT_COMPONENT);
1287          comp;
1288          comp = icalcomponent_get_next_component(calendar, ICAL_VEVENT_COMPONENT)) {
1289         // remember whether we need to convert RECURRENCE-ID
1290         struct icaltimetype rid = icalcomponent_get_recurrenceid(comp);
1291         if (icaltime_is_utc(rid)) {
1292             ridInUTC = true;
1293         }
1294
1295         // is parent event? -> remember time zone unless it is UTC
1296         static const struct icaltimetype null = { 0 };
1297         if (!memcmp(&rid, &null, sizeof(null))) {
1298             struct icaltimetype dtstart = icalcomponent_get_dtstart(comp);
1299             if (!icaltime_is_utc(dtstart)) {
1300                 zone = icaltime_get_timezone(dtstart);
1301             }
1302         }
1303
1304         // remove useless X-LIC-ERROR
1305         icalproperty *prop = icalcomponent_get_first_property(comp, ICAL_ANY_PROPERTY);
1306         while (prop) {
1307             icalproperty *next = icalcomponent_get_next_property(comp, ICAL_ANY_PROPERTY);
1308             const char *name = icalproperty_get_property_name(prop);
1309             if (name && !strcmp("X-LIC-ERROR", name)) {
1310                 icalcomponent_remove_property(comp, prop);
1311                 icalproperty_free(prop);
1312             }
1313             prop = next;
1314         }
1315     }
1316
1317     // now update RECURRENCE-ID?
1318     if (zone && ridInUTC) {
1319         for (icalcomponent *comp = icalcomponent_get_first_component(calendar, ICAL_VEVENT_COMPONENT);
1320              comp;
1321              comp = icalcomponent_get_next_component(calendar, ICAL_VEVENT_COMPONENT)) {
1322             icalproperty *prop = icalcomponent_get_first_property(comp, ICAL_RECURRENCEID_PROPERTY);
1323             if (prop) {
1324                 struct icaltimetype rid = icalproperty_get_recurrenceid(prop);
1325                 if (icaltime_is_utc(rid)) {
1326                     rid = icaltime_convert_to_zone(rid, const_cast<icaltimezone *>(zone)); // icaltime_convert_to_zone should take a "const timezone" but doesn't
1327                     icalproperty_set_recurrenceid(prop, rid);
1328                     icalproperty_remove_parameter_by_kind(prop, ICAL_TZID_PARAMETER);
1329                     icalparameter *param = icalparameter_new_from_value_string(ICAL_TZID_PARAMETER,
1330                                                                                icaltimezone_get_tzid(const_cast<icaltimezone *>(zone)));
1331                     icalproperty_set_parameter(prop, param);
1332                 }
1333             }
1334         }
1335     }
1336 }
1337
1338 std::string CalDAVSource::Event::icalTime2Str(const icaltimetype &tt)
1339 {
1340     static const struct icaltimetype null = { 0 };
1341     if (!memcmp(&tt, &null, sizeof(null))) {
1342         return "";
1343     } else {
1344         eptr<char> timestr(ical_strdup(icaltime_as_ical_string(tt)));
1345         if (!timestr) {
1346             SE_THROW("cannot convert to time string");
1347         }
1348         return timestr.get();
1349     }
1350 }
1351
1352 std::string CalDAVSource::Event::getSubID(icalcomponent *comp)
1353 {
1354     struct icaltimetype rid = icalcomponent_get_recurrenceid(comp);
1355     return icalTime2Str(rid);
1356 }
1357
1358 std::string CalDAVSource::Event::getUID(icalcomponent *comp)
1359 {
1360     std::string uid;
1361     icalproperty *prop = icalcomponent_get_first_property(comp, ICAL_UID_PROPERTY);
1362     if (prop) {
1363         uid = icalproperty_get_uid(prop);
1364     }
1365     return uid;
1366 }
1367
1368 void CalDAVSource::Event::setUID(icalcomponent *comp, const std::string &uid)
1369 {
1370     icalproperty *prop = icalcomponent_get_first_property(comp, ICAL_UID_PROPERTY);
1371     if (prop) {
1372         icalproperty_set_uid(prop, uid.c_str());
1373     } else {
1374         icalcomponent_add_property(comp, icalproperty_new_uid(uid.c_str()));
1375     }
1376 }
1377
1378 int CalDAVSource::Event::getSequence(icalcomponent *comp)
1379 {
1380     int sequence = 0;
1381     icalproperty *prop = icalcomponent_get_first_property(comp, ICAL_SEQUENCE_PROPERTY);
1382     if (prop) {
1383         sequence = icalproperty_get_sequence(prop);
1384     }
1385     return sequence;
1386 }
1387
1388 void CalDAVSource::Event::setSequence(icalcomponent *comp, int sequence)
1389 {
1390     icalproperty *prop = icalcomponent_get_first_property(comp, ICAL_SEQUENCE_PROPERTY);
1391     if (prop) {
1392         icalproperty_set_sequence(prop, sequence);
1393     } else {
1394         icalcomponent_add_property(comp, icalproperty_new_sequence(sequence));
1395     }
1396 }
1397
1398 CalDAVSource::EventCache::iterator CalDAVSource::EventCache::findByUID(const std::string &uid)
1399 {
1400     for (iterator it = begin();
1401          it != end();
1402          ++it) {
1403         if (it->second->m_UID == uid) {
1404             return it;
1405         }
1406     }
1407     return end();
1408 }
1409
1410 void CalDAVSource::backupData(const SyncSource::Operations::ConstBackupInfo &oldBackup,
1411                               const SyncSource::Operations::BackupInfo &newBackup,
1412                               BackupReport &backupReport)
1413 {
1414     contactServer();
1415
1416     // If this runs as part of the sync preparations, then we might
1417     // use the result to populate our m_cache. But because dumping
1418     // data is typically disabled, this optimization isn't really
1419     // worth that much.
1420
1421     ItemCache cache;
1422     cache.init(oldBackup, newBackup, false);
1423
1424     // stream directly from REPORT with full data into backup
1425     const std::string query =
1426         "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n"
1427         "<C:calendar-query xmlns:D=\"DAV:\"\n"
1428         "xmlns:C=\"urn:ietf:params:xml:ns:caldav\">\n"
1429         "<D:prop>\n"
1430         "<D:getetag/>\n"
1431         "<C:calendar-data/>\n"
1432         "</D:prop>\n"
1433         // filter expected by Yahoo! Calendar
1434         "<C:filter>\n"
1435         "<C:comp-filter name=\"VCALENDAR\">\n"
1436         "<C:comp-filter name=\"VEVENT\">\n"
1437         "</C:comp-filter>\n"
1438         "</C:comp-filter>\n"
1439         "</C:filter>\n"
1440         "</C:calendar-query>\n";
1441     string data;
1442     Neon::XMLParser parser;
1443     parser.initReportParser(boost::bind(&CalDAVSource::backupItem, this,
1444                                         boost::ref(cache),
1445                                         _1, _2, boost::ref(data)));
1446     parser.pushHandler(boost::bind(Neon::XMLParser::accept, "urn:ietf:params:xml:ns:caldav", "calendar-data", _2, _3),
1447                        boost::bind(Neon::XMLParser::append, boost::ref(data), _2, _3));
1448     Timespec deadline = createDeadline();
1449     getSession()->startOperation("REPORT 'full calendar'", deadline);
1450     while (true) {
1451         Neon::Request report(*getSession(), "REPORT", getCalendar().m_path, query, parser);
1452         report.addHeader("Depth", "1");
1453         report.addHeader("Content-Type", "application/xml; charset=\"utf-8\"");
1454         if (report.run()) {
1455             break;
1456         }
1457         cache.reset();
1458     }
1459     cache.finalize(backupReport);
1460 }
1461
1462 int CalDAVSource::backupItem(ItemCache &cache,
1463                              const std::string &href,
1464                              const std::string &etag,
1465                              std::string &data)
1466 {
1467     // detect and ignore empty items, like we do in appendItem()
1468     eptr<icalcomponent> calendar(icalcomponent_new_from_string((char *)data.c_str()), // cast is a hack for broken definition in old libical
1469                                  "iCalendar 2.0");
1470     if (icalcomponent_get_first_component(calendar, ICAL_VEVENT_COMPONENT)) {
1471         Event::unescapeRecurrenceID(data);
1472         std::string luid = path2luid(Neon::URI::parse(href).m_path);
1473         std::string rev = ETag2Rev(etag);
1474         cache.backupItem(data, luid, rev);
1475     } else {
1476         SE_LOG_DEBUG(NULL, NULL, "ignoring broken item %s during backup (is empty)", href.c_str());
1477     }
1478
1479     // reset data for next item
1480     data.clear();
1481     return 0;
1482 }
1483
1484 void CalDAVSource::restoreData(const SyncSource::Operations::ConstBackupInfo &oldBackup,
1485                                bool dryrun,
1486                                SyncSourceReport &report)
1487 {
1488     // TODO: implement restore
1489     throw("not implemented");
1490 }
1491
1492 bool CalDAVSource::typeMatches(const StringMap &props) const
1493 {
1494     StringMap::const_iterator it = props.find("urn:ietf:params:xml:ns:caldav:supported-calendar-component-set");
1495     if (it != props.end() &&
1496         it->second.find("<urn:ietf:params:xml:ns:caldavcomp name='VEVENT'></urn:ietf:params:xml:ns:caldavcomp>") != std::string::npos) {
1497         return true;
1498     } else {
1499         return false;
1500     }
1501 }
1502
1503 SE_END_CXX
1504
1505 #endif // ENABLE_DAV