src/net/base/sdch_filter_unittest.cc

   1 // Copyright (c) 2011 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include <limits.h>
   6
   7 #include <algorithm>
   8 #include <string>
   9 #include <vector>
  10
  11 #include "base/logging.h"
  12 #include "base/memory/scoped_ptr.h"
  13 #include "net/base/filter.h"
  14 #include "net/base/io_buffer.h"
  15 #include "net/base/mock_filter_context.h"
  16 #include "net/base/sdch_filter.h"
  17 #include "net/url_request/url_request_http_job.h"
  18 #include "testing/gtest/include/gtest/gtest.h"
  19 #include "third_party/zlib/zlib.h"
  20
  21 namespace net {
  22
  23 //------------------------------------------------------------------------------
  24 // Provide sample data and compression results with a sample VCDIFF dictionary.
  25 // Note an SDCH dictionary has extra meta-data before the VCDIFF dictionary.
  26 static const char kTestVcdiffDictionary[] = "DictionaryFor"
  27     "SdchCompression1SdchCompression2SdchCompression3SdchCompression\n";
  28 // Pre-compression test data.  Note that we pad with a lot of highly gzip
  29 // compressible content to help to exercise the chaining pipeline.  That is why
  30 // there are a PILE of zeros at the start and end.
  31 // This will ensure that gzip compressed data can be fed to the chain in one
  32 // gulp, but (with careful selection of intermediate buffers) that it takes
  33 // several sdch buffers worth of data to satisfy the sdch filter.  See detailed
  34 // CHECK() calls in FilterChaining test for specifics.
  35 static const char kTestData[] = "0000000000000000000000000000000000000000000000"
  36     "0000000000000000000000000000TestData "
  37     "SdchCompression1SdchCompression2SdchCompression3SdchCompression"
  38     "00000000000000000000000000000000000000000000000000000000000000000000000000"
  39     "000000000000000000000000000000000000000\n";
  40
  41 // Note SDCH compressed data will include a reference to the SDCH dictionary.
  42 static const char kSdchCompressedTestData[] =
  43     "\326\303\304\0\0\001M\0\201S\202\004\0\201E\006\001"
  44     "00000000000000000000000000000000000000000000000000000000000000000000000000"
  45     "TestData 00000000000000000000000000000000000000000000000000000000000000000"
  46     "000000000000000000000000000000000000000000000000\n\001S\023\077\001r\r";
  47
  48 //------------------------------------------------------------------------------
  49
  50 class SdchFilterTest : public testing::Test {
  51  protected:
  52   SdchFilterTest()
  53     : test_vcdiff_dictionary_(kTestVcdiffDictionary,
  54                               sizeof(kTestVcdiffDictionary) - 1),
  55       vcdiff_compressed_data_(kSdchCompressedTestData,
  56                               sizeof(kSdchCompressedTestData) - 1),
  57       expanded_(kTestData, sizeof(kTestData) - 1),
  58       sdch_manager_(new SdchManager) {
  59   }
  60
  61   std::string NewSdchCompressedData(const std::string dictionary);
  62
  63   const std::string test_vcdiff_dictionary_;
  64   const std::string vcdiff_compressed_data_;
  65   const std::string expanded_;  // Desired final, decompressed data.
  66
  67   scoped_ptr<SdchManager> sdch_manager_;  // A singleton database.
  68 };
  69
  70 std::string SdchFilterTest::NewSdchCompressedData(
  71     const std::string dictionary) {
  72   std::string client_hash;
  73   std::string server_hash;
  74   SdchManager::GenerateHash(dictionary, &client_hash, &server_hash);
  75
  76   // Build compressed data that refers to our dictionary.
  77   std::string compressed(server_hash);
  78   compressed.append("\0", 1);
  79   compressed.append(vcdiff_compressed_data_);
  80   return compressed;
  81 }
  82
  83 //------------------------------------------------------------------------------
  84
  85
  86 TEST_F(SdchFilterTest, Hashing) {
  87   std::string client_hash, server_hash;
  88   std::string dictionary("test contents");
  89   SdchManager::GenerateHash(dictionary, &client_hash, &server_hash);
  90
  91   EXPECT_EQ(client_hash, "lMQBjS3P");
  92   EXPECT_EQ(server_hash, "MyciMVll");
  93 }
  94
  95
  96 //------------------------------------------------------------------------------
  97 // Provide a generic helper function for trying to filter data.
  98 // This function repeatedly calls the filter to process data, until the entire
  99 // source is consumed.  The return value from the filter is appended to output.
 100 // This allows us to vary input and output block sizes in order to test for edge
 101 // effects (boundary effects?) during the filtering process.
 102 // This function provides data to the filter in blocks of no-more-than the
 103 // specified input_block_length.  It allows the filter to fill no more than
 104 // output_buffer_length in any one call to proccess (a.k.a., Read) data, and
 105 // concatenates all these little output blocks into the singular output string.
 106 static bool FilterTestData(const std::string& source,
 107                            size_t input_block_length,
 108                            const size_t output_buffer_length,
 109                            Filter* filter, std::string* output) {
 110   CHECK_GT(input_block_length, 0u);
 111   Filter::FilterStatus status(Filter::FILTER_NEED_MORE_DATA);
 112   size_t source_index = 0;
 113   scoped_ptr<char[]> output_buffer(new char[output_buffer_length]);
 114   size_t input_amount = std::min(input_block_length,
 115       static_cast<size_t>(filter->stream_buffer_size()));
 116
 117   do {
 118     int copy_amount = std::min(input_amount, source.size() - source_index);
 119     if (copy_amount > 0 && status == Filter::FILTER_NEED_MORE_DATA) {
 120       memcpy(filter->stream_buffer()->data(), source.data() + source_index,
 121              copy_amount);
 122       filter->FlushStreamBuffer(copy_amount);
 123       source_index += copy_amount;
 124     }
 125     int buffer_length = output_buffer_length;
 126     status = filter->ReadData(output_buffer.get(), &buffer_length);
 127     output->append(output_buffer.get(), buffer_length);
 128     if (status == Filter::FILTER_ERROR)
 129       return false;
 130     // Callers assume that FILTER_OK with no output buffer means FILTER_DONE.
 131     if (Filter::FILTER_OK == status && 0 == buffer_length)
 132       return true;
 133     if (copy_amount == 0 && buffer_length == 0)
 134       return true;
 135   } while (1);
 136 }
 137 //------------------------------------------------------------------------------
 138 static std::string NewSdchDictionary(const std::string& domain) {
 139   std::string dictionary;
 140   if (!domain.empty()) {
 141     dictionary.append("Domain: ");
 142     dictionary.append(domain);
 143     dictionary.append("\n");
 144   }
 145   dictionary.append("\n");
 146   dictionary.append(kTestVcdiffDictionary, sizeof(kTestVcdiffDictionary) - 1);
 147   return dictionary;
 148 }
 149
 150 //------------------------------------------------------------------------------
 151
 152 TEST_F(SdchFilterTest, EmptyInputOk) {
 153   std::vector<Filter::FilterType> filter_types;
 154   filter_types.push_back(Filter::FILTER_TYPE_SDCH);
 155   char output_buffer[20];
 156   MockFilterContext filter_context;
 157   std::string url_string("http://ignore.com");
 158   filter_context.SetURL(GURL(url_string));
 159   scoped_ptr<Filter> filter(Filter::Factory(filter_types, filter_context));
 160
 161
 162   // With no input data, try to read output.
 163   int output_bytes_or_buffer_size = sizeof(output_buffer);
 164   Filter::FilterStatus status = filter->ReadData(output_buffer,
 165                                                  &output_bytes_or_buffer_size);
 166
 167   EXPECT_EQ(0, output_bytes_or_buffer_size);
 168   EXPECT_EQ(Filter::FILTER_NEED_MORE_DATA, status);
 169 }
 170
 171 TEST_F(SdchFilterTest, PassThroughWhenTentative) {
 172   std::vector<Filter::FilterType> filter_types;
 173   // Selective a tentative filter (which can fall back to pass through).
 174   filter_types.push_back(Filter::FILTER_TYPE_GZIP_HELPING_SDCH);
 175   char output_buffer[20];
 176   MockFilterContext filter_context;
 177   // Response code needs to be 200 to allow a pass through.
 178   filter_context.SetResponseCode(200);
 179   std::string url_string("http://ignore.com");
 180   filter_context.SetURL(GURL(url_string));
 181   scoped_ptr<Filter> filter(Filter::Factory(filter_types, filter_context));
 182
 183   // Supply enough data to force a pass-through mode..
 184   std::string non_gzip_content("not GZIPed data");
 185
 186   char* input_buffer = filter->stream_buffer()->data();
 187   int input_buffer_size = filter->stream_buffer_size();
 188
 189   EXPECT_LT(static_cast<int>(non_gzip_content.size()),
 190             input_buffer_size);
 191   memcpy(input_buffer, non_gzip_content.data(),
 192          non_gzip_content.size());
 193   filter->FlushStreamBuffer(non_gzip_content.size());
 194
 195   // Try to read output.
 196   int output_bytes_or_buffer_size = sizeof(output_buffer);
 197   Filter::FilterStatus status = filter->ReadData(output_buffer,
 198                                                  &output_bytes_or_buffer_size);
 199
 200   EXPECT_EQ(non_gzip_content.size(),
 201               static_cast<size_t>(output_bytes_or_buffer_size));
 202   ASSERT_GT(sizeof(output_buffer),
 203               static_cast<size_t>(output_bytes_or_buffer_size));
 204   output_buffer[output_bytes_or_buffer_size] = '\0';
 205   EXPECT_TRUE(non_gzip_content == output_buffer);
 206   EXPECT_EQ(Filter::FILTER_NEED_MORE_DATA, status);
 207 }
 208
 209 TEST_F(SdchFilterTest, RefreshBadReturnCode) {
 210   std::vector<Filter::FilterType> filter_types;
 211   // Selective a tentative filter (which can fall back to pass through).
 212   filter_types.push_back(Filter::FILTER_TYPE_SDCH_POSSIBLE);
 213   char output_buffer[20];
 214   MockFilterContext filter_context;
 215   // Response code needs to be 200 to allow a pass through.
 216   filter_context.SetResponseCode(403);
 217   // Meta refresh will only appear for html content
 218   filter_context.SetMimeType("text/html");
 219   std::string url_string("http://ignore.com");
 220   filter_context.SetURL(GURL(url_string));
 221   scoped_ptr<Filter> filter(Filter::Factory(filter_types, filter_context));
 222
 223   // Supply enough data to force a pass-through mode, which means we have
 224   // provided more than 9 characters that can't be a dictionary hash.
 225   std::string non_sdch_content("This is not SDCH");
 226
 227   char* input_buffer = filter->stream_buffer()->data();
 228   int input_buffer_size = filter->stream_buffer_size();
 229
 230   EXPECT_LT(static_cast<int>(non_sdch_content.size()),
 231             input_buffer_size);
 232   memcpy(input_buffer, non_sdch_content.data(),
 233          non_sdch_content.size());
 234   filter->FlushStreamBuffer(non_sdch_content.size());
 235
 236   // Try to read output.
 237   int output_bytes_or_buffer_size = sizeof(output_buffer);
 238   Filter::FilterStatus status = filter->ReadData(output_buffer,
 239                                                  &output_bytes_or_buffer_size);
 240
 241   // We should have read a long and complicated meta-refresh request.
 242   EXPECT_TRUE(sizeof(output_buffer) == output_bytes_or_buffer_size);
 243   // Check at least the prefix of the return.
 244   EXPECT_EQ(0, strncmp(output_buffer,
 245       "<head><META HTTP-EQUIV=\"Refresh\" CONTENT=\"0\"></head>",
 246       sizeof(output_buffer)));
 247   EXPECT_EQ(Filter::FILTER_OK, status);
 248 }
 249
 250 TEST_F(SdchFilterTest, ErrorOnBadReturnCode) {
 251   std::vector<Filter::FilterType> filter_types;
 252   // Selective a tentative filter (which can fall back to pass through).
 253   filter_types.push_back(Filter::FILTER_TYPE_SDCH_POSSIBLE);
 254   char output_buffer[20];
 255   MockFilterContext filter_context;
 256   // Response code needs to be 200 to allow a pass through.
 257   filter_context.SetResponseCode(403);
 258   // Meta refresh will only appear for html content, so set to something else
 259   // to induce an error (we can't meta refresh).
 260   filter_context.SetMimeType("anything");
 261   std::string url_string("http://ignore.com");
 262   filter_context.SetURL(GURL(url_string));
 263   scoped_ptr<Filter> filter(Filter::Factory(filter_types, filter_context));
 264
 265   // Supply enough data to force a pass-through mode, which means we have
 266   // provided more than 9 characters that can't be a dictionary hash.
 267   std::string non_sdch_content("This is not SDCH");
 268
 269   char* input_buffer = filter->stream_buffer()->data();
 270   int input_buffer_size = filter->stream_buffer_size();
 271
 272   EXPECT_LT(static_cast<int>(non_sdch_content.size()),
 273             input_buffer_size);
 274   memcpy(input_buffer, non_sdch_content.data(),
 275          non_sdch_content.size());
 276   filter->FlushStreamBuffer(non_sdch_content.size());
 277
 278   // Try to read output.
 279   int output_bytes_or_buffer_size = sizeof(output_buffer);
 280   Filter::FilterStatus status = filter->ReadData(output_buffer,
 281                                                  &output_bytes_or_buffer_size);
 282
 283   EXPECT_EQ(0, output_bytes_or_buffer_size);
 284   EXPECT_EQ(Filter::FILTER_ERROR, status);
 285 }
 286
 287 TEST_F(SdchFilterTest, ErrorOnBadReturnCodeWithHtml) {
 288   std::vector<Filter::FilterType> filter_types;
 289   // Selective a tentative filter (which can fall back to pass through).
 290   filter_types.push_back(Filter::FILTER_TYPE_SDCH_POSSIBLE);
 291   char output_buffer[20];
 292   MockFilterContext filter_context;
 293   // Response code needs to be 200 to allow a pass through.
 294   filter_context.SetResponseCode(403);
 295   // Meta refresh will only appear for html content
 296   filter_context.SetMimeType("text/html");
 297   std::string url_string("http://ignore.com");
 298   filter_context.SetURL(GURL(url_string));
 299   scoped_ptr<Filter> filter(Filter::Factory(filter_types, filter_context));
 300
 301   // Supply enough data to force a pass-through mode, which means we have
 302   // provided more than 9 characters that can't be a dictionary hash.
 303   std::string non_sdch_content("This is not SDCH");
 304
 305   char* input_buffer = filter->stream_buffer()->data();
 306   int input_buffer_size = filter->stream_buffer_size();
 307
 308   EXPECT_LT(static_cast<int>(non_sdch_content.size()),
 309             input_buffer_size);
 310   memcpy(input_buffer, non_sdch_content.data(),
 311          non_sdch_content.size());
 312   filter->FlushStreamBuffer(non_sdch_content.size());
 313
 314   // Try to read output.
 315   int output_bytes_or_buffer_size = sizeof(output_buffer);
 316   Filter::FilterStatus status = filter->ReadData(output_buffer,
 317                                                  &output_bytes_or_buffer_size);
 318
 319   // We should have read a long and complicated meta-refresh request.
 320   EXPECT_EQ(sizeof(output_buffer),
 321             static_cast<size_t>(output_bytes_or_buffer_size));
 322   // Check at least the prefix of the return.
 323   EXPECT_EQ(0, strncmp(output_buffer,
 324       "<head><META HTTP-EQUIV=\"Refresh\" CONTENT=\"0\"></head>",
 325       sizeof(output_buffer)));
 326   EXPECT_EQ(Filter::FILTER_OK, status);
 327 }
 328
 329 TEST_F(SdchFilterTest, BasicBadDictionary) {
 330   std::vector<Filter::FilterType> filter_types;
 331   filter_types.push_back(Filter::FILTER_TYPE_SDCH);
 332   char output_buffer[20];
 333   MockFilterContext filter_context;
 334   std::string url_string("http://ignore.com");
 335   filter_context.SetURL(GURL(url_string));
 336   scoped_ptr<Filter> filter(Filter::Factory(filter_types, filter_context));
 337
 338   // Supply bogus data (which doesn't yet specify a full dictionary hash).
 339   // Dictionary hash is 8 characters followed by a null.
 340   std::string dictionary_hash_prefix("123");
 341
 342   char* input_buffer = filter->stream_buffer()->data();
 343   int input_buffer_size = filter->stream_buffer_size();
 344
 345   EXPECT_LT(static_cast<int>(dictionary_hash_prefix.size()),
 346             input_buffer_size);
 347   memcpy(input_buffer, dictionary_hash_prefix.data(),
 348          dictionary_hash_prefix.size());
 349   filter->FlushStreamBuffer(dictionary_hash_prefix.size());
 350
 351   // With less than a dictionary specifier, try to read output.
 352   int output_bytes_or_buffer_size = sizeof(output_buffer);
 353   Filter::FilterStatus status = filter->ReadData(output_buffer,
 354                                                  &output_bytes_or_buffer_size);
 355
 356   EXPECT_EQ(0, output_bytes_or_buffer_size);
 357   EXPECT_EQ(Filter::FILTER_NEED_MORE_DATA, status);
 358
 359   // Provide enough data to complete *a* hash, but it is bogus, and not in our
 360   // list of dictionaries, so the filter should error out immediately.
 361   std::string dictionary_hash_postfix("4abcd\0", 6);
 362
 363   CHECK_LT(dictionary_hash_postfix.size(),
 364            static_cast<size_t>(input_buffer_size));
 365   memcpy(input_buffer, dictionary_hash_postfix.data(),
 366          dictionary_hash_postfix.size());
 367   filter->FlushStreamBuffer(dictionary_hash_postfix.size());
 368
 369   // With a non-existant dictionary specifier, try to read output.
 370   output_bytes_or_buffer_size = sizeof(output_buffer);
 371   status = filter->ReadData(output_buffer, &output_bytes_or_buffer_size);
 372
 373   EXPECT_EQ(0, output_bytes_or_buffer_size);
 374   EXPECT_EQ(Filter::FILTER_ERROR, status);
 375
 376   EXPECT_FALSE(SdchManager::Global()->IsInSupportedDomain(GURL(url_string)));
 377   SdchManager::ClearBlacklistings();
 378   EXPECT_TRUE(SdchManager::Global()->IsInSupportedDomain(GURL(url_string)));
 379 }
 380
 381 TEST_F(SdchFilterTest, DictionaryAddOnce) {
 382   // Construct a valid SDCH dictionary from a VCDIFF dictionary.
 383   const std::string kSampleDomain = "sdchtest.com";
 384   std::string dictionary(NewSdchDictionary(kSampleDomain));
 385
 386   std::string url_string = "http://" + kSampleDomain;
 387   GURL url(url_string);
 388   EXPECT_TRUE(sdch_manager_->AddSdchDictionary(dictionary, url));
 389
 390   // Check we can't add it twice.
 391   EXPECT_FALSE(sdch_manager_->AddSdchDictionary(dictionary, url));
 392
 393   const std::string kSampleDomain2 = "sdchtest2.com";
 394
 395   // Construct a second SDCH dictionary from a VCDIFF dictionary.
 396   std::string dictionary2(NewSdchDictionary(kSampleDomain2));
 397
 398   std::string url_string2 = "http://" + kSampleDomain2;
 399   GURL url2(url_string2);
 400   EXPECT_TRUE(sdch_manager_->AddSdchDictionary(dictionary2, url2));
 401 }
 402
 403 TEST_F(SdchFilterTest, BasicDictionary) {
 404   // Construct a valid SDCH dictionary from a VCDIFF dictionary.
 405   const std::string kSampleDomain = "sdchtest.com";
 406   std::string dictionary(NewSdchDictionary(kSampleDomain));
 407
 408   std::string url_string = "http://" + kSampleDomain;
 409
 410   GURL url(url_string);
 411   EXPECT_TRUE(sdch_manager_->AddSdchDictionary(dictionary, url));
 412
 413   std::string compressed(NewSdchCompressedData(dictionary));
 414
 415   std::vector<Filter::FilterType> filter_types;
 416   filter_types.push_back(Filter::FILTER_TYPE_SDCH);
 417
 418   MockFilterContext filter_context;
 419   filter_context.SetURL(url);
 420
 421   scoped_ptr<Filter> filter(Filter::Factory(filter_types, filter_context));
 422
 423   size_t feed_block_size = 100;
 424   size_t output_block_size = 100;
 425   std::string output;
 426   EXPECT_TRUE(FilterTestData(compressed, feed_block_size, output_block_size,
 427                              filter.get(), &output));
 428   EXPECT_EQ(output, expanded_);
 429
 430   // Decode with really small buffers (size 1) to check for edge effects.
 431   filter.reset(Filter::Factory(filter_types, filter_context));
 432
 433   feed_block_size = 1;
 434   output_block_size = 1;
 435   output.clear();
 436   EXPECT_TRUE(FilterTestData(compressed, feed_block_size, output_block_size,
 437                              filter.get(), &output));
 438   EXPECT_EQ(output, expanded_);
 439 }
 440
 441 TEST_F(SdchFilterTest, NoDecodeHttps) {
 442   // Construct a valid SDCH dictionary from a VCDIFF dictionary.
 443   const std::string kSampleDomain = "sdchtest.com";
 444   std::string dictionary(NewSdchDictionary(kSampleDomain));
 445
 446   std::string url_string = "http://" + kSampleDomain;
 447
 448   GURL url(url_string);
 449   EXPECT_TRUE(sdch_manager_->AddSdchDictionary(dictionary, url));
 450
 451   std::string compressed(NewSdchCompressedData(dictionary));
 452
 453   std::vector<Filter::FilterType> filter_types;
 454   filter_types.push_back(Filter::FILTER_TYPE_SDCH);
 455
 456   MockFilterContext filter_context;
 457   filter_context.SetURL(GURL("https://" + kSampleDomain));
 458   scoped_ptr<Filter> filter(Filter::Factory(filter_types, filter_context));
 459
 460   const size_t feed_block_size(100);
 461   const size_t output_block_size(100);
 462   std::string output;
 463
 464   EXPECT_FALSE(FilterTestData(compressed, feed_block_size, output_block_size,
 465                              filter.get(), &output));
 466 }
 467
 468 // Current failsafe TODO/hack refuses to decode any content that doesn't use
 469 // http as the scheme (see use of DICTIONARY_SELECTED_FOR_NON_HTTP).
 470 // The following tests this blockage.  Note that blacklisting results, so we
 471 // we need separate tests for each of these.
 472 TEST_F(SdchFilterTest, NoDecodeFtp) {
 473   // Construct a valid SDCH dictionary from a VCDIFF dictionary.
 474   const std::string kSampleDomain = "sdchtest.com";
 475   std::string dictionary(NewSdchDictionary(kSampleDomain));
 476
 477   std::string url_string = "http://" + kSampleDomain;
 478
 479   GURL url(url_string);
 480   EXPECT_TRUE(sdch_manager_->AddSdchDictionary(dictionary, url));
 481
 482   std::string compressed(NewSdchCompressedData(dictionary));
 483
 484   std::vector<Filter::FilterType> filter_types;
 485   filter_types.push_back(Filter::FILTER_TYPE_SDCH);
 486
 487   MockFilterContext filter_context;
 488   filter_context.SetURL(GURL("ftp://" + kSampleDomain));
 489   scoped_ptr<Filter> filter(Filter::Factory(filter_types, filter_context));
 490
 491   const size_t feed_block_size(100);
 492   const size_t output_block_size(100);
 493   std::string output;
 494
 495   EXPECT_FALSE(FilterTestData(compressed, feed_block_size, output_block_size,
 496                              filter.get(), &output));
 497 }
 498
 499 TEST_F(SdchFilterTest, NoDecodeFileColon) {
 500   // Construct a valid SDCH dictionary from a VCDIFF dictionary.
 501   const std::string kSampleDomain = "sdchtest.com";
 502   std::string dictionary(NewSdchDictionary(kSampleDomain));
 503
 504   std::string url_string = "http://" + kSampleDomain;
 505
 506   GURL url(url_string);
 507   EXPECT_TRUE(sdch_manager_->AddSdchDictionary(dictionary, url));
 508
 509   std::string compressed(NewSdchCompressedData(dictionary));
 510
 511   std::vector<Filter::FilterType> filter_types;
 512   filter_types.push_back(Filter::FILTER_TYPE_SDCH);
 513
 514   MockFilterContext filter_context;
 515   filter_context.SetURL(GURL("file://" + kSampleDomain));
 516   scoped_ptr<Filter> filter(Filter::Factory(filter_types, filter_context));
 517
 518   const size_t feed_block_size(100);
 519   const size_t output_block_size(100);
 520   std::string output;
 521
 522   EXPECT_FALSE(FilterTestData(compressed, feed_block_size, output_block_size,
 523                              filter.get(), &output));
 524 }
 525
 526 TEST_F(SdchFilterTest, NoDecodeAboutColon) {
 527   // Construct a valid SDCH dictionary from a VCDIFF dictionary.
 528   const std::string kSampleDomain = "sdchtest.com";
 529   std::string dictionary(NewSdchDictionary(kSampleDomain));
 530
 531   std::string url_string = "http://" + kSampleDomain;
 532
 533   GURL url(url_string);
 534   EXPECT_TRUE(sdch_manager_->AddSdchDictionary(dictionary, url));
 535
 536   std::string compressed(NewSdchCompressedData(dictionary));
 537
 538   std::vector<Filter::FilterType> filter_types;
 539   filter_types.push_back(Filter::FILTER_TYPE_SDCH);
 540
 541   MockFilterContext filter_context;
 542   filter_context.SetURL(GURL("about://" + kSampleDomain));
 543   scoped_ptr<Filter> filter(Filter::Factory(filter_types, filter_context));
 544
 545   const size_t feed_block_size(100);
 546   const size_t output_block_size(100);
 547   std::string output;
 548
 549   EXPECT_FALSE(FilterTestData(compressed, feed_block_size, output_block_size,
 550                              filter.get(), &output));
 551 }
 552
 553 TEST_F(SdchFilterTest, NoDecodeJavaScript) {
 554   // Construct a valid SDCH dictionary from a VCDIFF dictionary.
 555   const std::string kSampleDomain = "sdchtest.com";
 556   std::string dictionary(NewSdchDictionary(kSampleDomain));
 557
 558   std::string url_string = "http://" + kSampleDomain;
 559
 560   GURL url(url_string);
 561   EXPECT_TRUE(sdch_manager_->AddSdchDictionary(dictionary, url));
 562
 563   std::string compressed(NewSdchCompressedData(dictionary));
 564
 565   std::vector<Filter::FilterType> filter_types;
 566   filter_types.push_back(Filter::FILTER_TYPE_SDCH);
 567
 568   MockFilterContext filter_context;
 569   filter_context.SetURL(GURL("javascript://" + kSampleDomain));
 570   scoped_ptr<Filter> filter(Filter::Factory(filter_types, filter_context));
 571
 572   const size_t feed_block_size(100);
 573   const size_t output_block_size(100);
 574   std::string output;
 575
 576   EXPECT_FALSE(FilterTestData(compressed, feed_block_size, output_block_size,
 577                              filter.get(), &output));
 578 }
 579
 580 TEST_F(SdchFilterTest, CanStillDecodeHttp) {
 581   // Construct a valid SDCH dictionary from a VCDIFF dictionary.
 582   const std::string kSampleDomain = "sdchtest.com";
 583   std::string dictionary(NewSdchDictionary(kSampleDomain));
 584
 585   std::string url_string = "http://" + kSampleDomain;
 586
 587   GURL url(url_string);
 588   EXPECT_TRUE(sdch_manager_->AddSdchDictionary(dictionary, url));
 589
 590   std::string compressed(NewSdchCompressedData(dictionary));
 591
 592   std::vector<Filter::FilterType> filter_types;
 593   filter_types.push_back(Filter::FILTER_TYPE_SDCH);
 594
 595   MockFilterContext filter_context;
 596   filter_context.SetURL(GURL("http://" + kSampleDomain));
 597   scoped_ptr<Filter> filter(Filter::Factory(filter_types, filter_context));
 598
 599   const size_t feed_block_size(100);
 600   const size_t output_block_size(100);
 601   std::string output;
 602
 603   EXPECT_TRUE(FilterTestData(compressed, feed_block_size, output_block_size,
 604                              filter.get(), &output));
 605 }
 606
 607 TEST_F(SdchFilterTest, CrossDomainDictionaryUse) {
 608   // Construct a valid SDCH dictionary from a VCDIFF dictionary.
 609   const std::string kSampleDomain = "sdchtest.com";
 610   std::string dictionary(NewSdchDictionary(kSampleDomain));
 611
 612   std::string url_string = "http://" + kSampleDomain;
 613
 614   GURL url(url_string);
 615   EXPECT_TRUE(sdch_manager_->AddSdchDictionary(dictionary, url));
 616
 617   std::string compressed(NewSdchCompressedData(dictionary));
 618
 619   std::vector<Filter::FilterType> filter_types;
 620   filter_types.push_back(Filter::FILTER_TYPE_SDCH);
 621
 622   // Decode with content arriving from the "wrong" domain.
 623   // This tests SdchManager::CanSet().
 624   MockFilterContext filter_context;
 625   GURL wrong_domain_url("http://www.wrongdomain.com");
 626   filter_context.SetURL(wrong_domain_url);
 627   scoped_ptr<Filter> filter(Filter::Factory(filter_types,  filter_context));
 628
 629   size_t feed_block_size = 100;
 630   size_t output_block_size = 100;
 631   std::string output;
 632   EXPECT_FALSE(FilterTestData(compressed, feed_block_size, output_block_size,
 633                               filter.get(), &output));
 634   EXPECT_EQ(output.size(), 0u);  // No output written.
 635
 636   EXPECT_TRUE(SdchManager::Global()->IsInSupportedDomain(GURL(url_string)));
 637   EXPECT_FALSE(SdchManager::Global()->IsInSupportedDomain(wrong_domain_url));
 638   SdchManager::ClearBlacklistings();
 639   EXPECT_TRUE(SdchManager::Global()->IsInSupportedDomain(wrong_domain_url));
 640 }
 641
 642 TEST_F(SdchFilterTest, DictionaryPathValidation) {
 643   // Construct a valid SDCH dictionary from a VCDIFF dictionary.
 644   const std::string kSampleDomain = "sdchtest.com";
 645   std::string dictionary(NewSdchDictionary(kSampleDomain));
 646
 647   std::string url_string = "http://" + kSampleDomain;
 648
 649   GURL url(url_string);
 650   EXPECT_TRUE(sdch_manager_->AddSdchDictionary(dictionary, url));
 651
 652   // Create a dictionary with a path restriction, by prefixing dictionary.
 653   const std::string path("/special_path/bin");
 654   std::string dictionary_with_path("Path: " + path + "\n");
 655   dictionary_with_path.append(dictionary);
 656   EXPECT_TRUE(sdch_manager_->AddSdchDictionary(dictionary_with_path, url));
 657
 658   std::string compressed_for_path(NewSdchCompressedData(dictionary_with_path));
 659
 660   std::vector<Filter::FilterType> filter_types;
 661   filter_types.push_back(Filter::FILTER_TYPE_SDCH);
 662
 663   // Test decode the path data, arriving from a valid path.
 664   MockFilterContext filter_context;
 665   filter_context.SetURL(GURL(url_string + path));
 666   scoped_ptr<Filter> filter(Filter::Factory(filter_types, filter_context));
 667
 668   size_t feed_block_size = 100;
 669   size_t output_block_size = 100;
 670   std::string output;
 671
 672   EXPECT_TRUE(FilterTestData(compressed_for_path, feed_block_size,
 673                              output_block_size, filter.get(), &output));
 674   EXPECT_EQ(output, expanded_);
 675
 676   // Test decode the path data, arriving from a invalid path.
 677   filter_context.SetURL(GURL(url_string));
 678   filter.reset(Filter::Factory(filter_types, filter_context));
 679
 680   feed_block_size = 100;
 681   output_block_size = 100;
 682   output.clear();
 683   EXPECT_FALSE(FilterTestData(compressed_for_path, feed_block_size,
 684                               output_block_size, filter.get(), &output));
 685   EXPECT_EQ(output.size(), 0u);  // No output written.
 686
 687   EXPECT_FALSE(SdchManager::Global()->IsInSupportedDomain(GURL(url_string)));
 688   SdchManager::ClearBlacklistings();
 689   EXPECT_TRUE(SdchManager::Global()->IsInSupportedDomain(GURL(url_string)));
 690 }
 691
 692 TEST_F(SdchFilterTest, DictionaryPortValidation) {
 693   // Construct a valid SDCH dictionary from a VCDIFF dictionary.
 694   const std::string kSampleDomain = "sdchtest.com";
 695   std::string dictionary(NewSdchDictionary(kSampleDomain));
 696
 697   std::string url_string = "http://" + kSampleDomain;
 698
 699   GURL url(url_string);
 700   EXPECT_TRUE(sdch_manager_->AddSdchDictionary(dictionary, url));
 701
 702
 703   // Create a dictionary with a port restriction, by prefixing old dictionary.
 704   const std::string port("502");
 705   std::string dictionary_with_port("Port: " + port + "\n");
 706   dictionary_with_port.append("Port: 80\n");  // Add default port.
 707   dictionary_with_port.append(dictionary);
 708   EXPECT_TRUE(sdch_manager_->AddSdchDictionary(dictionary_with_port,
 709                                             GURL(url_string + ":" + port)));
 710
 711   std::string compressed_for_port(NewSdchCompressedData(dictionary_with_port));
 712
 713   std::vector<Filter::FilterType> filter_types;
 714   filter_types.push_back(Filter::FILTER_TYPE_SDCH);
 715
 716   // Test decode the port data, arriving from a valid port.
 717   MockFilterContext filter_context;
 718   filter_context.SetURL(GURL(url_string + ":" + port));
 719   scoped_ptr<Filter> filter(Filter::Factory(filter_types, filter_context));
 720
 721   size_t feed_block_size = 100;
 722   size_t output_block_size = 100;
 723   std::string output;
 724   EXPECT_TRUE(FilterTestData(compressed_for_port, feed_block_size,
 725                              output_block_size, filter.get(), &output));
 726   EXPECT_EQ(output, expanded_);
 727
 728   // Test decode the port data, arriving from a valid (default) port.
 729   filter_context.SetURL(GURL(url_string));  // Default port.
 730   filter.reset(Filter::Factory(filter_types, filter_context));
 731
 732   feed_block_size = 100;
 733   output_block_size = 100;
 734   output.clear();
 735   EXPECT_TRUE(FilterTestData(compressed_for_port, feed_block_size,
 736                              output_block_size, filter.get(), &output));
 737   EXPECT_EQ(output, expanded_);
 738
 739   // Test decode the port data, arriving from a invalid port.
 740   filter_context.SetURL(GURL(url_string + ":" + port + "1"));
 741   filter.reset(Filter::Factory(filter_types, filter_context));
 742
 743   feed_block_size = 100;
 744   output_block_size = 100;
 745   output.clear();
 746   EXPECT_FALSE(FilterTestData(compressed_for_port, feed_block_size,
 747                               output_block_size, filter.get(), &output));
 748   EXPECT_EQ(output.size(), 0u);  // No output written.
 749
 750   EXPECT_FALSE(SdchManager::Global()->IsInSupportedDomain(GURL(url_string)));
 751   SdchManager::ClearBlacklistings();
 752   EXPECT_TRUE(SdchManager::Global()->IsInSupportedDomain(GURL(url_string)));
 753 }
 754
 755 //------------------------------------------------------------------------------
 756 // Helper function to perform gzip compression of data.
 757
 758 static std::string gzip_compress(const std::string &input) {
 759   z_stream zlib_stream;
 760   memset(&zlib_stream, 0, sizeof(zlib_stream));
 761   int code;
 762
 763   // Initialize zlib
 764   code = deflateInit2(&zlib_stream, Z_DEFAULT_COMPRESSION, Z_DEFLATED,
 765                       -MAX_WBITS,
 766                       8,  // DEF_MEM_LEVEL
 767                       Z_DEFAULT_STRATEGY);
 768
 769   CHECK_EQ(Z_OK, code);
 770
 771   // Fill in zlib control block
 772   zlib_stream.next_in = bit_cast<Bytef*>(input.data());
 773   zlib_stream.avail_in = input.size();
 774
 775   // Assume we can compress into similar buffer (add 100 bytes to be sure).
 776   size_t gzip_compressed_length = zlib_stream.avail_in + 100;
 777   scoped_ptr<char[]> gzip_compressed(new char[gzip_compressed_length]);
 778   zlib_stream.next_out = bit_cast<Bytef*>(gzip_compressed.get());
 779   zlib_stream.avail_out = gzip_compressed_length;
 780
 781   // The GZIP header (see RFC 1952):
 782   //   +---+---+---+---+---+---+---+---+---+---+
 783   //   |ID1|ID2|CM |FLG|     MTIME     |XFL|OS |
 784   //   +---+---+---+---+---+---+---+---+---+---+
 785   //     ID1     \037
 786   //     ID2     \213
 787   //     CM      \010 (compression method == DEFLATE)
 788   //     FLG     \000 (special flags that we do not support)
 789   //     MTIME   Unix format modification time (0 means not available)
 790   //     XFL     2-4? DEFLATE flags
 791   //     OS      ???? Operating system indicator (255 means unknown)
 792   //
 793   // Header value we generate:
 794   const char kGZipHeader[] = { '\037', '\213', '\010', '\000', '\000',
 795                                '\000', '\000', '\000', '\002', '\377' };
 796   CHECK_GT(zlib_stream.avail_out, sizeof(kGZipHeader));
 797   memcpy(zlib_stream.next_out, kGZipHeader, sizeof(kGZipHeader));
 798   zlib_stream.next_out += sizeof(kGZipHeader);
 799   zlib_stream.avail_out -= sizeof(kGZipHeader);
 800
 801   // Do deflate
 802   code = deflate(&zlib_stream, Z_FINISH);
 803   gzip_compressed_length -= zlib_stream.avail_out;
 804   std::string compressed(gzip_compressed.get(), gzip_compressed_length);
 805   deflateEnd(&zlib_stream);
 806   return compressed;
 807 }
 808
 809 //------------------------------------------------------------------------------
 810
 811 class SdchFilterChainingTest {
 812  public:
 813   static Filter* Factory(const std::vector<Filter::FilterType>& types,
 814                            const FilterContext& context, int size) {
 815     return Filter::FactoryForTests(types, context, size);
 816   }
 817 };
 818
 819 // Test that filters can be cascaded (chained) so that the output of one filter
 820 // is processed by the next one.  This is most critical for SDCH, which is
 821 // routinely followed by gzip (during encoding).  The filter we'll test for will
 822 // do the gzip decoding first, and then decode the SDCH content.
 823 TEST_F(SdchFilterTest, FilterChaining) {
 824   // Construct a valid SDCH dictionary from a VCDIFF dictionary.
 825   const std::string kSampleDomain = "sdchtest.com";
 826   std::string dictionary(NewSdchDictionary(kSampleDomain));
 827
 828   std::string url_string = "http://" + kSampleDomain;
 829
 830   GURL url(url_string);
 831   EXPECT_TRUE(sdch_manager_->AddSdchDictionary(dictionary, url));
 832
 833   std::string sdch_compressed(NewSdchCompressedData(dictionary));
 834
 835   // Use Gzip to compress the sdch sdch_compressed data.
 836   std::string gzip_compressed_sdch = gzip_compress(sdch_compressed);
 837
 838   // Construct a chained filter.
 839   std::vector<Filter::FilterType> filter_types;
 840   filter_types.push_back(Filter::FILTER_TYPE_SDCH);
 841   filter_types.push_back(Filter::FILTER_TYPE_GZIP);
 842
 843   // First try with a large buffer (larger than test input, or compressed data).
 844   const size_t kLargeInputBufferSize(1000);  // Used internally in filters.
 845   CHECK_GT(kLargeInputBufferSize, gzip_compressed_sdch.size());
 846   CHECK_GT(kLargeInputBufferSize, sdch_compressed.size());
 847   CHECK_GT(kLargeInputBufferSize, expanded_.size());
 848   MockFilterContext filter_context;
 849   filter_context.SetURL(url);
 850   scoped_ptr<Filter> filter(
 851       SdchFilterChainingTest::Factory(filter_types, filter_context,
 852                                       kLargeInputBufferSize));
 853   EXPECT_EQ(static_cast<int>(kLargeInputBufferSize),
 854             filter->stream_buffer_size());
 855
 856   // Verify that chained filter is waiting for data.
 857   char tiny_output_buffer[10];
 858   int tiny_output_size = sizeof(tiny_output_buffer);
 859   EXPECT_EQ(Filter::FILTER_NEED_MORE_DATA,
 860             filter->ReadData(tiny_output_buffer, &tiny_output_size));
 861
 862   // Make chain process all data.
 863   size_t feed_block_size = kLargeInputBufferSize;
 864   size_t output_block_size = kLargeInputBufferSize;
 865   std::string output;
 866   EXPECT_TRUE(FilterTestData(gzip_compressed_sdch, feed_block_size,
 867                              output_block_size, filter.get(), &output));
 868   EXPECT_EQ(output, expanded_);
 869
 870   // Next try with a mid-sized internal buffer size.
 871   const size_t kMidSizedInputBufferSize(100);
 872   // Buffer should be big enough to swallow whole gzip content.
 873   CHECK_GT(kMidSizedInputBufferSize, gzip_compressed_sdch.size());
 874   // Buffer should be small enough that entire SDCH content can't fit.
 875   // We'll go even further, and force the chain to flush the buffer between the
 876   // two filters more than once (that is why we multiply by 2).
 877   CHECK_LT(kMidSizedInputBufferSize * 2, sdch_compressed.size());
 878   filter_context.SetURL(url);
 879   filter.reset(
 880       SdchFilterChainingTest::Factory(filter_types, filter_context,
 881                                       kMidSizedInputBufferSize));
 882   EXPECT_EQ(static_cast<int>(kMidSizedInputBufferSize),
 883             filter->stream_buffer_size());
 884
 885   feed_block_size = kMidSizedInputBufferSize;
 886   output_block_size = kMidSizedInputBufferSize;
 887   output.clear();
 888   EXPECT_TRUE(FilterTestData(gzip_compressed_sdch, feed_block_size,
 889                              output_block_size, filter.get(), &output));
 890   EXPECT_EQ(output, expanded_);
 891
 892   // Next try with a tiny input and output buffer to cover edge effects.
 893   filter.reset(SdchFilterChainingTest::Factory(filter_types, filter_context,
 894                                                kLargeInputBufferSize));
 895   EXPECT_EQ(static_cast<int>(kLargeInputBufferSize),
 896             filter->stream_buffer_size());
 897
 898   feed_block_size = 1;
 899   output_block_size = 1;
 900   output.clear();
 901   EXPECT_TRUE(FilterTestData(gzip_compressed_sdch, feed_block_size,
 902                              output_block_size, filter.get(), &output));
 903   EXPECT_EQ(output, expanded_);
 904 }
 905
 906 TEST_F(SdchFilterTest, DefaultGzipIfSdch) {
 907   // Construct a valid SDCH dictionary from a VCDIFF dictionary.
 908   const std::string kSampleDomain = "sdchtest.com";
 909   std::string dictionary(NewSdchDictionary(kSampleDomain));
 910
 911   std::string url_string = "http://" + kSampleDomain;
 912
 913   GURL url(url_string);
 914   EXPECT_TRUE(sdch_manager_->AddSdchDictionary(dictionary, url));
 915
 916   std::string sdch_compressed(NewSdchCompressedData(dictionary));
 917
 918   // Use Gzip to compress the sdch sdch_compressed data.
 919   std::string gzip_compressed_sdch = gzip_compress(sdch_compressed);
 920
 921   // Only claim to have sdch content, but really use the gzipped sdch content.
 922   // System should automatically add the missing (optional) gzip.
 923   std::vector<Filter::FilterType> filter_types;
 924   filter_types.push_back(Filter::FILTER_TYPE_SDCH);
 925
 926   MockFilterContext filter_context;
 927   filter_context.SetMimeType("anything/mime");
 928   filter_context.SetSdchResponse(true);
 929   Filter::FixupEncodingTypes(filter_context, &filter_types);
 930   ASSERT_EQ(filter_types.size(), 2u);
 931   EXPECT_EQ(filter_types[0], Filter::FILTER_TYPE_SDCH);
 932   EXPECT_EQ(filter_types[1], Filter::FILTER_TYPE_GZIP_HELPING_SDCH);
 933
 934   // First try with a large buffer (larger than test input, or compressed data).
 935   filter_context.SetURL(url);
 936   scoped_ptr<Filter> filter(Filter::Factory(filter_types, filter_context));
 937
 938
 939   // Verify that chained filter is waiting for data.
 940   char tiny_output_buffer[10];
 941   int tiny_output_size = sizeof(tiny_output_buffer);
 942   EXPECT_EQ(Filter::FILTER_NEED_MORE_DATA,
 943             filter->ReadData(tiny_output_buffer, &tiny_output_size));
 944
 945   size_t feed_block_size = 100;
 946   size_t output_block_size = 100;
 947   std::string output;
 948   EXPECT_TRUE(FilterTestData(gzip_compressed_sdch, feed_block_size,
 949                              output_block_size, filter.get(), &output));
 950   EXPECT_EQ(output, expanded_);
 951
 952   // Next try with a tiny buffer to cover edge effects.
 953   filter.reset(Filter::Factory(filter_types, filter_context));
 954
 955   feed_block_size = 1;
 956   output_block_size = 1;
 957   output.clear();
 958   EXPECT_TRUE(FilterTestData(gzip_compressed_sdch, feed_block_size,
 959                              output_block_size, filter.get(), &output));
 960   EXPECT_EQ(output, expanded_);
 961 }
 962
 963 TEST_F(SdchFilterTest, AcceptGzipSdchIfGzip) {
 964   // Construct a valid SDCH dictionary from a VCDIFF dictionary.
 965   const std::string kSampleDomain = "sdchtest.com";
 966   std::string dictionary(NewSdchDictionary(kSampleDomain));
 967
 968   std::string url_string = "http://" + kSampleDomain;
 969
 970   GURL url(url_string);
 971   EXPECT_TRUE(sdch_manager_->AddSdchDictionary(dictionary, url));
 972
 973   std::string sdch_compressed(NewSdchCompressedData(dictionary));
 974
 975   // Use Gzip to compress the sdch sdch_compressed data.
 976   std::string gzip_compressed_sdch = gzip_compress(sdch_compressed);
 977
 978   // Some proxies strip the content encoding statement down to a mere gzip, but
 979   // pass through the original content (with full sdch,gzip encoding).
 980   // Only claim to have gzip content, but really use the gzipped sdch content.
 981   // System should automatically add the missing (optional) sdch.
 982   std::vector<Filter::FilterType> filter_types;
 983   filter_types.push_back(Filter::FILTER_TYPE_GZIP);
 984
 985   MockFilterContext filter_context;
 986   filter_context.SetMimeType("anything/mime");
 987   filter_context.SetSdchResponse(true);
 988   Filter::FixupEncodingTypes(filter_context, &filter_types);
 989   ASSERT_EQ(filter_types.size(), 3u);
 990   EXPECT_EQ(filter_types[0], Filter::FILTER_TYPE_SDCH_POSSIBLE);
 991   EXPECT_EQ(filter_types[1], Filter::FILTER_TYPE_GZIP_HELPING_SDCH);
 992   EXPECT_EQ(filter_types[2], Filter::FILTER_TYPE_GZIP);
 993
 994   // First try with a large buffer (larger than test input, or compressed data).
 995   filter_context.SetURL(url);
 996   scoped_ptr<Filter> filter(Filter::Factory(filter_types, filter_context));
 997
 998
 999   // Verify that chained filter is waiting for data.
1000   char tiny_output_buffer[10];
1001   int tiny_output_size = sizeof(tiny_output_buffer);
1002   EXPECT_EQ(Filter::FILTER_NEED_MORE_DATA,
1003             filter->ReadData(tiny_output_buffer, &tiny_output_size));
1004
1005   size_t feed_block_size = 100;
1006   size_t output_block_size = 100;
1007   std::string output;
1008   EXPECT_TRUE(FilterTestData(gzip_compressed_sdch, feed_block_size,
1009                              output_block_size, filter.get(), &output));
1010   EXPECT_EQ(output, expanded_);
1011
1012   // Next try with a tiny buffer to cover edge effects.
1013   filter.reset(Filter::Factory(filter_types, filter_context));
1014
1015   feed_block_size = 1;
1016   output_block_size = 1;
1017   output.clear();
1018   EXPECT_TRUE(FilterTestData(gzip_compressed_sdch, feed_block_size,
1019                              output_block_size, filter.get(), &output));
1020   EXPECT_EQ(output, expanded_);
1021 }
1022
1023 TEST_F(SdchFilterTest, DefaultSdchGzipIfEmpty) {
1024   // Construct a valid SDCH dictionary from a VCDIFF dictionary.
1025   const std::string kSampleDomain = "sdchtest.com";
1026   std::string dictionary(NewSdchDictionary(kSampleDomain));
1027
1028   std::string url_string = "http://" + kSampleDomain;
1029
1030   GURL url(url_string);
1031   EXPECT_TRUE(sdch_manager_->AddSdchDictionary(dictionary, url));
1032
1033   std::string sdch_compressed(NewSdchCompressedData(dictionary));
1034
1035   // Use Gzip to compress the sdch sdch_compressed data.
1036   std::string gzip_compressed_sdch = gzip_compress(sdch_compressed);
1037
1038   // Only claim to have non-encoded content, but really use the gzipped sdch
1039   // content.
1040   // System should automatically add the missing (optional) sdch,gzip.
1041   std::vector<Filter::FilterType> filter_types;
1042
1043   MockFilterContext filter_context;
1044   filter_context.SetMimeType("anything/mime");
1045   filter_context.SetSdchResponse(true);
1046   Filter::FixupEncodingTypes(filter_context, &filter_types);
1047   ASSERT_EQ(filter_types.size(), 2u);
1048   EXPECT_EQ(filter_types[0], Filter::FILTER_TYPE_SDCH_POSSIBLE);
1049   EXPECT_EQ(filter_types[1], Filter::FILTER_TYPE_GZIP_HELPING_SDCH);
1050
1051   // First try with a large buffer (larger than test input, or compressed data).
1052   filter_context.SetURL(url);
1053   scoped_ptr<Filter> filter(Filter::Factory(filter_types, filter_context));
1054
1055
1056   // Verify that chained filter is waiting for data.
1057   char tiny_output_buffer[10];
1058   int tiny_output_size = sizeof(tiny_output_buffer);
1059   EXPECT_EQ(Filter::FILTER_NEED_MORE_DATA,
1060             filter->ReadData(tiny_output_buffer, &tiny_output_size));
1061
1062   size_t feed_block_size = 100;
1063   size_t output_block_size = 100;
1064   std::string output;
1065   EXPECT_TRUE(FilterTestData(gzip_compressed_sdch, feed_block_size,
1066                              output_block_size, filter.get(), &output));
1067   EXPECT_EQ(output, expanded_);
1068
1069   // Next try with a tiny buffer to cover edge effects.
1070   filter.reset(Filter::Factory(filter_types, filter_context));
1071
1072   feed_block_size = 1;
1073   output_block_size = 1;
1074   output.clear();
1075   EXPECT_TRUE(FilterTestData(gzip_compressed_sdch, feed_block_size,
1076                              output_block_size, filter.get(), &output));
1077   EXPECT_EQ(output, expanded_);
1078 }
1079
1080 TEST_F(SdchFilterTest, AcceptGzipGzipSdchIfGzip) {
1081   // Construct a valid SDCH dictionary from a VCDIFF dictionary.
1082   const std::string kSampleDomain = "sdchtest.com";
1083   std::string dictionary(NewSdchDictionary(kSampleDomain));
1084
1085   std::string url_string = "http://" + kSampleDomain;
1086
1087   GURL url(url_string);
1088   EXPECT_TRUE(sdch_manager_->AddSdchDictionary(dictionary, url));
1089
1090   std::string sdch_compressed(NewSdchCompressedData(dictionary));
1091
1092   // Vodaphone (UK) Mobile Broadband provides double gzipped sdch with a content
1093   // encoding of merely gzip (apparently, only listing the extra level of
1094   // wrapper compression they added, but discarding the actual content encoding.
1095   // Use Gzip to double compress the sdch sdch_compressed data.
1096   std::string double_gzip_compressed_sdch = gzip_compress(gzip_compress(
1097       sdch_compressed));
1098
1099   // Only claim to have gzip content, but really use the double gzipped sdch
1100   // content.
1101   // System should automatically add the missing (optional) sdch, gzip decoders.
1102   std::vector<Filter::FilterType> filter_types;
1103   filter_types.push_back(Filter::FILTER_TYPE_GZIP);
1104
1105   MockFilterContext filter_context;
1106   filter_context.SetMimeType("anything/mime");
1107   filter_context.SetSdchResponse(true);
1108   Filter::FixupEncodingTypes(filter_context, &filter_types);
1109   ASSERT_EQ(filter_types.size(), 3u);
1110   EXPECT_EQ(filter_types[0], Filter::FILTER_TYPE_SDCH_POSSIBLE);
1111   EXPECT_EQ(filter_types[1], Filter::FILTER_TYPE_GZIP_HELPING_SDCH);
1112   EXPECT_EQ(filter_types[2], Filter::FILTER_TYPE_GZIP);
1113
1114   // First try with a large buffer (larger than test input, or compressed data).
1115   filter_context.SetURL(url);
1116   scoped_ptr<Filter> filter(Filter::Factory(filter_types, filter_context));
1117
1118
1119   // Verify that chained filter is waiting for data.
1120   char tiny_output_buffer[10];
1121   int tiny_output_size = sizeof(tiny_output_buffer);
1122   EXPECT_EQ(Filter::FILTER_NEED_MORE_DATA,
1123             filter->ReadData(tiny_output_buffer, &tiny_output_size));
1124
1125   size_t feed_block_size = 100;
1126   size_t output_block_size = 100;
1127   std::string output;
1128   EXPECT_TRUE(FilterTestData(double_gzip_compressed_sdch, feed_block_size,
1129                              output_block_size, filter.get(), &output));
1130   EXPECT_EQ(output, expanded_);
1131
1132   // Next try with a tiny buffer to cover edge effects.
1133   filter.reset(Filter::Factory(filter_types, filter_context));
1134
1135   feed_block_size = 1;
1136   output_block_size = 1;
1137   output.clear();
1138   EXPECT_TRUE(FilterTestData(double_gzip_compressed_sdch, feed_block_size,
1139                              output_block_size, filter.get(), &output));
1140   EXPECT_EQ(output, expanded_);
1141 }
1142
1143 TEST_F(SdchFilterTest, DomainSupported) {
1144   GURL google_url("http://www.google.com");
1145
1146   net::SdchManager::EnableSdchSupport(false);
1147   EXPECT_FALSE(SdchManager::Global()->IsInSupportedDomain(google_url));
1148   net::SdchManager::EnableSdchSupport(true);
1149   EXPECT_TRUE(SdchManager::Global()->IsInSupportedDomain(google_url));
1150 }
1151
1152 TEST_F(SdchFilterTest, DomainBlacklisting) {
1153   GURL test_url("http://www.test.com");
1154   GURL google_url("http://www.google.com");
1155
1156   SdchManager::BlacklistDomain(test_url);
1157   EXPECT_FALSE(SdchManager::Global()->IsInSupportedDomain(test_url));
1158   EXPECT_TRUE(SdchManager::Global()->IsInSupportedDomain(google_url));
1159
1160   SdchManager::BlacklistDomain(google_url);
1161   EXPECT_FALSE(SdchManager::Global()->IsInSupportedDomain(google_url));
1162 }
1163
1164 TEST_F(SdchFilterTest, DomainBlacklistingCaseSensitivity) {
1165   GURL test_url("http://www.TesT.com");
1166   GURL test2_url("http://www.tEst.com");
1167
1168   EXPECT_TRUE(SdchManager::Global()->IsInSupportedDomain(test_url));
1169   EXPECT_TRUE(SdchManager::Global()->IsInSupportedDomain(test2_url));
1170   SdchManager::BlacklistDomain(test_url);
1171   EXPECT_FALSE(SdchManager::Global()->IsInSupportedDomain(test2_url));
1172 }
1173
1174 TEST_F(SdchFilterTest, BlacklistingReset) {
1175   GURL gurl("http://mytest.DoMain.com");
1176   std::string domain(gurl.host());
1177
1178   SdchManager::ClearBlacklistings();
1179   EXPECT_EQ(SdchManager::BlackListDomainCount(domain), 0);
1180   EXPECT_EQ(SdchManager::BlacklistDomainExponential(domain), 0);
1181   EXPECT_TRUE(SdchManager::Global()->IsInSupportedDomain(gurl));
1182 }
1183
1184 TEST_F(SdchFilterTest, BlacklistingSingleBlacklist) {
1185   GURL gurl("http://mytest.DoMain.com");
1186   std::string domain(gurl.host());
1187   SdchManager::ClearBlacklistings();
1188
1189   SdchManager::Global()->BlacklistDomain(gurl);
1190   EXPECT_EQ(SdchManager::BlackListDomainCount(domain), 1);
1191   EXPECT_EQ(SdchManager::BlacklistDomainExponential(domain), 1);
1192
1193   // Check that any domain lookup reduces the blacklist counter.
1194   EXPECT_FALSE(SdchManager::Global()->IsInSupportedDomain(gurl));
1195   EXPECT_EQ(SdchManager::BlackListDomainCount(domain), 0);
1196   EXPECT_TRUE(SdchManager::Global()->IsInSupportedDomain(gurl));
1197 }
1198
1199 TEST_F(SdchFilterTest, BlacklistingExponential) {
1200   GURL gurl("http://mytest.DoMain.com");
1201   std::string domain(gurl.host());
1202   SdchManager::ClearBlacklistings();
1203
1204   int exponential = 1;
1205   for (int i = 1; i < 100; ++i) {
1206     SdchManager::Global()->BlacklistDomain(gurl);
1207     EXPECT_EQ(SdchManager::BlacklistDomainExponential(domain), exponential);
1208
1209     EXPECT_EQ(SdchManager::BlackListDomainCount(domain), exponential);
1210     EXPECT_FALSE(SdchManager::Global()->IsInSupportedDomain(gurl));
1211     EXPECT_EQ(SdchManager::BlackListDomainCount(domain), exponential - 1);
1212
1213     // Simulate a large number of domain checks (which eventually remove the
1214     // blacklisting).
1215     SdchManager::ClearDomainBlacklisting(domain);
1216     EXPECT_EQ(SdchManager::BlackListDomainCount(domain), 0);
1217     EXPECT_TRUE(SdchManager::Global()->IsInSupportedDomain(gurl));
1218
1219     // Predict what exponential backoff will be.
1220     exponential = 1 + 2 * exponential;
1221     if (exponential < 0)
1222       exponential = INT_MAX;  // We don't wrap.
1223   }
1224 }
1225
1226 TEST_F(SdchFilterTest, CanSetExactMatchDictionary) {
1227   std::string dictionary_domain("x.y.z.google.com");
1228   std::string dictionary_text(NewSdchDictionary(dictionary_domain));
1229
1230   // Perfect match should work.
1231   EXPECT_TRUE(sdch_manager_->AddSdchDictionary(dictionary_text,
1232               GURL("http://" + dictionary_domain)));
1233 }
1234
1235 TEST_F(SdchFilterTest, FailToSetDomainMismatchDictionary) {
1236   std::string dictionary_domain("x.y.z.google.com");
1237   std::string dictionary_text(NewSdchDictionary(dictionary_domain));
1238
1239   // Fail the "domain match" requirement.
1240   EXPECT_FALSE(sdch_manager_->AddSdchDictionary(dictionary_text,
1241                GURL("http://y.z.google.com")));
1242 }
1243
1244 TEST_F(SdchFilterTest, FailToSetDotHostPrefixDomainDictionary) {
1245   std::string dictionary_domain("x.y.z.google.com");
1246   std::string dictionary_text(NewSdchDictionary(dictionary_domain));
1247
1248   // Fail the HD with D being the domain and H having a dot requirement.
1249   EXPECT_FALSE(sdch_manager_->AddSdchDictionary(dictionary_text,
1250                GURL("http://w.x.y.z.google.com")));
1251 }
1252
1253 TEST_F(SdchFilterTest, FailToSetRepeatPrefixWithDotDictionary) {
1254   // Make sure that a prefix that matches the domain postfix won't confuse
1255   // the validation checks.
1256   std::string dictionary_domain("www.google.com");
1257   std::string dictionary_text(NewSdchDictionary(dictionary_domain));
1258
1259   // Fail the HD with D being the domain and H having a dot requirement.
1260   EXPECT_FALSE(sdch_manager_->AddSdchDictionary(dictionary_text,
1261                GURL("http://www.google.com.www.google.com")));
1262 }
1263
1264 TEST_F(SdchFilterTest, CanSetLeadingDotDomainDictionary) {
1265   // Make sure that a prefix that matches the domain postfix won't confuse
1266   // the validation checks.
1267   std::string dictionary_domain(".google.com");
1268   std::string dictionary_text(NewSdchDictionary(dictionary_domain));
1269
1270   // Verify that a leading dot in the domain is acceptable, as long as the host
1271   // name does not contain any dots preceding the matched domain name.
1272   EXPECT_TRUE(sdch_manager_->AddSdchDictionary(dictionary_text,
1273                GURL("http://www.google.com")));
1274 }
1275
1276 // Make sure the order of the tests is not helping us or confusing things.
1277 // See test CanSetExactMatchDictionary above for first try.
1278 TEST_F(SdchFilterTest, CanStillSetExactMatchDictionary) {
1279   std::string dictionary_domain("x.y.z.google.com");
1280   std::string dictionary_text(NewSdchDictionary(dictionary_domain));
1281
1282   // Perfect match should *STILL* work.
1283   EXPECT_TRUE(sdch_manager_->AddSdchDictionary(dictionary_text,
1284               GURL("http://" + dictionary_domain)));
1285 }
1286
1287 // Make sure the DOS protection precludes the addition of too many dictionaries.
1288 TEST_F(SdchFilterTest, TooManyDictionaries) {
1289   std::string dictionary_domain(".google.com");
1290   std::string dictionary_text(NewSdchDictionary(dictionary_domain));
1291
1292   size_t count = 0;
1293   while (count <= SdchManager::kMaxDictionaryCount + 1) {
1294     if (!sdch_manager_->AddSdchDictionary(dictionary_text,
1295                                           GURL("http://www.google.com")))
1296       break;
1297
1298     dictionary_text += " ";  // Create dictionary with different SHA signature.
1299     ++count;
1300   }
1301   EXPECT_EQ(SdchManager::kMaxDictionaryCount, count);
1302 }
1303
1304 TEST_F(SdchFilterTest, DictionaryNotTooLarge) {
1305   std::string dictionary_domain(".google.com");
1306   std::string dictionary_text(NewSdchDictionary(dictionary_domain));
1307
1308   dictionary_text.append(
1309       SdchManager::kMaxDictionarySize  - dictionary_text.size(), ' ');
1310   EXPECT_TRUE(sdch_manager_->AddSdchDictionary(dictionary_text,
1311               GURL("http://" + dictionary_domain)));
1312 }
1313
1314 TEST_F(SdchFilterTest, DictionaryTooLarge) {
1315   std::string dictionary_domain(".google.com");
1316   std::string dictionary_text(NewSdchDictionary(dictionary_domain));
1317
1318   dictionary_text.append(
1319       SdchManager::kMaxDictionarySize + 1 - dictionary_text.size(), ' ');
1320   EXPECT_FALSE(sdch_manager_->AddSdchDictionary(dictionary_text,
1321               GURL("http://" + dictionary_domain)));
1322 }
1323
1324 TEST_F(SdchFilterTest, PathMatch) {
1325   bool (*PathMatch)(const std::string& path, const std::string& restriction) =
1326       SdchManager::Dictionary::PathMatch;
1327   // Perfect match is supported.
1328   EXPECT_TRUE(PathMatch("/search", "/search"));
1329   EXPECT_TRUE(PathMatch("/search/", "/search/"));
1330
1331   // Prefix only works if last character of restriction is a slash, or first
1332   // character in path after a match is a slash.  Validate each case separately.
1333
1334   // Rely on the slash in the path (not at the end of the restriction).
1335   EXPECT_TRUE(PathMatch("/search/something", "/search"));
1336   EXPECT_TRUE(PathMatch("/search/s", "/search"));
1337   EXPECT_TRUE(PathMatch("/search/other", "/search"));
1338   EXPECT_TRUE(PathMatch("/search/something", "/search"));
1339
1340   // Rely on the slash at the end of the restriction.
1341   EXPECT_TRUE(PathMatch("/search/something", "/search/"));
1342   EXPECT_TRUE(PathMatch("/search/s", "/search/"));
1343   EXPECT_TRUE(PathMatch("/search/other", "/search/"));
1344   EXPECT_TRUE(PathMatch("/search/something", "/search/"));
1345
1346   // Make sure less that sufficient prefix match is false.
1347   EXPECT_FALSE(PathMatch("/sear", "/search"));
1348   EXPECT_FALSE(PathMatch("/", "/search"));
1349   EXPECT_FALSE(PathMatch(std::string(), "/search"));
1350
1351   // Add examples with several levels of direcories in the restriction.
1352   EXPECT_FALSE(PathMatch("/search/something", "search/s"));
1353   EXPECT_FALSE(PathMatch("/search/", "/search/s"));
1354
1355   // Make sure adding characters to path will also fail.
1356   EXPECT_FALSE(PathMatch("/searching", "/search/"));
1357   EXPECT_FALSE(PathMatch("/searching", "/search"));
1358
1359   // Make sure we're case sensitive.
1360   EXPECT_FALSE(PathMatch("/ABC", "/abc"));
1361   EXPECT_FALSE(PathMatch("/abc", "/ABC"));
1362 }
1363
1364 // The following are only applicable while we have a latency test in the code,
1365 // and can be removed when that functionality is stripped.
1366 TEST_F(SdchFilterTest, LatencyTestControls) {
1367   GURL url("http://www.google.com");
1368   GURL url2("http://www.google2.com");
1369
1370   // First make sure we default to false.
1371   EXPECT_FALSE(sdch_manager_->AllowLatencyExperiment(url));
1372   EXPECT_FALSE(sdch_manager_->AllowLatencyExperiment(url2));
1373
1374   // That we can set each to true.
1375   sdch_manager_->SetAllowLatencyExperiment(url, true);
1376   EXPECT_TRUE(sdch_manager_->AllowLatencyExperiment(url));
1377   EXPECT_FALSE(sdch_manager_->AllowLatencyExperiment(url2));
1378
1379   sdch_manager_->SetAllowLatencyExperiment(url2, true);
1380   EXPECT_TRUE(sdch_manager_->AllowLatencyExperiment(url));
1381   EXPECT_TRUE(sdch_manager_->AllowLatencyExperiment(url2));
1382
1383   // And can reset them to false.
1384   sdch_manager_->SetAllowLatencyExperiment(url, false);
1385   EXPECT_FALSE(sdch_manager_->AllowLatencyExperiment(url));
1386   EXPECT_TRUE(sdch_manager_->AllowLatencyExperiment(url2));
1387
1388   sdch_manager_->SetAllowLatencyExperiment(url2, false);
1389   EXPECT_FALSE(sdch_manager_->AllowLatencyExperiment(url));
1390   EXPECT_FALSE(sdch_manager_->AllowLatencyExperiment(url2));
1391 }
1392
1393 }  // namespace net