Merge "Enforce intra_y_mode_mask for inter rd."

diff --cc vp9/encoder/vp9_rdopt.c
index ecfae18,5f75941..ed81fbe
--- 1/vp9/encoder/vp9_rdopt.c
--- 2/vp9/encoder/vp9_rdopt.c
+++ b/vp9/encoder/vp9_rdopt.c
@@@ -3305,62 -3343,29 +3307,66 @@@ int64_t vp9_rd_pick_inter_mode_sb(VP9_C
          mode_excluded = cm->reference_mode == COMPOUND_REFERENCE;
      }
  
-     if (ref_frame == INTRA_FRAME && this_mode != DC_PRED) {
-       // Disable intra modes other than DC_PRED for blocks with low variance
-       // Threshold for intra skipping based on source variance
-       // TODO(debargha): Specialize the threshold for super block sizes
-       const unsigned int skip_intra_var_thresh = 64;
-       if ((mode_search_skip_flags & FLAG_SKIP_INTRA_LOWVAR) &&
-           x->source_variance < skip_intra_var_thresh)
+     if (ref_frame == INTRA_FRAME) {
+       if (!(intra_y_mode_mask & (1 << this_mode)))
          continue;
-       // Only search the oblique modes if the best so far is
-       // one of the neighboring directional modes
-       if ((mode_search_skip_flags & FLAG_SKIP_INTRA_BESTINTER) &&
-           (this_mode >= D45_PRED && this_mode <= TM_PRED)) {
-         if (vp9_mode_order[best_mode_index].ref_frame[0] > INTRA_FRAME)
+       if (this_mode != DC_PRED) {
+         // Disable intra modes other than DC_PRED for blocks with low variance
+         // Threshold for intra skipping based on source variance
+         // TODO(debargha): Specialize the threshold for super block sizes
+         const unsigned int skip_intra_var_thresh = 64;
+         if ((mode_search_skip_flags & FLAG_SKIP_INTRA_LOWVAR) &&
+             x->source_variance < skip_intra_var_thresh)
            continue;
-       }
-       if (mode_search_skip_flags & FLAG_SKIP_INTRA_DIRMISMATCH) {
-         if (conditional_skipintra(this_mode, best_intra_mode))
+         // Only search the oblique modes if the best so far is
+         // one of the neighboring directional modes
+         if ((mode_search_skip_flags & FLAG_SKIP_INTRA_BESTINTER) &&
+             (this_mode >= D45_PRED && this_mode <= TM_PRED)) {
+           if (vp9_mode_order[best_mode_index].ref_frame[0] > INTRA_FRAME)
              continue;
+         }
+         if (mode_search_skip_flags & FLAG_SKIP_INTRA_DIRMISMATCH) {
+           if (conditional_skipintra(this_mode, best_intra_mode))
+               continue;
+         }
        }
 +    } else {
 +      // if we're near/nearest and mv == 0,0, compare to zeromv
 +      if ((this_mode == NEARMV || this_mode == NEARESTMV ||
 +          this_mode == ZEROMV) &&
 +          frame_mv[this_mode][ref_frame].as_int == 0 &&
 +          !vp9_segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP) &&
 +          (!comp_pred || frame_mv[this_mode][second_ref_frame].as_int == 0)) {
 +        int rfc = mbmi->mode_context[ref_frame];
 +        int c1 = cost_mv_ref(cpi, NEARMV, rfc);
 +        int c2 = cost_mv_ref(cpi, NEARESTMV, rfc);
 +        int c3 = cost_mv_ref(cpi, ZEROMV, rfc);
 +
 +        if (this_mode == NEARMV) {
 +          if (c1 > c3)
 +            continue;
 +        } else if (this_mode == NEARESTMV) {
 +          if (c2 > c3)
 +            continue;
 +        } else {
 +          assert(this_mode == ZEROMV);
 +          if (!comp_pred) {
 +            if ((c3 >= c2 &&
 +                 frame_mv[NEARESTMV][ref_frame].as_int == 0) ||
 +                (c3 >= c1 &&
 +                 frame_mv[NEARMV][ref_frame].as_int == 0))
 +              continue;
 +          } else {
 +            if ((c3 >= c2 &&
 +                 frame_mv[NEARESTMV][ref_frame].as_int == 0 &&
 +                 frame_mv[NEARESTMV][second_ref_frame].as_int == 0) ||
 +                (c3 >= c1 &&
 +                 frame_mv[NEARMV][ref_frame].as_int == 0 &&
 +                 frame_mv[NEARMV][second_ref_frame].as_int == 0))
 +              continue;
 +          }
 +        }
 +      }
      }
  
      mbmi->mode = this_mode;