From 47d04d077c58610652a92aa22de24dfa9eb4f77a Mon Sep 17 00:00:00 2001
From: Parichay Kapoor <pk.kapoor@samsung.com>
Date: Thu, 16 Jul 2020 14:12:43 +0900
Subject: [PATCH] [Loss] Bug fix for loss

Added bug fix for loss forwarding
- for sigmoid with cross entropy, formula was correct, however implementation was wrong, also inverted sign of the output
- for MSE, average is needed than sum
- for softmax with cross entropy, divide by input width is not needed but still mismatch

Signed-off-by: Parichay Kapoor <pk.kapoor@samsung.com>
---
 nntrainer/include/lazy_tensor.h  |  11 +++++++++--
 nntrainer/include/tensor.h       |  10 ++++++++--
 nntrainer/include/tensor_dim.h   |   1 +
 nntrainer/src/lazy_tensor.cpp    |  19 +++++++++++++++++++
 nntrainer/src/loss_layer.cpp     |  25 ++++++++++---------------
 nntrainer/src/optimizer.cpp      |   2 +-
 nntrainer/src/tensor.cpp         |  12 ++++++++++++
 packaging/unittest_layers.tar.gz | Bin 5696 -> 6084 bytes
 8 files changed, 60 insertions(+), 20 deletions(-)

diff --git a/nntrainer/include/lazy_tensor.h b/nntrainer/include/lazy_tensor.h
index f0a810d..0901218 100644
--- a/nntrainer/include/lazy_tensor.h
+++ b/nntrainer/include/lazy_tensor.h
@@ -127,7 +127,7 @@ public:
    * width direction
    * @retval    LazyTensor *this
    */
-  LazyTensor &sum(int axis = 0);
+  LazyTensor &sum(int axis);
 
   /**
    * @brief     Wrapper method of average. see tensor.h for more detail (memcopy
@@ -135,7 +135,14 @@ public:
    * width direction
    * @retval    LazyTensor *this
    */
-  LazyTensor &average(int axis = 0);
+  LazyTensor &average(int axis);
+
+  /**
+   * @brief     Wrapper method of average. see tensor.h for more detail (memcopy
+   * happens)
+   * @retval    LazyTensor *this
+   */
+  LazyTensor &average();
 
   /**
    * @brief     apply A tensor function when predicate is true
diff --git a/nntrainer/include/tensor.h b/nntrainer/include/tensor.h
index 165d84f..20fee1c 100644
--- a/nntrainer/include/tensor.h
+++ b/nntrainer/include/tensor.h
@@ -302,7 +302,7 @@ public:
    *            3 : width direction
    * @retval    Calculated Tensor
    */
-  Tensor sum(int axis = 0) const;
+  Tensor sum(int axis) const;
 
   /**
    * @brief     Averaging the Tensor elements according to the axis
@@ -312,7 +312,13 @@ public:
    *            3 : width direction
    * @retval    Calculated Tensor
    */
-  Tensor average(int axis = 0) const;
+  Tensor average(int axis) const;
+
+  /**
+   * @brief     Averaging the Tensor elements by all axis
+   * @retval    Calculated Tensor
+   */
+  Tensor average() const;
 
   /**
    * @brief     Anchor a starting point to defer following evaluation
diff --git a/nntrainer/include/tensor_dim.h b/nntrainer/include/tensor_dim.h
index 89cabd0..ebea783 100644
--- a/nntrainer/include/tensor_dim.h
+++ b/nntrainer/include/tensor_dim.h
@@ -59,6 +59,7 @@ public:
   void width(unsigned int w) { setTensorDim(3, w); }
 
   const unsigned int *getDim() const { return dim; }
+  const unsigned int getNumDim() const { return MAXDIM; }
 
   void setTensorDim(unsigned int idx, unsigned int value);
   int setTensorDim(std::string input_shape);
diff --git a/nntrainer/src/lazy_tensor.cpp b/nntrainer/src/lazy_tensor.cpp
index e57433a..10e84ff 100644
--- a/nntrainer/src/lazy_tensor.cpp
+++ b/nntrainer/src/lazy_tensor.cpp
@@ -202,6 +202,25 @@ LazyTensor &LazyTensor::average(int axis) {
 }
 
 /**
+ * @brief     Wrapper method of average. see tensor.h for more detail (memcopy
+ * happens)
+ * @retval    LazyTensor *this
+ */
+LazyTensor &LazyTensor::average() {
+  auto f = [](Tensor &t) mutable -> int {
+    try {
+      t = t.average();
+      return ML_ERROR_NONE;
+    } catch (std::runtime_error &e) {
+      return ML_ERROR_INVALID_PARAMETER;
+    }
+  };
+
+  call_chain.push_back(f);
+  return *this;
+}
+
+/**
  * @brief execute the call_chain to evaluate
  * @retval calculated tensor
  */
diff --git a/nntrainer/src/loss_layer.cpp b/nntrainer/src/loss_layer.cpp
index 94eb327..a3afffb 100644
--- a/nntrainer/src/loss_layer.cpp
+++ b/nntrainer/src/loss_layer.cpp
@@ -55,7 +55,7 @@ Tensor LossLayer::forwarding(Tensor output, Tensor label, int &status) {
     // y2 <- y2 - y;
     y2.subtract_i(y);
 
-    l = y2.chain().multiply_i(y2).sum_by_batch().multiply_i(0.5).run();
+    l = y2.chain().multiply_i(y2).average().run();
   } break;
   case COST_ENTROPY_SIGMOID: {
     // @todo: change this to apply_i
@@ -66,23 +66,18 @@ Tensor LossLayer::forwarding(Tensor output, Tensor label, int &status) {
                         .apply(static_cast<float (*)(float)>(&std::exp))
                         .add(1.0)
                         .apply(logFloat);
-    mid_term = mid_term.add(mid_term.apply(relu));
-
-    // loss = y * y2 - (log(1 + exp(-abs(y))) + max(y, 0))
-    l = y2.chain()
-          .multiply_i(y)
-          .add_i(mid_term)
-          .multiply_i(-1.0 / y2.getWidth())
-          .run()
-          .sum_by_batch();
+    mid_term = mid_term.add(y.apply(relu));
+
+    // y * y2
+    Tensor end_term = y2.chain().multiply_i(y).run();
+
+    // loss = log(1 + exp(-abs(y))) + max(y, 0) - (y * y2)
+    l = mid_term.subtract(end_term).average();
+    y = y.apply(sigmoid);
   } break;
   case COST_ENTROPY_SOFTMAX: {
     y = y.apply(softmax);
-    l = y2.chain()
-          .multiply_i(y.apply(logFloat))
-          .multiply_i(-1.0 / y2.getWidth())
-          .run()
-          .sum_by_batch();
+    l = y2.chain().multiply_i(y.apply(logFloat)).run().sum_by_batch();
 
   } break;
   case COST_ENTROPY: {
diff --git a/nntrainer/src/optimizer.cpp b/nntrainer/src/optimizer.cpp
index 8dcdc80..501e248 100644
--- a/nntrainer/src/optimizer.cpp
+++ b/nntrainer/src/optimizer.cpp
@@ -117,7 +117,7 @@ void Optimizer::apply_gradients(std::shared_ptr<UpdatableParam> params,
     /// @note: that current implementation does not update grad since updating
     /// grad changes it's dimension
     Tensor x_grad = param.grad;
-    x_grad = x_grad.average();
+    x_grad = x_grad.average(0);
     switch (type) {
     case OptType::sgd:
       x.add_i(x_grad, -ll);
diff --git a/nntrainer/src/tensor.cpp b/nntrainer/src/tensor.cpp
index 94923f6..449119d 100644
--- a/nntrainer/src/tensor.cpp
+++ b/nntrainer/src/tensor.cpp
@@ -892,6 +892,18 @@ Tensor Tensor::average(int axis) const {
   return result;
 }
 
+/**
+ * @brief Calculate average value according to the axis.
+ */
+Tensor Tensor::average() const {
+  LazyTensor lazy_result = this->chain();
+
+  for (unsigned int axis = 0; axis < dim.getNumDim(); ++axis)
+    lazy_result = lazy_result.average(axis);
+
+  return lazy_result.run();
+}
+
 void Tensor::setValue(float val) {
   float *data = getData();
   std::fill(data, data + length(), val);
diff --git a/packaging/unittest_layers.tar.gz b/packaging/unittest_layers.tar.gz
index 94cf9ea537247e66e311aa65ab2f7ef158132d03..1610013a1ebf9a1e15f42062f73bc6a279e6934d 100644
GIT binary patch
literal 6084
zcmZv7XEdAvw{=7a(TU!p_Xvp|WVGnLi_YkbZZHu+5WSZndT$|#Hb#kF29eQ6M2iwc
zZ!^#LzTdq+?^@@_UVE>#_m8vJIqZo<L?o^EZSUa+db<S#IQs|KdfJCNL;OVp>>>9K
z@}b7MZfnC_I+`^ahvi&U6&-cAqI($Rk;!g#87LrkDkea*CX*@7@8gih)Q-MeVd<06
zOiE6k@(&IyT90DZ?J*Wd`L0Df7Oj|bCUmz?F{}}R#(4ZiG$CHD?FVDg$z%!f=i<l2
zABa*USQI(=W###eVJ__pd@%r?W7Mw=u-%hQAo7lu3(t0<cVt0@Lz0Stf#JhHviC#0
zybervdbbQf(V}=u<O~y&L+cZ4sVm{cvDOvEJa+LKDe!HoGb+ALsZ*qAQ>6d6vLS8x
zZX#<=Yt8o-ol-UT1dqS9Aiy)O-w`4l_9ZeV?PlI>?lc~(A+-LulP-P#htZtn@94=H
z$A5n{*vQ|U6NsUc3~en+FoJm-T0Nh7v?T4HLj}DRDc-vOgofRn+KnynKAfL#l8&Qy
z3DA#=^A`k7FHWVN`b*b$EeZJ5kUqhF`>_}<P*_9RTerac5F-4kz+yziL#o26iQd--
zldRS@6(QU6f_T=ZcERS1Y^rW<I;UtJNz#J6T<a6#<iE+-2V8M5{Wr(zK#ZE20uDyL
zolQ0%R3v}%v$j`Xnri+~{@0Wns<&NkAdS5a59e#fKPJ!B^|83xB3R5q6Ir#Q`e@+}
zOl}JO7+3i;O*RFcy2T+rf*36+_8RIFlqZ)<nW{Vi#p~mgiBlfRaQJt~T<RnX4DM>!
z$M5lPHYN^jF^{pVU9rKRvuoczKQajF)vyjuiyf8#$)_!7hO^4l*CoFy?3S`*3C*qY
zidG)5&B%_S6rl+GWc?REr_`%-Y}GJTwm|FiB-f`78HT{<Xw^JR=tPf?#Jio}vW7Z|
z)Yo*h)9GH7bCi}7qq=`d789ggcnc^Xw7NQTRo;TxrVV8qq4Oj5I+En%(@L1=Fxg01
z%2Np@xAa@$F^fG?$IG37t&98aLFOtpSRGE=-ycXW=hL^+Ntsf4qBGvf8Z?hVyw@1;
zHkWO{)(R9u5`yhR=h1dHu*{&Jp|g$qBHYiPWo0H!L3C}(o?7UotnjgvYnG84cnymh
z8fsR&O`{sAPg{R<ld}GKT+iR_kzwI^u-B6`oc4-|-vea-M+ps&t*`RQvHvRJk6j(&
z3dSl|s-?+DWf+q9EOJII%fyz<zWP-Rl&2oMkPQ2i4zi)3D~nLXZe+2!w9K`=wK6_I
zBZ-=WHz~$NtP)Wa?)e9zX0AiiktnuqDRmy@Gyz&FhomG3BmADbQ+h9J_X`ul1dFl;
z6!Pwp-@F&(=t+@fogmvET|sgpw~$}9Y>jUtHs|l8aHwmsD-j@uFude=;9uO`4)ggO
ztT={5J@%EMH=XAdQ!_u^r{MNy@klO7T0J#H9-UgOWLg`TU&s8@TdoamgB-SK_+PPV
zrxXTn70tf3URq6*D`X=)z;Db)-pTX5Mx-Jh=gtiX6Hd-T&OdWyg;M=d5z@SPmD*ZF
zg@68e^=xT2{~(%u=9`r(r`{69x%XPSK0qn5|BNRq*6s`;@)m1WIZaF!JITUKgw0xC
zRgCWDHqi(-67A5Q>7(nj6zv^NsnZ@Q-7|<T>Bitob(;N{Z6ORW;#QUWii*an(rAt@
zC<|>1e)4Sl{6{(Lso@!QiyT6c&3kHt#RxJYKJ(aPj~49MF3f9Vu1oTp=4rLRWTac3
zUTdQy#FfH|Yi5v+*oH2*{P8;<sXaT3+WR*R!KH1_K>XQ*cIsaD`W3eAw=B)T<>#Q7
zGgRA&&b`BA?cTw-=n}ffLI|i8yvo7;v13yev-IRWBU_ba30L=>Cuc`LQPBF{A5o~5
z|FHI;7|lmcuVrmBl_QnirbK=ahSYDA^|9>vJ$Nd6vS{m@%#Ljj*>L^$!T3<4nnB_v
zKPRGBf$r^}7?Y*B=*ezb<3CvM#C^sx9n-xp^S+iQ`mMv^QZfLiMuNm!X9fK2MTxik
zJ20-o$7Omx+ET0pP%)iqMOkI0VJi)LHxV90h;sLaKNZ+F{lZ+2a`k}rQM}1ZhQaZw
zUD5ns#3cN<8aX(Mgga(Wb}wM^*L|s!-a~W$<8SXlmCc!ZNK5tCR6OOg&xoVR3AwP^
z3Ju!n5<*_+t>hD0a4(T_fEL<+qix=x{LO<Zk*Z`y`UI6E5w@pYJ@G-k_vuNuI3@h}
zb6}QiULCRw8IEdPu`)_u!ueTd*xztTrln(_Hzs{3gR38Wa#)Y|5Un)tXZ$t!CFo{s
zoW|-*Et++tq^b4BuFipn^)#GMHbahbOy=-vRy2|rCkL?J-2tUnK<j@h%>;-81Vi^D
z_5O1h%WtpK)k68}BaRIB7a7g3WV7W0Ra6Sz<oO?spFa&{p@^Wn9HeL=Ec`7lBW-$E
z@uB#o6LLyHmsgXZB6zbT^(Q!6q$zfLF0cBmCC)SW+0D?@P&GC5{YZV~R?|<vGfzIU
zvlgD88P08d!-<Jy#R7c3q8B#Un0?*DP45vu^WTl{g^2%dJ`Tue6(!~T(rfql=1m3r
zkj6S>Gqkl#Z{BDnYFQz0n{P812k=0Fl)D-r_73)M&3s`4*x}F%lo$D|ua0o1u^3z8
zbS@?n?y=QX6`JKA9HO%Fas|OpRg(<x@^TY753*hc4Kg_5F8RRD$t;_KgB-={d|`o;
zk7(hMzFm-yFGP(7t&wEb?l$5Ut7Pi-yF{d?FZCSR7wwl8SkL&a@hk-}`o$pyrtNYL
z8KX)GU+Fo<)Q<rO87}>954oQ67a$>ntpW6rD?r{c!GB+lJIH#5`Y6Q<REa-pf1I}-
z>vtM4gFq9Mlx(#j9SOX)mN)FPNPpww&uBpzN69|+kAqcLPL{PSeFCW*e<$QS@vIba
zl**y8H7-EFpZMjp1s(<MO_SM?HWFi~J}onor|;&33?yM)IxgA1y09_6aVDXV0Yy@)
zx$y_9Hx!}|OB5_)N%f}JC8L2$!$FH@xYy!vnQy5~t`$my->JnC$<n<3l86ctuLjcp
zF~#7I8P7%S)VZ~nE9eLOZe%`~bE5jq5>+*e&IF9{Z$(d9eh+hN*0M#lrMgLKz@tdK
zD_HR=j62c9UEydWR^aj0PgOhHclWduyNfqFWJbaYKT;PHHd^-+t#*iwtVao6tj(zf
z_pk;f$%ACfupMGpGC=As1F)?Ge8Au{F&qhW4?qo^zSPxN`bEY*;GgO1A|KnGM=TOZ
zd*jz4D;hI0FzQMCO|2)fEuTKE%Z>kR=xh4vsGtFM8?x7aI1;04rQLBP?DqE_OxA-s
zO5mu<e*LAZbdWsP>Q92lCJov(YcZ)$mq@NpY8Tp9DQ@bdD9I8l&*5e3)M)xHcG7ay
zsnC_wj{KEm8?Kq*0VXrsL*5|Sl#PA==ECFj`rx>X$P#XD)B8RH7e@2HS{4+$6(hSH
z_+>3l-gApHpT_16bUHC^S)(e|)er^-9P01}TYKdiy8H%2uY?Cre6&%Cb*fUf+4eN<
zv)W#lS>lfj>%lBChUhLx0zXlFX}ixxUq=_G+#+ev=Sa$?e9|4nX#2tuNR*JyPu`Zo
z+p_*;aOl8mNY65}iGlpd$v<8=+&y(Pz0Y~Tt{h!uXX3!MSUb2*-ym9R+(bLsoLME3
z($=!;C;13P73-0!gtmLy<2JeB`5iLaZ$RL&R+AQBCrB)1UFEurwWnFkk(_#{fBV4A
zU_G3oeM=p*Q7WlBhT$E>XDpXAqT{ofD2ZH-ehja&+wh<TlQ74B`x2e(POiLsnLOgV
zq-$@i{BQiPeQSv5R9#lOyU7a^lQB++425&vUM@4+{T3p|!-6oyFhi<Dgo{8kB~s%8
zTA@n4sMmhir8eF84gcG7szuX2{k$I<C44I!9zw5U=+!u^q{z!8u+5rs@E*3v_ygI8
z_N`_Lqn;A4LMSPcW3&Xk-qL0c9?kTgMfw`S-!5@8cfX+fy?Y+BIc-Z#?tmBK09N^;
z%g4XpOuY2&^9MGHjqcb=qi_UYzG%FDs$c%$gX_2FI|C<}rz}Rsug3lG(!ul<3{52u
zh8>=nQzfc|^=Gjim2*-(k~ZmxJ&V}>@fNIbmDsc6G#AP;_2<Vv^HA;>h4OiA2_+28
zEIZ}S5FReR3g=ZC<`h?%oO`$AOTK2E`c|zs#5zJQ|ArJu7Sar;s1(>X7ctcp%}K10
z1ydU&$pl75#7Z!Kji_G@RXx+4jn-Vb_o!!2eUDD`DRIl&F=4zpFSAs@jCIZ{OO<EV
zh@-0{N;u(4+HaNFvQxfW^56Fw^!>&*DoH6tf=+~?oARA!y8woJJn+WJP*B~!(mdJ~
zlQO4S<bPD@nr8Az;e8qF`ioIg`sMM_S?v#8Rm&r?AaZB-vF1)wor2=_4buTSs+hK!
z#y%}@;7{VP6{je(QA~(?jk<kn*i7?wRbjc8IO1c^#B&v<ezjL*Upt=%ym?KJYc<nf
zcoH!uqvlW_DoOZ)z~LODEz4s1swU-3qdtu2XUJvW^yZ6Bq-I8tjR#*$qUq+>TY^EB
zk*Yt~+ui9w*R#D2){3E>*_rRUmWy_Y`u~Xd1$KQyNrb6tq>041U%d{DKo&*xy8ix#
z1<pQ9_kQvnx2<~{uPojQay;j98ICN4-t7X4gs?kcc+*j<*%b?rY8OHZ^E6lN#4c>?
zejpMS=7b6sD&n)WcjCG{jY4TXBGXV|r(+B<Naqthc5(j{cV7DC{_>=yJXDr0ufmRa
zVg#n|@fO?Ds&36}tC@bfJd9%pP5$E92TOG5mAZ>TNQ!E<yc6H{nkzBRY_S>}I3c)A
zNR#1ZpHE+~n0uYGh}Y1n?J^2=S=K|#w{DYX<U+G5R4^{fMezkb&$FH)*Jj3&uLE17
zQjS`a9_bv=$ML#+ZB4L|YIL^RHft7BNP(H*MqwV~0Ax*`11<1G{*RGrFVGG6oNy9_
z-2p5CWVT};D_)3KU283xtOcNr;l147pv{nRNteJ7g&*L8zD#mGCrM}GI?7i2^k*51
z8B<BZvq6Lf;g|w}I#Guj#FGWKU6mI@ydTbgLY%D{3@rjjG<p9XPW{OiFW8pJxZX>0
zy)gOe*>!6e1j*v^r>khOUh~X0{2XN>1@vNIIU`@)M4jW3x0)|p<#I6zSqD)wAr}`X
z4s^BY-z_nXb}6FXc8VQ@tzrtUK+e@SoL;_+g0O?jFZ$E9V@IRrVF1;4b(B6WB1JK@
z`#M+t1*1EPx@7+8Lt<3A{uMuCa4F5xPG0gss@KgK#bf}+Uw;Y47J#K`o|RyeIe9&u
zuKoQE*5?00$-u^pg)f}eyqttd>9s*w!>lPu*P%lJ0*ZQ`q@>*<p3hL<^0?3LTK_&?
zrimb!*J0?Bnj_6VeffLs4a&!OWZh35m9%^^H%uks)l{O|0^QxKfX3ul%smhu_YOe$
zZ1=JSg-Xa{001L%R5ai6?yJn8(fs9`s*5xsH@FH2^x3gy`44}mI*lICtYc+F_<h=6
zfLL<kIxy@Ea^GaDv2@XfsdHD1X)hc~;ALH)behsw%9vIME{l$>L^A8x;M0KnXoPB}
zGnGy|pPyaLh18RsjbP8&8L3&>7+kfn8#lHCzi_C@%@qIoIRS&CR8_sLv;w(1jlCX^
zTD3S*NbMn$g_&He6TIGFQMOyBC!AOPi<c_A7MopR`P@VDdaHqc^uj;OX?g72$y<}@
zANUcUo~te=D-A@$8%=a>)T96k686%UsdU#s*HD$eL}z`x>dcbY;!Q`?+8J(nR_QAK
z6PG#pZuK1ndiAyHnu-}((|L7fk?tq>u5I**F>aqB6xM|A7f<rU9=V{NFE?&HgmF!w
z#-0JUBO}ezA9!xO@kosUkFCgH>`~G=(*9RqkNN&S`r080<^N{wl+q(oeCN4h)BfO6
z(foOL<WApy*c)V`Ow(%Uv1L(QF7xe?RN&1az}O1xYz3mQiavSYNY8zUOvQGi8Z9NV
z%{E4;e;=677f5|FuRFE)ocJsK&q!qc%X}=j9$zc}=Yy>b=1)RhR6*BTxhPzkr~e&4
zy2uChlC0TLYhrTIYm^LZ&EZV9+UThsMF5rh#_SO3PgkBoul%8nX&VV)347%I?nwi&
z12yt}F{+2~nkxGsn%m>Wj|SRCYsE**L0FPXm@x%l;d?(){z8(we5ti4X4Ti*lv!Fw
z+>t$%so8HoO6y_6!myiJNd`P_j4G}VWI+M<nCgvb5(P~@_HdE7m6sm;s^t7nwL0H*
zGCXCemA-Mg*(DCO&bmfeEo)51DB<&VL&e|pHik<M7tdPYdmw(1i{b&#7)Ni1LeUYU
z30~(d&O2xr!Q?9eVQrzP*SFpSx!AH?uhPtjkfA7>V_9PO6rE>iedZT!)=^WiB0GC9
ziSA^u6AzEBuv;R*a&h^2CyK?~0;_>$avFZ7$-HiEDHpyuOUPWt?HLinswL(gLys<u
z%SYqe$sitWgBxF6Y1vS+zad8Acpt84EpvXK&-Sudz>{i}R8oV8Ys;=Zpeb&37;l~G
zuim2uzx+XpB>8MV6$nqwvbt~3llRok??op2L6r6m;RjAvJBTV{z{^k~-twZ@N?1Tt
zMaA=suXCU<ELCoyKg~O)`o)WT{@eCGQ$yBwV)X|tE~8aE!V^v8wGEB0F6^~}BzhCF
zau%AC)`#utDydhH;CJoa`y8VP($jimEZi^4?qGw7kp+9VrZErGL_zP?mYHuA36;+x
z4^QRWFB5xEptF#f(~fValnS=<1EDMMMBF5z;329bI8o*jJX8E1W<RX8(ahg=`^_XG
z=T@BJ5LtS3GB4l0b3%zQpXk_uP|!tWJ-oD9Erg0(8nKn45@aqH<RX`L^5old5+X;V
zZ>znhutsOPY%4-ZSO8fD8!TnSsaXLHziiuXF3~-iP@U7w^J4A&2ZU%~w)xkuquUjw
zBgOK&X*7)DQhu!1;ne^gPzx9w(YFZmhFR-xeh$I10Ql!}TblnuE{(M_Mvvr=RP6tR
zn#p5yeuqe1;BY1X;5YLdH9o#KX6{XX2nEEZHSnKBVDQdUZlshCtALz$pqy?dY(9p?
zjsq(se<TBplSU$csGK*!sB2z2N^e)3Uw@=<Jo;NQs|O%#nqJb|anAG^oMwxVe`MS1
zAI6)P<3B_rFC`g^0J1S9*~FhS7AnP4Z?J6eFH5?QF|xaY)h4E|w7L>~EBaqE(APdT
zjW*_QXr?*H?jnj?;w3V!cwBi9K=T6HLE`L9*T3W0&C$Rll=&AM&#}x+Rf#eq{ydsO
zGK5h~5nO8k7^9S7$z&fb@0K~6pkc>1YgWAKSmyZeSS+$&cl(hS%XF;u4*hqOGZ_Rb
ztEgvu;iZ?u>FHypyLR#UAyVPTcF)ZbV)ePivxI%vpI`)z0_)L4tPjL0Eg3Wx!A}S)
zn8(4r<ES5n{{-`QdkQ8Rt(7EtWYn~3u{F^Fq*+tu#{%8qum3{7=P%KEe^P{6Ru&cb
znEnZyjP3N);DyU^#WyujL>OEP2kvTAUhfahme0w5QUA-E&z)B$8o3pbUs$L`QuBVu
zThLH!39d<__mzdN6H|_R2>Nq(UxwC{zFKI=BK5?opvEFw?B5El=8pu?RM<|FNOi<O
z5!=VqnSHICx~judeOUI3H#VKi_0tzk7iXTDm0=#g+WV6!E~mfacTuFwKbY<~dhMoq
zNPIJz{bg>a1381FZ{Qqfx+J}-FJJ%r{_<If!U$sX5zkSa0BH{nrE>lj`}BuPh}=3t
zz8$xv+0FKXv>Nt51ZfXD?(g;5_%9S7Zh+lqKZsM*CiUU#&gr`xP{j&uJ>%J1OSMy6
zkz~Z^hKNx_Vw>hn?^VM#-eP@?9RAml76SpHfV^s43wtsL(Yya^PJE+JVU-Bn!yTA5
zb0Gl2mf>{h3jqnQ|3nr7(A?f@#W}@?JAgLLf5prHBI5rqZ``r}Uxp)PeZal%wG-0!
IM(*AFe}XI|-2eap

literal 5696
zcmV-G7Qg8qiwFRZ`3GMB1MHj&T#RY^$A{=Jp~)c;DyI!$W}2Czp8LA)N+qJwqS!Jz
zD9e-%QVz8aODh!1B1F#F94jOlWs?+&LdmiJMvID)L|%uj^=kkB_VxCk)|&6<^XsX(
zr|0Q&_x!H=zOU<^$KTaus>FroVmfBDILO_1q{Jt{e~6ca)tqqOAQA}P9(jgB?uW;B
z7>Kwmo`K<;7YzkMfgy{_6B&vOSvuTi{VV<#0{r~NzB)Rr>0Z;ly(QCTzWMT}?`x|4
zGbOI4Iay+=4Fx@K6SMXsA>?OEFpXSJj)$)xA(l1}Z5068%oMn6aFiHQ7R;<30z3Rf
zurTr@SsAMaYMCtHtQDJ7UmXGIm0}p5n?$01=m~Sh8=27jWHREe1dd7jLm%h!#He&6
zOf-rir4n`M9pVhy_P0p5qYZ2_>_mL61u)A+0BqGMMCZ&5u$Xd}JkIch#V`Ds(+MVI
zQ#EB?pV0@0VVclpsWPnb=P{j=)W9Zl5AjZA!F9d=VcN~o0_hzNsP8N?-plC)_d>lO
z)5RLv*USfH{{R?}Vht_b-+wp!|K0x++`XoG`ujC+1Zc|tLhi@@H{=_N<o^FW35Dj=
z^mePs@)z2Wm{CbOxmb~&r_#vo*vZUnQbJ<g=EBc;ub7(omCSWD29`N$KyFY48RK5a
zSR9*9dbxSPu_<|E&0rO{<a6FQ@zDk{``|5-mCyr})V>8vt`3+c3!t^68&Rkua68!n
z3XFThSJQ73wn+~%YLPl53{!&q2tU|fQAJXAzlk$GDlq)09cVo9Vura4`t0LLz5l0q
z&v0{>{2m3Uhrhe;*FJ7yfA=BY0sjB(HJb82U-*&#`8<J0Aoss4Y077TANWl2yU#rT
z51&=2^p&p2)szyGw$jH>yGqM1b(HFOiKHROk!D<COWCu!N>4oLBLx$Yba1qmbPcbU
zG*e4KnniV`oP2F*=JuXa{%TdJ(L@s|U$2X_rTgcf%Afk*RP6d~pxD>Vd`@%N|39Dq
z^Lczjx&K>;(D;8pvOfG-_(lnDWbnZ+)W7f0Up(IjuZN0$Ki`M^7w6lW`1|>e{PpvF
z*?jDOdw0Kp8U7aDzHj~CEbjj#|8otwa{tQ$PUS^1cKK3(Rhi)AXa|Y9CJ@_e8#DIu
zB-m4u4ri~AfexJRz%iY45*;5At29Mmp9+WmW_;q1#5$!}tqfhm7c<YIy%{SNOBgS*
zhOU*1A?bT<SQL4j47jlx7OC2kGtQe}@#14}3REF*bt-Y$773xbkCUafVKC;X2TV*c
z0BJ}Hxum1OTzs_#Scj|_?p80@ky^@JNLrF|V`)1$D&fQP>9fd?0|Usp9l9iA8|#$H
z#&&RLc`R8c);U$VHwHozwk7M%cLIkf4qQ5x1BZWH0+yq<!0yzAjL-cT($f9?cjJx!
zUA-l<`EGAwz`JkEN4fh-+-Edr2B1m*3;yswkH>$z{ulCva{tSM9}Cn;l%ze3%UVtr
z%?W}ol>^~b@j~)#ZV{7l!~%kCtsv;RDU8#Y4&O!T!4IM8pt@lUS&_zv&S`!SbHx=T
zP77e=Iy=x=p$h5j60)+^74Ci&1Q#B3g<g^qWZ#UPWPKMG7<_RuaqedVGx;aT6`>7G
z7F&T*xB-lmXhTifOEP!x1b8({1Z=B&B-f<QB;v(!vX)~Ev*$l0GbS7)CpmrLxp6D<
z(~m>RkwYd>*Qqs;nw}&v3Eg3a-etz!(+19}`GR#!@Mpg+mlciu|851)tO>v-{Lka@
z1Rwj~K*W>#Ulx36`TxIrKJm2J9NJk5VgBn;F#TKy;Lt!AeNcm#-+W>+O+^zdqaH98
zo_?U6wu`(lvmn+UrR2NlZy}B?hJx|m5#f--<cH_OiPjAdP`)JqYsKN9H^T!g11&)~
zwHr8hSW6}=X%U4@`-tt=day;Xj&U6$W-^0zlAHluq1y&0xL&iGgc<Qjp@|BK*|q?z
zFBp=LwJ{`ot~#U*>}GO5!UIC|gJD-h2Y9eKj<E@eB{sps<uUg&<iq&?`x?+o>}TmK
zZmz{2tLgb4m-o^9kI&^B%HzL`$e`~f=S!rq|IOab0^jF>nkNP{x&CY9f8N`J-2XCw
z?rKA~Z&ab3hjpT3?AlY8d+PMiJ}o-1swZ7NUyUlC{tXxIXh%bSXif8umf<vS6<X!2
zLFZm+Pis~y(6n9cX^5Z$HC?JgUvOD;Qn?Z>wo|0%eU#|cT?+JqH=9;lvZ+QPi$=!Q
zV&23Dm^ZZqd29u`Bv^^g=e)ps(N!4XbQgn4ui<lXItqhR(Rb;uXrZq}#gg+_mR^i|
zjN4F^@&Yts7op3*N0{~KC4SSj3UzWWqVB>o*wX#@xzgDG@3w$7>jdCGUjOmALcToz
zD-&)z93_J2T<{&*5$pAH*fW?C(g*nKl+ogT6e>Kw(4f+vf54|C3nor6MD_AQJR7J>
z>m`N~d{$?JpW2ojpo8XCyYb4^ng%`Fv=@76_rrC1#;{I}g`LZ$HRy5QVfZ5DA{fUc
zLp%tvgR8hf<?0IPxq=I;%e$es#X5AqpVOco<%{v!(#K%_a0XUN%Tcw59<A52)7R1R
zF~Lu5`zopfcGq2pk>B5HP@U0we3aP{-K%#3yGno_(eoN~WcvV2bXW!PbLT^hL>-6E
z5;Z8Q?_-$C?hJa09NhB#c1*X+YS7^^i!oY%JG8yvgtm(+Fe!B)t(R}F`)Jk13O}_?
z(_0H!S$pu>=obwdz4m83*s2dEty&9iHCnjeAgn>5(Uw?I_)xz7`wVFu|KI2Tn{xnY
za{gz~X#FP=$m73E2#cReQ&ipP5VkvY$n&IgS2@#(%iL&$Xe<@0jinLZ<LImr#&mfr
zQ>qX#lD^(DmZmYbG*M*~&23{w&t#0C1pzkHA<~Az96Op4WKZwh8AcV&M3lL1M9<ck
zQ}Z}8>T?&VEU(ql{rb7^hyVF4R{wiD*=tn)lkfkP2}K?ILejH6M18ClaDwc?K2-;H
zepkV)a$ZdWqviptOA%3O)d|MeC_;q0H3>Q}fIL7JC@h!{qmG{=YtD%Pbp%B3Tm<uE
znl1_S?nmwzMUqH&6{vYK0{G5i(9rJ#ea)R<W2bIV7R>{%#VW9Nhb!238P6mwumYPs
zok`kEAI4_KEau!T2PU>_8;G5Bk*GWffl9}lOzD>0Bx6<x+;q++qk2yT^&{%AUdbO$
z?wJZ^A(o&&vNbbEG8?WawuM1w9f+6W6sTV6P7+_XCsuohfw6TEOz5Hl5y|&RXzv7*
z_Pw>>{>$0WDLxRKH6)POgCTyw+HlSLGSRztoK!#T!qgFGm_zizuI~}Xfa3}q##+Jg
zfEc1M-wt%kTEUvlaV8PZib%gj#h>+_g2w)rdHs*i|9Jd=TmO^ie`JEZ{wJ^hHCO%b
zpSqXCXT2EG4(NchKmz&8hk;;@A+SvKn9~Y(h~Kvn<fp~^h>)|7X?@ld_Pex(X~PGA
z^<-VpE<H(fP81XMuq$L<RTa@ZF&Er;r^s%LQ6`10W<c9jE-*kdo=mgV1x?{mGUt&a
zIUnan)N}GlrQKS_ZF7Iv<xhdTMFQ=PNlAg?dNR*Dg=iIzgD!_WAmVHRSvgP5SbA>|
zlgL#k9s75K8l?heSikkeN8g+2r=|^_S;tL^ik-mt+9nbMrjW(wKs);$O#Ck#DBGt8
zgJX?J|30Ik=b0X0107&;e_aUKI~1<=pG~$Wj07#yNpRJ6I4om$P<yq4C{CUPTl5D4
z9*iYJcea7(&Y2`>Qk}`!eVw3NtRtlV=m$rx4uKxEQA~1G4$0m+h*5Ut6K)Czrm}ft
zYI}9aSgQczB+d}$X#{C@(_m$vm1MWqB63>W0F1Id!0hr|c`iXld^rBMT>ZCE{#X9_
zKbc`=`iJCvX*Bk~%<I2AUZecqyONOH|1!YLp&H|@Dp16^jU%1&@O(fa1}DA7sB3k&
zpgW7McTLA@TG!D3csgF*9Ea8YlTbXg1a*EZM%S=i$P?vaFQY=t{_P2dU$}}vTXXT3
zSuAR|=qYApCF9Ty7jfy%atuzW#j?xlRCjbCZuzD)H7QY`7K*>3=8i{rZ+#~E_Nl<2
zkx%f3e>Sq0<m1;bZsPu*DzVPF2#0WQW1k~6C{cKbc|PTMJ+>TQb}dBT>PPr`=1t_-
z*5Zf}RakKAF~&S-OZj$BaeGh>ns%<mipj6hXVXJG6;_1#p%3u3NhxM4NHKcMRn!W3
zim9<rFl1ga-l(cZrLQY7F#Z&}7hb{6k4tcQMH;F}ZeoRBCNf)J;`qQ?v{0?X5t+&~
zDM5t_H`ieI#Ts<sK@OEJZ$%HU`wcr@e1P3ot5EA24Qd(j3bo71aAD-HSea9SA$RZN
zrZstZIOilj^1Fx@lj?9lQ3`Hovm5pL9>?6tk8xx8BUEyFh34zobeXLZHK;1bNGnyk
zU}XnN`zX_=wr%O~CC_ofR~@P9srK~c7-d@1<`o{@mx&R^9C{;Fg^HfnV35TH{HA9)
z_EW9GIR@8pnbAYMAjn4Vz(Q<w^d$y+Dp7H<It}@TO>39d;Gqh2TI;7t2N@Nk_ql3J
zD{W0v-(0i9flhR=`fJ?&q#S*w7NXT_1uDAPnkML!VBYaOJRr)$cEVEJ5t4;al!0u`
zBN%6!jxBv(%lT_Gj{h>R|2NM6ysiJs_y5TRWAhV`tgMI6F0X3P#kQs>CS{<SvKWJ}
zUBVkTv}nEd*B-_NsW#|H)?lnAo4Q?gXpoUWl?DW?{p7YE3#{;&z7)M4w4?P}=AMAZ
zM;YMFHup*VAw|?KaBa|{>lT=An*`ylvS5p~8}^v6yg{?#xahHRBy9Gdh$p*dVdAp(
zv|c^0?#Ibbd*ZXQ4akq_LD%KEG$^X5BR%Kw{U^8ezh#3O+GkN^V0T)t1#a2++{+20
z^Ao|=(Fs#G>}ZfMV<t|IF9IodIK+4k!D}gz4GO97$L<45;mRFFRF)pV(d=%tUUAzW
z<0^-RXqzv_#EuFyrR}r^=|>Eq9e+Lh$!$8;2H3q~KaT6)i`MI>-FGp3(h97(p8)BZ
z`dIQTszLTOlTo>kD$4IA`MuO!_fiXo#V>}&<G<|pe+apa@;`6iE+7A8K+5cwv~~9v
zw9e@P&G_XNJ(cs64jNWM+q}3#<69NcEgHELr~g9p<FhDD&ZY&zEV}UZd77h`My&)V
zXzwd2)aZILtvQ!X{naj0`=4&m*A}Pg-udZNaOM)7xH+3%jJQlo_*dya^;#|cKm2ca
z<9`oV7oLlSS@ZAvsHds?kD$@#KSf-5{Z}@~YnSrcrM!0ezINHtUTg{ei2tL-Q~!U$
zf78$Z6MY>2g(8uGJpRjq&vCy^OZVA7i4XnX-249OY103V_WueEdGh`LGU4yG|F5O{
z<4dF7|L=1@7H043{?yffjJKp&9{$Vvgz=xxHIzU9S2oo24MRhfO=z<)4D+W%;H{h>
zRNm%<XUtaMh$9<Nr_2*Ij@TpldKsp(EK#YQ13s#ZLLVznbU5#f-Fhy;kXGKfn;ncp
zEZtGLw;e{02*sS~E0OK!h|En9PMG9{Q}oTzI@=1jIfh}%DjOVYvI6%_*^3KOHe$}b
zHTe4bVBD%3hKG}4@kx0M>W8ny`v&G%dub`o)S8FIFT?S-3mY-j&I4Pz&;CjLx&Oy`
zy}8l7WcB>d#{TDV|JC04z(iTbaePsz0baL8Cm>NmizOw3$Dh$X&+lwVkZU!TR$8lr
zAz8q=3lHwNW9X9krxGL5X*ws+m|KdNBrT^ctybxVxELe>(;4VyP=R8B`P)3Zf582L
z=f&gq*`C33_ul)yyyJVn-}}7Z=l3k4_%DRiauZyCpcE1-Oz={&86x_O@Z|X-F!q_?
z=w&l>=?pNxMGs$VP4Jw2EoAu>!3kL@#AF%Z{guVAUv7fPnMPRuw-Nq)&;TL3^bn*i
zfv!L^tU6o-VWB$MXEwlV+Y6y?#ag(y#RTst^l<7A6R3`s!r_0+@X<OmC`OFXv%v&k
zohpSoZ7KA~%uu_d5JEI2I38XC$9qlC^sgCCFECJ!FoBGsf9<Nme9gpE|MpD(3FQA3
z!n|)PS$3vBFV|JE?5CU9!0rrIdGMO*cUc{4?!3k}oDXC5XWLb?WEHG^Nu%mxW2b6x
z_&%PpG?XnEUd}$f*up+3+02%0sbR6b`E1Glr`SMLBWpS|oAY;<uw8FoQpvWNS;M*6
zth*?L$uIP?boLmlR^_l?6ajos^8xm4_y;Wgi9oiaCWFlklJlQ_?Pj&-3Rr)9fhw(h
zG22ruV~XNzri(eqt_=RDlKPtY61VifMyt(RU65;40^r5lucT`9IXX>lj+YOE$DRN0
zQ2%KWE(t>V7eX11r@{Zt(zjlR=gxaByyk3wB91|Lf=l}MG7lIx{x7uGf4q|PF9v<y
z0p8L&3xcn3=-T=!$P>Qcdz<C(^GF74tWSa8q6hhVC)@bT4S|sUOe}=9`@#0goxEf7
zZeEeb;AX7?Zah90^h@uBFPl5~&V4d?Qx^hPX9UBFlR@yAwu)c6zn;f7`a|2XXvmqH
z$)Al_&fC_V;l@ofD3_O*vGt#ur_Ivjz2RjV;C}p9CeZpHVhCuBV!qI%zK(Y*?&#ms
zG+_MszYh67t^X^CU_%hR5H&*i^EAaJ{pV|ny%_@V;Ql{JN&cTHLYV|Px)jHpp17)i
zZ-xLoi2rd)@_!2f{XOXallnXeX&77oUIzZ#xF7%HmGKn+1u|;9Ro{9~ZM&WIwl<$r
zTlLkWY-6`?>z~&9F0*^j{xf&Bqwg&*8CU-k-~n)d{68*{^e+bV{#ty#*wEu^-A9iv
zdVVebUi9{2!#;X@(Z`D(UTo;$#XjZ}*l<1pdU~z<IF|tD5#ZbcY*PRI(;Zv=Pm2BD
z{rDf3kVwz}#4y$T|DMiAlKMK{t=Q;4UK;;TqW%0I^#9QM--7TJ|BtU6Lo6=of0FP2
zmCpP>l>ZAsYX8rX;V|f(&tT*Fd9dl`99Wqg4Sk;{!}B+yK@;#0WF{Qp4X;GPiPIPO
z_lq-m_3F#Ks{9c!)U@)2Il+AITXW!0PXzRr`9svPX8zq@*ZI}XH1PXS4X?L`LeIb%
zKJ;)5oL!@WTK*{LmUZ*gp>Rm~K>_zf22A_BVQl@2@&7o+e`P`v#eX44?c1Ryt&hna
z{ZE?z$1(p;qV?a!VDbNSY#rkA5{ie@4VUyk$^IWF_y0*W{x?<d1^>@fdrmlY&c{{#
zPpbdNQU7sN|5XeY{}0ye!rG2fpAR9HvGp&;|Kk|{<CRqZLnJ=y|FLYZ9{rqbejjBU
z`}1>#!#S&+@5BB$&W<IHarH02|Km9RAD2k$e~UqC{}0v-#oWVEGt|D1wL`H^C^oDe
zigiLQeM^l{tY3`{>sMp#YWqIcvc}v78`iPLn$?!R)Yo5XZ1pdV|HmQzqyLA-|ApWy
m{vTgi|6A(+2ZA67f*=TjAP9mW2!bGdhyMTr`%D=C$N&H#Fs5bz

-- 
2.7.4