/home/rex/ebltrunk/core/libeblearn/include/ebl_nonlinearity.h

/***************************************************************************
 *   Copyright (C) 2008 by Yann LeCun and Pierre Sermanet *
 *   yann@cs.nyu.edu, pierre.sermanet@gmail.com *
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *     * Redistributions of source code must retain the above copyright
 *       notice, this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above copyright
 *       notice, this list of conditions and the following disclaimer in the
 *       documentation and/or other materials provided with the distribution.
 *     * Redistribution under a license not approved by the Open Source
 *       Initiative (http://www.opensource.org) must display the
 *       following acknowledgement in all advertising material:
 *        This product includes software developed at the Courant
 *        Institute of Mathematical Sciences (http://cims.nyu.edu).
 *     * The names of the authors may not be used to endorse or promote products
 *       derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED
 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL ThE AUTHORS BE LIABLE FOR ANY
 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ***************************************************************************/

#ifndef EBL_NONLINEARITY_H_
#define EBL_NONLINEARITY_H_

#include "ebl_defines.h"
#include "libidx.h"
#include "ebl_states.h"
#include "ebl_basic.h"
#include "ebl_arch.h"

namespace ebl {

  template <typename T, class Tstate = bbstate_idx<T> >
    class stdsigmoid_module: public module_1_1<T,Tstate> {
  public:
    stdsigmoid_module();
    virtual ~stdsigmoid_module();
    virtual void fprop(Tstate &in, Tstate &out);
    virtual void bprop(Tstate &in, Tstate &out);
    virtual void bbprop(Tstate &in, Tstate &out);
    virtual stdsigmoid_module<T,Tstate>* copy();
  protected:
    idx<T> tmp; 
  };

  template <typename T, class Tstate = bbstate_idx<T> >
    class tanh_module: public module_1_1<T,Tstate> {
  public:
    tanh_module();
    virtual ~tanh_module();
    void fprop(Tstate &in, Tstate &out);
    void bprop(Tstate &in, Tstate &out);
    void bbprop(Tstate &in, Tstate &out);
    virtual tanh_module<T,Tstate>* copy();
  protected:
    idx<T> tmp; 
  };

  template <typename T, class Tstate = bbstate_idx<T> >
    class softmax: public module_1_1<T,Tstate> {
  public:
    double beta;

    // <b> is the parameter beta in the softmax
    // large <b> turns the softmax into a max
    // <b> equal to 0 turns the softmax into 1/N

  private:
    void resize_nsame(Tstate &in, Tstate &out, int n);

  public:
    softmax(double b);
    ~softmax() {};
    void fprop(Tstate &in, Tstate &out);
    void bprop(Tstate &in, Tstate &out);
    void bbprop(Tstate &in, Tstate &out);
  };

  // abs_module
  template <typename T, class Tstate = bbstate_idx<T> >
    class abs_module: public module_1_1<T, Tstate> {    
  public:
    abs_module(double thresh = 0.0);
    virtual ~abs_module();
    virtual void fprop(Tstate &in, Tstate &out);
    virtual void bprop(Tstate &in, Tstate &out);
    virtual void bbprop(Tstate &in, Tstate &out);
    virtual abs_module<T,Tstate>* copy();
  private:
    double threshold;
  };

  // linear_shrink_module
  template <typename T, class Tstate = bbstate_idx<T> >
    class linear_shrink_module: public module_1_1<T, Tstate> {
  public:
    linear_shrink_module(parameter<T,Tstate> *p, intg nf, T bias = 0);
    virtual ~linear_shrink_module();
    virtual void fprop(Tstate &in, Tstate &out);
    virtual void bprop(Tstate &in, Tstate &out);
    virtual void bbprop(Tstate &in, Tstate &out);
    virtual linear_shrink_module<T,Tstate>* copy();
    virtual std::string describe();
  protected:
    Tstate bias;
    T default_bias;
  };

  // smooth_shrink_module
  template <typename T, class Tstate = bbstate_idx<T> >
    class smooth_shrink_module: public module_1_1<T, Tstate> {
  public:
    smooth_shrink_module(parameter<T,Tstate> *p, intg nf, T beta = 10,
                         T bias = .3);
    virtual ~smooth_shrink_module();
    virtual void fprop(Tstate &in, Tstate &out);
    virtual void bprop(Tstate &in, Tstate &out);
    virtual void bbprop(Tstate &in, Tstate &out);
    virtual smooth_shrink_module<T,Tstate>* copy();

  public:
    Tstate beta, bias;
  private:
    Tstate ebb, ebx, tin;
    abs_module<T,Tstate> absmod;
    T default_beta, default_bias;
  };

  // tanh_shrink_module
  template <typename T, class Tstate = bbstate_idx<T> >
    class tanh_shrink_module: public module_1_1<T, Tstate> {
  public:
    tanh_shrink_module(parameter<T,Tstate> *p, intg nf, bool diags = false);
    virtual ~tanh_shrink_module();
    virtual void fprop(Tstate &in, Tstate &out);
    virtual void bprop(Tstate &in, Tstate &out);
    virtual void bbprop(Tstate &in, Tstate &out);
    virtual tanh_shrink_module<T,Tstate>* copy();
    virtual std::string describe();
  protected:
    intg                         nfeatures;
    Tstate                       abuf, tbuf, bbuf;
    diag_module<T,Tstate>       *alpha, *beta;
    tanh_module<T,Tstate>        mtanh;
    diff_module<T,Tstate>        difmod;        
    bool                         diags; 
  };

} // namespace ebl {

#include "ebl_nonlinearity.hpp"

#endif /* EBL_NONLINEARITY_H_ */