Add Activations

tensorflow-framework/src/main/java/org/tensorflow/framework/activations/Activation.java

@@ -0,0 +1,68 @@
+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+=======================================================================*/
+package org.tensorflow.framework.activations;
+
+import org.tensorflow.Operand;
+import org.tensorflow.op.Ops;
+import org.tensorflow.types.family.TNumber;
+
+/**
+ * Abstract base class for Activations
+ *
+ * <p><b>Note:</b> The {@link #tf} attribute must be set prior to invoking the call method. See
+ * {@link #setTF(Ops)} and the constructor {@link #Activation(Ops)}.
+ *
+ * @param <T> the data type of the activation
+ */
+public abstract class Activation<T extends TNumber> {
+
+  /** The TensorFlow Ops */
+  protected Ops tf;
+
+  /**
+   * Creates the abstract class for an Activation
+   *
+   * @param tf the TensorFlow Ops
+   */
+  protected Activation(Ops tf) {
+    this.tf = tf;
+  }
+
+  /**
+   * Sets the TensorFlow Ops
+   *
+   * @param tf the TensorFlow Ops
+   */
+  protected void setTF(Ops tf) {
+    this.tf = tf;
+  }
+
+  /**
+   * Gets the TensorFlow Ops
+   *
+   * @return the TensorFlow Ops
+   */
+  protected Ops getTF() {
+    return this.tf;
+  }
+
+  /**
+   * Gets the calculation operation for the activation.
+   *
+   * @param input the input tensor
+   * @return The operand for the activation
+   */
+  public abstract Operand<T> call(Operand<T> input);
+}

tensorflow-framework/src/main/java/org/tensorflow/framework/activations/ELU.java

@@ -0,0 +1,103 @@
+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+=======================================================================*/
+package org.tensorflow.framework.activations;
+
+import org.tensorflow.DataType;
+import org.tensorflow.Operand;
+import org.tensorflow.op.Ops;
+import org.tensorflow.types.TBool;
+import org.tensorflow.types.family.TNumber;
+
+/**
+ * Exponential linear unit.
+ *
+ * <p>The exponential linear unit (ELU) with <code>alpha &gt; 0</code> is:
+ *
+ * <p><code>x</code> if <code>x &gt; 0</code> and <code>alpha * (exp(x) -
+ * 1)</code> if <code>x &lt; 0</code>.
+ *
+ * <p>The ELU hyperparameter <code>alpha</code> controls the value to which an ELU saturates for
+ * negative net inputs. ELUs diminish the vanishing gradient effect.
+ *
+ * <p>ELUs have negative values which pushes the mean of the activations closer to zero. Mean
+ * activations that are closer to zero enable faster learning as they bring the gradient closer to
+ * the natural gradient. ELUs saturate to a negative value when the argument gets smaller.
+ * Saturation means a small derivative which decreases the variation and the information that is
+ * propagated to the next layer.
+ *
+ * <p>Example Usage:
+ *
+ * <pre>
+ *     Operand&lt;TFloat32&gt; input = &#46;&#46;&#46;;
+ *     ELU&lt;TFloat32&gt; elu = new ELU&lt;&gt;(tf, 2.0f);
+ *     Operand&lt;TFloat32&gt; result = elu.call(input);
+ * </pre>
+ *
+ * @param <T> the data type of the activation
+ * @see <a href="https://arxiv.org/abs/1511.07289">Clevert et al, 2016, Fast and Accurate Deep
+ *     Network Learning by Exponential Linear Units (ELUs)</a>
+ */
+public class ELU<T extends TNumber> extends Activation<T> {
+
+  private static final double ALPHA_DEFAULT = 1.0;
+
+  /** A scalar, slope of negative section. */
+  private final double alpha;
+
+  /**
+   * Creates a new ELU with alpha={@link #ALPHA_DEFAULT}.
+   *
+   * @param tf the TensorFlow Ops
+   */
+  public ELU(Ops tf) {
+    this(tf, ALPHA_DEFAULT);
+  }
+
+  /**
+   * Creates a new ELU
+   *
+   * @param tf the TensorFlow Ops
+   * @param alpha A scalar, slope of negative section. It controls the value to which an ELU
+   *     saturates for negative net inputs.
+   */
+  public ELU(Ops tf, double alpha) {
+    super(tf);
+    this.alpha = alpha;
+  }
+
+  /**
+   * Gets the calculation operation for the activation.
+   *
+   * @param input the input tensor
+   * @return The operand for the activation
+   * @throws IllegalArgumentException if the data type is not a floating data type.
+   */
+  @Override
+  public Operand<T> call(Operand<T> input) {
+    if (!input.asOutput().dataType().isFloating()) {
+      throw new IllegalArgumentException(
+          "Must be a Floating Point DataType: " + input.asOutput().dataType());
+    }
+    Operand<T> result = tf.nn.elu(input);
+    if (alpha == 1.0) {
+      return result;
+    } else {
+      DataType<T> dataType = input.asOutput().dataType();
+      Operand<T> y = tf.math.mul(result, tf.dtypes.cast(tf.constant(alpha), dataType));
+      Operand<TBool> cond = tf.math.greater(result, tf.dtypes.cast(tf.constant(0), dataType));
+      return tf.select(cond, result, y);
+    }
+  }
+}

tensorflow-framework/src/main/java/org/tensorflow/framework/activations/Exponential.java

@@ -0,0 +1,62 @@
+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+=======================================================================*/
+package org.tensorflow.framework.activations;
+
+import org.tensorflow.Operand;
+import org.tensorflow.op.Ops;
+import org.tensorflow.types.family.TNumber;
+
+/**
+ * Exponential activation function.
+ *
+ * <p>For example:
+ *
+ * <pre>
+ *   Operand&lt;TFloat32&gt; input = tf.constant(
+ *          new float[] {-3.0f,-1.0f, 0.0f,1.0f,3.0f});
+ *   Exponential&lt;TFloat32&gt; exp = new Exponential&lt;&gt;(tf);
+ *   Operand&lt;TFloat32&gt; result = exp.call(input);
+ *   // result is [0.04978707f,  0.36787945f,  1.f,  2.7182817f, 20.085537f]
+ * </pre>
+ *
+ * @param <T> the data type of the activation
+ */
+public class Exponential<T extends TNumber> extends Activation<T> {
+
+  /**
+   * Creates an Exponential activation.
+   *
+   * @param tf the TensorFlow Ops
+   */
+  public Exponential(Ops tf) {
+    super(tf);
+  }
+
+  /**
+   * Calculates the Exponential activation.
+   *
+   * @param input the input tensor
+   * @return an Operand for the exponential activation: <code>exp(x)</code>.
+   * @throws IllegalArgumentException if the input is not a floating type
+   */
+  @Override
+  public Operand<T> call(Operand<T> input) {
+    if (!input.asOutput().dataType().isFloating()) {
+      throw new IllegalArgumentException(
+          "Must be a Floating Point DataType: " + input.asOutput().dataType());
+    }
+    return tf.math.exp(input);
+  }
+}

tensorflow-framework/src/main/java/org/tensorflow/framework/activations/HardSigmoid.java

@@ -0,0 +1,79 @@
+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+=======================================================================*/
+package org.tensorflow.framework.activations;
+
+import org.tensorflow.DataType;
+import org.tensorflow.Operand;
+import org.tensorflow.op.Ops;
+import org.tensorflow.types.family.TNumber;
+
+/**
+ * Hard sigmoid activation.
+ *
+ * <p>A faster approximation of the sigmoid activation.
+ *
+ * <p>Defined as:
+ *
+ * <ul>
+ *   <li><code>if x &lt; -2.5: return 0</code>
+ *   <li><code>if x &gt; 2.5: return 1</code>
+ *   <li><code>if -2.5 &lt;= x &lt;= 2.5: return 0.2 * x + 0.5</code>
+ * </ul>
+ *
+ * <p>For example:
+ *
+ * <pre>
+ *     Operand&lt;TFloat32&gt; input = tf.constant(
+ *              new float[] {-3.0f,-1.0f, 0.0f,1.0f,3.0f});
+ *     HardSigmoid&lt;TFloat32&gt; hardSigmoid = new HardSigmoid&lt;&gt;(tf);
+ *     Operand&lt;TFloat32&gt; result = hardSigmoid.call(input);
+ *     // result is [0.f , 0.3f, 0.5f, 0.7f, 1.f]
+ * </pre>
+ *
+ * @param <T> the data type of the result
+ */
+public class HardSigmoid<T extends TNumber> extends Activation<T> {
+
+  /**
+   * Creates Hard sigmoid activation.
+   *
+   * @param tf the TensorFlow Ops
+   */
+  public HardSigmoid(Ops tf) {
+    super(tf);
+  }
+
+  /**
+   * Gets the calculation operation for the activation.
+   *
+   * @param input the input tensor
+   * @return The operand for the activation
+   * @throws IllegalArgumentException if the data type is not a floating data type.
+   */
+  @Override
+  public Operand<T> call(Operand<T> input) {
+    if (!input.asOutput().dataType().isFloating()) {
+      throw new IllegalArgumentException(
+          "Must be a Floating Point DataType: " + input.asOutput().dataType());
+    }
+    DataType<T> dataType = input.asOutput().dataType();
+    Operand<T> point2 = tf.dtypes.cast(tf.constant(0.2), dataType);
+    Operand<T> point5 = tf.dtypes.cast(tf.constant(0.5), dataType);
+
+    Operand<T> x = tf.math.add(tf.math.mul(input, point2), point5);
+    return tf.clipByValue(
+        x, tf.dtypes.cast(tf.constant(0), dataType), tf.dtypes.cast(tf.constant(1), dataType));
+  }
+}

tensorflow-framework/src/main/java/org/tensorflow/framework/activations/Linear.java

@@ -0,0 +1,52 @@
+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+=======================================================================*/
+package org.tensorflow.framework.activations;
+
+import org.tensorflow.Operand;
+import org.tensorflow.op.Ops;
+import org.tensorflow.types.family.TNumber;
+
+/**
+ * Linear activation function  (pass-through).
+ *
+ * <p>The linear activation returns its input. It is also known as the Identity activation function.</p>
+ *
+ * <p>For example:
+ *
+ * <pre>
+ *    Operand&lt;TFloat32&gt; input = tf.constant(
+ *              new float[] {-3.0f,-1.0f, 0.0f,1.0f,3.0f});
+ *    Linear&lt;TFloat32&gt; linear = new Linear&lt;&gt;(tf);
+ *    Operand&lt;TFloat32&gt; result = linear.call(input);
+ *    // result is [-3.0f,-1.0f, 0.0f,1.0f,3.0f]
+ * </pre>
+ */
+public class Linear<U extends TNumber> extends Activation<U> {
+
+  /**
+   * Creates a linear activation.
+   *
+   * @param tf the TensorFlow Ops
+   */
+  public Linear(Ops tf) {
+    super(tf);
+  }
+
+  /** {@inheritDoc} */
+  @Override
+  public Operand<U> call(Operand<U> input) {
+    return input;
+  }
+}