1 Basic knowledge of Neural Network

1.1 A Single Neuron

(From link)

  1. Inputs: \(X_{1},X_{2},\ldots,X_{m}\), \(+1\) intercept

  2. Output: \(h_{W,b}(X_{1},X_{2},\ldots,X_{m})\) where \(W,b\) are parameters

  3. Activation: \(h_{W,b}(X_{1},X_{2},\ldots,X_{m}) =f(\sum_{i=1}^{m}w_{i}X_{i}+b) =f(W^{\prime}X)\), where \(W=(w_{1},w_{2},\ldots,w_{m},b)^{\prime}\) and \(X=(X_{1},X_{2},\ldots,X_{m},1)^{\prime}\).

1.2 Activation Function \(f\)

1.Identical function

library(ggplot2)
demoDT = as.data.frame(cbind(seq(-5,5,0.01), sapply(seq(-5,5,0.01), function(x) {return(x)}), sapply(seq(-5,5,0.01), function(x) {return(1)}))) 
colnames(demoDT) = c("x","y", "z")
ggplot(demoDT, aes(x=x, y=value), show.legend = TRUE) +
geom_line(aes(y=y), show.legend = TRUE, col = "red") + 
  geom_point(aes(y=z), show.legend = TRUE, col = "green")

  1. Sigmoid (logistic) \[f(z) =\frac{1}{1+\exp(-z)}\]
library(ggplot2)
demoDT = as.data.frame(cbind(seq(-5,5,0.01), sapply(seq(-5,5,0.01), function(x) {return(1/(1+exp(-x)))}), sapply(seq(-5,5,0.01), function(x) {return((1/(1+exp(-x))*(1-1/(1+exp(-x)))))}))) 
colnames(demoDT) = c("x","y", "z")
ggplot(demoDT, aes(x=x, y=value)) +
geom_line(aes(y=y), show.legend = TRUE, col = "red") + 
  geom_point(aes(y=z), show.legend = TRUE, col = "green")

  1. tanh \[f(z) =\tanh(z) =\frac{\exp(z)-\exp(-z)}{\exp(z)+\exp(-z)}\]
demoDT = as.data.frame(cbind(seq(-5,5,0.01), sapply(seq(-5,5,0.01), function(x) {return((exp(x)-exp(-x))/(exp(x)+exp(-x)))}), sapply(seq(-5,5,0.01), function(x) {return(1-((exp(x)-exp(-x))/(exp(x)+exp(-x)))^2)}))) 
colnames(demoDT) = c("x","y", "z")
ggplot(demoDT, aes(x=x, y=value)) +
geom_line(aes(y=y), show.legend = TRUE, col = "red") + 
  geom_point(aes(y=z), show.legend = TRUE, col = "green")

  1. The rectified linear function \[f(z) =\max(0,z)\]
demoDT = as.data.frame(cbind(seq(-5,5,0.01), sapply(seq(-5,5,0.01), function(x) {return(max(x,0))}), sapply(seq(-5,5,0.01), function(x) {if(x>0) {return(1)} else {return(0)}}))) 
colnames(demoDT) = c("x","y", "z")
ggplot(demoDT, aes(x=x, y=value)) +
geom_line(aes(y=y), show.legend = TRUE, col = "red") + 
  geom_point(aes(y=z), show.legend = TRUE, col = "green")

  1. Threshold function \[f(z) = \left\{ \begin{array}{ll} 1 & \mbox{if $z>0$} \\ -1 & \mbox{if $z\leq0$} \end{array} \right.\]
demoDT = as.data.frame(cbind(seq(-5,5,0.01), sapply(seq(-5,5,0.01), function(x) {if(x>0) {return(1)} else {return(-1)}}), sapply(seq(-5,5,0.01), function(x) {if(x>0) {return(0)} else {return(0)}}))) 
colnames(demoDT) = c("x","y", "z")
ggplot(demoDT, aes(x=x, y=value)) +
geom_line(aes(y=y), show.legend = TRUE, col = "red") + 
  geom_point(aes(y=z), show.legend = TRUE, col = "green")