From fed613b5b995dc3c4801d50b936b0cef24e006fc Mon Sep 17 00:00:00 2001 From: Leandro Alvim Date: Sun, 26 Nov 2017 15:05:46 -0200 Subject: [PATCH 01/16] voted perceptron --- src/Perceptrons.jl | 13 +++++-- src/kernel_perceptron.jl | 9 +---- src/linear_perceptron.jl | 6 +-- src/types.jl | 59 ++++++++++++++++++++++++++++-- src/utils.jl | 17 ++++++--- src/voted_perceptron.jl | 79 ++++++++++++++++++++++++++++++++++++++++ 6 files changed, 159 insertions(+), 24 deletions(-) create mode 100644 src/voted_perceptron.jl diff --git a/src/Perceptrons.jl b/src/Perceptrons.jl index 67004be..b6b0aab 100644 --- a/src/Perceptrons.jl +++ b/src/Perceptrons.jl @@ -6,11 +6,13 @@ include("utils.jl") include("types.jl") include("linear_perceptron.jl") include("kernel_perceptron.jl") +include("voted_perceptron.jl") + """ - fit(X::Matrix{:=0 ? 1.0: -1.0 ) diff --git a/src/linear_perceptron.jl b/src/linear_perceptron.jl index 86efbdd..4fb52c8 100644 --- a/src/linear_perceptron.jl +++ b/src/linear_perceptron.jl @@ -1,7 +1,7 @@ -@inline function h(Θ,x) - return (Θ'*x >=0 ? 1.0: 0.0 ) -end +@inline sinal(x) = x>=0 ? 1.0 : 0.0 + +@inline h(Θ,x) = sinal(Θ'*x) function trainer{T<:AbstractFloat}(model::LinearPerceptron{T}, X::AbstractArray{T}, diff --git a/src/types.jl b/src/types.jl index a174c58..8406a35 100644 --- a/src/types.jl +++ b/src/types.jl @@ -40,6 +40,48 @@ function LinearPerceptron{T<:AbstractFloat}(X::AbstractArray{T}, size(X,2)) end +#################################################################################### + +#### Linear Perceptron type +mutable struct VotedPerceptron{T<:AbstractFloat} <: PerceptronModel{T} + α::T + Θ#::Dict{Integer,Vector{T}} + c#::Dict{Integer,Integer} + k::Integer + shuffle_epoch::Bool + random_state::Integer + max_epochs::Integer + last_epoch::Integer + history::Vector{Integer} + mx::Matrix{T} # mean stat after for z-scoring input data (X) + sx::Matrix{T} # standard deviation stat after for z-scoring target data (X) + centralize::Bool + nfeatures::Integer +end + +function VotedPerceptron{T<:AbstractFloat}(X::AbstractArray{T}, + alpha, + shuffle_epoch, + random_state, + max_epochs, + centralize) + + return VotedPerceptron(alpha, # I will refactor to a constructor. Cleaner + nothing, + nothing, + 0, + shuffle_epoch, + random_state, + max_epochs, + 0, + Vector{Integer}(1), + mean(X,1), + std(X,1), + centralize, + size(X,2)) + +end + #################################################################################### @@ -90,23 +132,32 @@ function Model{T<:AbstractFloat}(X::AbstractArray{T}, max_epochs, centralize, kernel, - width) + width, + mode) - if kernel == "linear" + if mode == "linear" return LinearPerceptron(X, alpha, shuffle_epoch, random_state, max_epochs, centralize) - elseif kernel == "rbf" + elseif mode == "kernel" return KernelPerceptron(X, max_epochs, centralize, kernel, width) + elseif mode == "voted" + return VotedPerceptron(X, + alpha, + shuffle_epoch, + random_state, + max_epochs, + centralize) + else - error("Invalid Kernel name: $(kernel)") + error("Invalid perceptron mode name: $(mode). \n Cadidates are: linear, kernel or voted") end end diff --git a/src/utils.jl b/src/utils.jl index 81134ce..ee8fadd 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -3,9 +3,15 @@ ## Auxiliary functions export acc -function acc(yt,yp) - count(x->x==true, yt .== yp)/length(yt) -end +@inline acc(yt,yp) = count(x->x==true, yt .== yp)/length(yt) + +# used in linear and voted perceptron +@inline sinal(x) = x>=0 ? 1.0 : 0.0 +# used in kernel perceptron +@inline sign(val) = (val >=0 ? 1.0: -1.0 ) +# use in linear and voted perceptron +@inline h(Θ,x) = sinal(Θ'*x) + ## checks PLS input data and params function check_data{T<:AbstractFloat}(X::Matrix{T},Y::Union{Vector{T},Matrix{T}}) @@ -25,8 +31,9 @@ function check_data{T<:AbstractFloat}(X::Matrix{T},nfeatures::Int) end -function check_params(kernel::AbstractString) - kernel == "rbf" || kernel == "linear" || error("kernel must be kernel='linear' or kernel='rbf'") +function check_params(kernel::AbstractString,mode::AbstractString) + kernel in ["rbf","linear"] || error("kernel must be 'linear' or 'rbf'") + mode in ["kernel","linear","voted"] || error("mode must be 'linear' or 'kernel' or 'voted'") end ## checks constant columns diff --git a/src/voted_perceptron.jl b/src/voted_perceptron.jl new file mode 100644 index 0000000..69a1e40 --- /dev/null +++ b/src/voted_perceptron.jl @@ -0,0 +1,79 @@ +@inline function vote(Θ,x,c,k) + + s = 0 + for j=1:k + s += c[j]*h(Θ,X[i,:]) # voting + end + s +end + +function trainer{T<:AbstractFloat}(model::LinearPerceptron{T}, + X::AbstractArray{T}, + Y::Vector{T}) + + shuffle_epoch = model.shuffle_epoch + random_state = model.random_state + max_epochs = model.max_epochs + + if random_state!=-1 + srand(random_state) + end + + n,m = size(X) + X = hcat(X,ones(n,1)) # adding bias + history = [] + nerrors,nlast_errors = Inf,0 + epochs = 0 + k,Θ,c,α = 0,Dict(0=>rand(m+1)),Dict(0=>0),model.α + while nerrors>0 && epochs < max_epochs + # stops when error is equal to zero or grater than last_error or reached max iterations + # shuffle dataset + if shuffle_epoch + sind = shuffle(1:n) + x = X[sind,:] + y = Y[sind] + end + nerrors = 0 + # weight updates for all samples + for i=1:n + xi = x[i,:] + ξ = h(Θ,xi) - y[i] + if ξ==0 + c[k] += 1 + else + nerrors+=1 + c[k+1] = 1 + Θ[k+1] = Θ[k] - α * ξ * xi + k += 1 + end + end + nlast_errors = nerrors + epochs+=1 + push!(history,nerrors) + end + if nerrors > 0 + warn("Perceptron: Not converged. Max epochs $(max_epochs) reached. Error history: $(history) \n Try to increase max_epochs or may be you have a non linear problem.") + end + model.Θ = Θ + model.c = c + model.k = k + model.history = history +end + +function predictor{T<:AbstractFloat}(model::LinearPerceptron{T}, + X::AbstractArray{T}) + + Θ = model.Θ + α = model.α + k = model.k + c = model.c + + n = size(X,1) + y = zeros(Real,n) + X = hcat(X,ones(n,1)) # adding bias + for i=1:n + y[i] = sinal(vote(Θ[i],X[i,:],c[i],k)) + end + y + +end From 9d2c1bc9f2b88ae778e4f55d36484617394c6f86 Mon Sep 17 00:00:00 2001 From: Leandro Alvim Date: Sun, 26 Nov 2017 15:06:50 -0200 Subject: [PATCH 02/16] fix --- src/voted_perceptron.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/voted_perceptron.jl b/src/voted_perceptron.jl index 69a1e40..2d9b156 100644 --- a/src/voted_perceptron.jl +++ b/src/voted_perceptron.jl @@ -2,7 +2,7 @@ s = 0 for j=1:k - s += c[j]*h(Θ,X[i,:]) # voting + s += c[j]*h(Θ,x) # voting end s end From bd056773c329ca7e4bab583cef70e47aefd1dacf Mon Sep 17 00:00:00 2001 From: Leandro Alvim Date: Sun, 26 Nov 2017 15:09:39 -0200 Subject: [PATCH 03/16] added an example with voted perceptron --- README.md | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 79210b7..e7c0d48 100644 --- a/README.md +++ b/README.md @@ -34,15 +34,22 @@ Examples println("[Perceptron] accuracy : $(acc(Y_train,Y_pred))") + # training a voted perceptron + model = Perceptrons.fit(X_train,Y_train,centralize=true,mode="voted") + Y_pred = Perceptrons.predict(model,X_test) + + println("[Voted Perceptron] accuracy : $(acc(Y_train,Y_pred))") + + # training a kernel perceptron (XOR) X_train = [1.0 1.0; 0.0 1.0; 1.0 0.0; 0.0 0.0] Y_train = [0.0 ; 1.0; 1.0; 0.0] X_test = X .+ .03 # adding noise - model = Perceptrons.fit(X_train,Y_train,centralize=true,kernel="rbf",width=.01) + model = Perceptrons.fit(X_train,Y_train,centralize=true,mode="kernel",kernel="rbf",width=.01) Y_pred = Perceptrons.predict(model,X_test) - println("[Perceptron] accuracy : $(acc(Y_train,Y_pred))") + println("[Kernel Perceptron] accuracy : $(acc(Y_train,Y_pred))") # if you want to save your model @@ -59,7 +66,6 @@ What is Implemented What is Upcoming ======= -* Kernel Perceptron * Multiclass Perceptron * Voted Perceptron * Average Perceptron From dd7234fb3658381f8aef8e43acbb413fae08b149 Mon Sep 17 00:00:00 2001 From: Leandro Alvim Date: Sun, 26 Nov 2017 15:10:07 -0200 Subject: [PATCH 04/16] added voted perceptron --- experiments/readme_example.jl | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/experiments/readme_example.jl b/experiments/readme_example.jl index 96d660c..1bcbb6d 100644 --- a/experiments/readme_example.jl +++ b/experiments/readme_example.jl @@ -10,15 +10,22 @@ Y_pred = Perceptrons.predict(model,X_test) println("[Perceptron] accuracy : $(acc(Y_train,Y_pred))") +# training a voted perceptron +model = Perceptrons.fit(X_train,Y_train,centralize=true,mode="voted") +Y_pred = Perceptrons.predict(model,X_test) + +println("[Voted Perceptron] accuracy : $(acc(Y_train,Y_pred))") + + # training a kernel perceptron (XOR) X_train = [1.0 1.0; 0.0 1.0; 1.0 0.0; 0.0 0.0] Y_train = [0.0 ; 1.0; 1.0; 0.0] X_test = X .+ .03 # adding noise -model = Perceptrons.fit(X_train,Y_train,centralize=true,kernel="rbf",width=.01) +model = Perceptrons.fit(X_train,Y_train,centralize=true,mode="kernel",kernel="rbf",width=.01) Y_pred = Perceptrons.predict(model,X_test) -println("[Perceptron] accuracy : $(acc(Y_train,Y_pred))") +println("[Kernel Perceptron] accuracy : $(acc(Y_train,Y_pred))") # if you want to save your model From 2e2f98d8c0ebf00c8f76e815b5441c0fe4c02a4a Mon Sep 17 00:00:00 2001 From: Leandro Alvim Date: Sun, 26 Nov 2017 16:14:45 -0200 Subject: [PATCH 05/16] voted perceptron --- src/kernel_perceptron.jl | 3 -- src/linear_perceptron.jl | 3 -- test/runtests.jl | 1 + test/voted_perceptron_test.jl | 78 +++++++++++++++++++++++++++++++++++ 4 files changed, 79 insertions(+), 6 deletions(-) create mode 100644 test/voted_perceptron_test.jl diff --git a/src/kernel_perceptron.jl b/src/kernel_perceptron.jl index 20f9bad..15f83d0 100644 --- a/src/kernel_perceptron.jl +++ b/src/kernel_perceptron.jl @@ -40,9 +40,6 @@ end @inline ∑(λ,y,n,K) = sum(λ .* y .* K) -@inline function sign(val) - return (val >=0 ? 1.0: -1.0 ) -end function trainer{T<:AbstractFloat}(model::KernelPerceptron{T}, X::AbstractArray{T}, diff --git a/src/linear_perceptron.jl b/src/linear_perceptron.jl index 4fb52c8..0ed6575 100644 --- a/src/linear_perceptron.jl +++ b/src/linear_perceptron.jl @@ -1,7 +1,4 @@ -@inline sinal(x) = x>=0 ? 1.0 : 0.0 - -@inline h(Θ,x) = sinal(Θ'*x) function trainer{T<:AbstractFloat}(model::LinearPerceptron{T}, X::AbstractArray{T}, diff --git a/test/runtests.jl b/test/runtests.jl index a9d2abc..bc8ac6e 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -4,3 +4,4 @@ using Base.Test include("utils_test.jl") include("linear_perceptron_test.jl") include("kernel_perceptron_test.jl") +#include("voted_perceptron_test.jl") diff --git a/test/voted_perceptron_test.jl b/test/voted_perceptron_test.jl new file mode 100644 index 0000000..41c0c77 --- /dev/null +++ b/test/voted_perceptron_test.jl @@ -0,0 +1,78 @@ +using Perceptrons + + +@testset "Linear Perceptron Tests (in sample)" begin + + + @testset "OR function" begin + + X = [1.0 1.0; 0.0 1.0; 1.0 0.0; 0.0 0.0] + Y = [1.0 ; 1.0; 1.0; 0.0] + + model = Perceptrons.fit(X,Y,centralize=false) + pred = Perceptrons.predict(model,X) + + @test all(pred .== Y) + + model = Perceptrons.fit(X,Y,centralize=true) + pred = Perceptrons.predict(model,X) + + @test all(pred .== Y) + + end + + @testset "AND function" begin + + X = [1.0 1.0; 0.0 1.0; 1.0 0.0; 0.0 0.0] + Y = [1.0 ; 0.0; 0.0; 0.0] + + model = Perceptrons.fit(X,Y,centralize=false) + pred = Perceptrons.predict(model,X) + + @test all(pred .== Y) + + model = Perceptrons.fit(X,Y,centralize=true) + pred = Perceptrons.predict(model,X) + + @test all(pred .== Y) + + end + +end + + +@testset "Linear Perceptron Tests (out of sample)" begin + + + @testset "OR function" begin + + X = [1.0 1.0; 0.0 1.0; 1.0 0.0; 0.0 0.0] + Y = [1.0 ; 1.0; 1.0; 0.0] + Xt = X .+ .3 + model = Perceptrons.fit(X,Y,centralize=true) + pred = Perceptrons.predict(model,Xt) + @test all(pred .== Y) + + end + + @testset "AND function" begin + + X = [1.0 1.0; 0.0 1.0; 1.0 0.0; 0.0 0.0] + Y = [1.0 ; 0.0; 0.0; 0.0] + Xt = X .+ .03 + + model = Perceptrons.fit(X,Y,centralize=true,alpha=1.0) + pred = Perceptrons.predict(model,Xt) + + end + +end + +@testset "Check Labels" begin + + X = [1.0 1.0; 0.0 1.0; 1.0 0.0; 0.0 0.0] + Y = [1.0 ; -1; 0.0; 0.0] + + try model = Perceptrons.fit(X,Y) catch @test true end + +end From 57b91fac72b1af9465181cbf6aa1b896d5f044d0 Mon Sep 17 00:00:00 2001 From: Leandro Alvim Date: Sun, 26 Nov 2017 16:16:03 -0200 Subject: [PATCH 06/16] Fix --- src/voted_perceptron.jl | 4 ++-- test/runtests.jl | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/voted_perceptron.jl b/src/voted_perceptron.jl index 2d9b156..1bc6a4c 100644 --- a/src/voted_perceptron.jl +++ b/src/voted_perceptron.jl @@ -7,7 +7,7 @@ s end -function trainer{T<:AbstractFloat}(model::LinearPerceptron{T}, +function trainer{T<:AbstractFloat}(model::VotedPerceptron{T}, X::AbstractArray{T}, Y::Vector{T}) @@ -60,7 +60,7 @@ function trainer{T<:AbstractFloat}(model::LinearPerceptron{T}, model.history = history end -function predictor{T<:AbstractFloat}(model::LinearPerceptron{T}, +function predictor{T<:AbstractFloat}(model::VotedPerceptron{T}, X::AbstractArray{T}) Θ = model.Θ diff --git a/test/runtests.jl b/test/runtests.jl index bc8ac6e..0fa72a8 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -2,6 +2,6 @@ using Perceptrons using Base.Test include("utils_test.jl") -include("linear_perceptron_test.jl") -include("kernel_perceptron_test.jl") +#include("linear_perceptron_test.jl") +#include("kernel_perceptron_test.jl") #include("voted_perceptron_test.jl") From afc78fe5734fb2c4bf1c5cadc54602744ddceba9 Mon Sep 17 00:00:00 2001 From: Leandro Alvim Date: Sun, 26 Nov 2017 16:29:32 -0200 Subject: [PATCH 07/16] added mode paremeter --- test/kernel_perceptron_test.jl | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/test/kernel_perceptron_test.jl b/test/kernel_perceptron_test.jl index 9ac6ab1..2624fc9 100644 --- a/test/kernel_perceptron_test.jl +++ b/test/kernel_perceptron_test.jl @@ -9,12 +9,12 @@ using Perceptrons X = [1.0 1.0; 0.0 1.0; 1.0 0.0; 0.0 0.0] Y = [1.0 ; 1.0; 1.0; 0.0] - model = Perceptrons.fit(X,Y,centralize=false,kernel="rbf",width=.1) + model = Perceptrons.fit(X,Y,centralize=false,mode="kernel",kernel="rbf",width=.1) pred = Perceptrons.predict(model,X) @test all(pred .== Y) - model = Perceptrons.fit(X,Y,centralize=true,kernel="rbf",width=.1) + model = Perceptrons.fit(X,Y,centralize=true,mode="kernel",kernel="rbf",width=.1) pred = Perceptrons.predict(model,X) @test all(pred .== Y) @@ -26,12 +26,12 @@ using Perceptrons X = [1.0 1.0; 0.0 1.0; 1.0 0.0; 0.0 0.0] Y = [1.0 ; 0.0; 0.0; 0.0] - model = Perceptrons.fit(X,Y,centralize=false,kernel="rbf",width=.1) + model = Perceptrons.fit(X,Y,centralize=false,mode="kernel",kernel="rbf",width=.1) pred = Perceptrons.predict(model,X) @test all(pred .== Y) - model = Perceptrons.fit(X,Y,centralize=true,kernel="rbf",width=.1) + model = Perceptrons.fit(X,Y,centralize=true,mode="kernel",kernel="rbf",width=.1) pred = Perceptrons.predict(model,X) @test all(pred .== Y) @@ -43,12 +43,12 @@ using Perceptrons X = [1.0 1.0; 0.0 1.0; 1.0 0.0; 0.0 0.0] Y = [0.0 ; 1.0; 1.0; 0.0] - model = Perceptrons.fit(X,Y,centralize=false,kernel="rbf",width=.01) + model = Perceptrons.fit(X,Y,centralize=false,mode="kernel",kernel="rbf",width=.01) pred = Perceptrons.predict(model,X) @test all(pred .== Y) - model = Perceptrons.fit(X,Y,centralize=true,kernel="rbf",width=.01) + model = Perceptrons.fit(X,Y,centralize=true,mode="kernel",kernel="rbf",width=.01) pred = Perceptrons.predict(model,X) @test all(pred .== Y) @@ -66,7 +66,7 @@ end X = [1.0 1.0; 0.0 1.0; 1.0 0.0; 0.0 0.0] Y = [1.0 ; 1.0; 1.0; 0.0] Xt = X .+ .3 - model = Perceptrons.fit(X,Y,centralize=true,kernel="rbf",width=.1) + model = Perceptrons.fit(X,Y,centralize=true,mode="kernel",kernel="rbf",width=.1) pred = Perceptrons.predict(model,Xt) @test all(pred .== Y) @@ -78,7 +78,7 @@ end Y = [1.0 ; 0.0; 0.0; 0.0] Xt = X .+ .03 - model = Perceptrons.fit(X,Y,centralize=true,kernel="rbf",width=.1) + model = Perceptrons.fit(X,Y,centralize=true,mode="kernel",kernel="rbf",width=.1) pred = Perceptrons.predict(model,Xt) @test all(pred .== Y) @@ -90,7 +90,7 @@ end Y = [0.0 ; 1.0; 1.0; 0.0] Xt = X .+ .03 - model = Perceptrons.fit(X,Y,centralize=true,kernel="rbf",width=.01) + model = Perceptrons.fit(X,Y,centralize=true,mode="kernel",kernel="rbf",width=.01) pred = Perceptrons.predict(model,Xt) @test all(pred .== Y) @@ -103,6 +103,6 @@ end X = [1.0 1.0; 0.0 1.0; 1.0 0.0; 0.0 0.0] Y = [1.0 ; -1; 0.0; 0.0] - try model = Perceptrons.fit(X,Y,kernel="rbf",width=1.0) catch @test true end + try model = Perceptrons.fit(X,Y,mode="kernel",kernel="rbf",width=1.0) catch @test true end end From 3cb4503013b84851ca488dc0fd13c59e47451186 Mon Sep 17 00:00:00 2001 From: Leandro Alvim Date: Sun, 26 Nov 2017 16:31:03 -0200 Subject: [PATCH 08/16] parameter added --- test/voted_perceptron_test.jl | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/test/voted_perceptron_test.jl b/test/voted_perceptron_test.jl index 41c0c77..b383903 100644 --- a/test/voted_perceptron_test.jl +++ b/test/voted_perceptron_test.jl @@ -1,7 +1,7 @@ using Perceptrons -@testset "Linear Perceptron Tests (in sample)" begin +@testset "Voted Perceptron Tests (in sample)" begin @testset "OR function" begin @@ -9,12 +9,12 @@ using Perceptrons X = [1.0 1.0; 0.0 1.0; 1.0 0.0; 0.0 0.0] Y = [1.0 ; 1.0; 1.0; 0.0] - model = Perceptrons.fit(X,Y,centralize=false) + model = Perceptrons.fit(X,Y,centralize=false,mode="voted") pred = Perceptrons.predict(model,X) @test all(pred .== Y) - model = Perceptrons.fit(X,Y,centralize=true) + model = Perceptrons.fit(X,Y,centralize=true,mode="voted") pred = Perceptrons.predict(model,X) @test all(pred .== Y) @@ -26,12 +26,12 @@ using Perceptrons X = [1.0 1.0; 0.0 1.0; 1.0 0.0; 0.0 0.0] Y = [1.0 ; 0.0; 0.0; 0.0] - model = Perceptrons.fit(X,Y,centralize=false) + model = Perceptrons.fit(X,Y,centralize=false,mode="voted") pred = Perceptrons.predict(model,X) @test all(pred .== Y) - model = Perceptrons.fit(X,Y,centralize=true) + model = Perceptrons.fit(X,Y,centralize=true,mode="voted") pred = Perceptrons.predict(model,X) @test all(pred .== Y) @@ -41,7 +41,7 @@ using Perceptrons end -@testset "Linear Perceptron Tests (out of sample)" begin +@testset "Voted Perceptron Tests (out of sample)" begin @testset "OR function" begin @@ -49,7 +49,7 @@ end X = [1.0 1.0; 0.0 1.0; 1.0 0.0; 0.0 0.0] Y = [1.0 ; 1.0; 1.0; 0.0] Xt = X .+ .3 - model = Perceptrons.fit(X,Y,centralize=true) + model = Perceptrons.fit(X,Y,centralize=true,mode="voted") pred = Perceptrons.predict(model,Xt) @test all(pred .== Y) @@ -61,7 +61,7 @@ end Y = [1.0 ; 0.0; 0.0; 0.0] Xt = X .+ .03 - model = Perceptrons.fit(X,Y,centralize=true,alpha=1.0) + model = Perceptrons.fit(X,Y,centralize=true,alpha=1.0,mode="voted") pred = Perceptrons.predict(model,Xt) end @@ -73,6 +73,6 @@ end X = [1.0 1.0; 0.0 1.0; 1.0 0.0; 0.0 0.0] Y = [1.0 ; -1; 0.0; 0.0] - try model = Perceptrons.fit(X,Y) catch @test true end + try model = Perceptrons.fit(X,Y,mode="voted") catch @test true end end From b31cf59c40514c3dca5fea181c5f5611471d1414 Mon Sep 17 00:00:00 2001 From: Leandro Alvim Date: Sun, 26 Nov 2017 16:33:16 -0200 Subject: [PATCH 09/16] fix --- src/voted_perceptron.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/voted_perceptron.jl b/src/voted_perceptron.jl index 1bc6a4c..7636327 100644 --- a/src/voted_perceptron.jl +++ b/src/voted_perceptron.jl @@ -37,7 +37,7 @@ function trainer{T<:AbstractFloat}(model::VotedPerceptron{T}, # weight updates for all samples for i=1:n xi = x[i,:] - ξ = h(Θ,xi) - y[i] + ξ = h(Θ[k],xi) - y[i] if ξ==0 c[k] += 1 else From 31e7be654a01f400a0126e3a35a8a38248276322 Mon Sep 17 00:00:00 2001 From: Leandro Alvim Date: Sun, 26 Nov 2017 16:41:46 -0200 Subject: [PATCH 10/16] fix --- src/voted_perceptron.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/voted_perceptron.jl b/src/voted_perceptron.jl index 7636327..4ed8237 100644 --- a/src/voted_perceptron.jl +++ b/src/voted_perceptron.jl @@ -72,7 +72,7 @@ function predictor{T<:AbstractFloat}(model::VotedPerceptron{T}, y = zeros(Real,n) X = hcat(X,ones(n,1)) # adding bias for i=1:n - y[i] = sinal(vote(Θ[i],X[i,:],c[i],k)) + y[i] = sinal(vote(Θ[i],X[i,:],c,k)) end y From 5406ae08e78228492501fe2e60889bd301620bd9 Mon Sep 17 00:00:00 2001 From: Leandro Alvim Date: Sun, 26 Nov 2017 17:10:27 -0200 Subject: [PATCH 11/16] reorganization --- src/linear_perceptron.jl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/linear_perceptron.jl b/src/linear_perceptron.jl index 0ed6575..302d8a6 100644 --- a/src/linear_perceptron.jl +++ b/src/linear_perceptron.jl @@ -1,3 +1,5 @@ +# use in linear perceptron +@inline h(Θ,x) = sinal(Θ'*x) function trainer{T<:AbstractFloat}(model::LinearPerceptron{T}, From 2c1e858f4118ba2f318afd08df51c19d4dd32c49 Mon Sep 17 00:00:00 2001 From: Leandro Alvim Date: Sun, 26 Nov 2017 17:10:39 -0200 Subject: [PATCH 12/16] reorganization --- src/utils.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils.jl b/src/utils.jl index ee8fadd..b7a79e3 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -6,7 +6,7 @@ export acc @inline acc(yt,yp) = count(x->x==true, yt .== yp)/length(yt) # used in linear and voted perceptron -@inline sinal(x) = x>=0 ? 1.0 : 0.0 +@inline sinal(x) = (x>=0 ? 1.0 : 0.0) # used in kernel perceptron @inline sign(val) = (val >=0 ? 1.0: -1.0 ) # use in linear and voted perceptron From 8b36731fe192df10a576cde855623f1dbd8849e3 Mon Sep 17 00:00:00 2001 From: Leandro Alvim Date: Sun, 26 Nov 2017 17:10:48 -0200 Subject: [PATCH 13/16] Fix --- src/voted_perceptron.jl | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/voted_perceptron.jl b/src/voted_perceptron.jl index 4ed8237..7c7c803 100644 --- a/src/voted_perceptron.jl +++ b/src/voted_perceptron.jl @@ -1,8 +1,11 @@ + + + @inline function vote(Θ,x,c,k) s = 0 for j=1:k - s += c[j]*h(Θ,x) # voting + s += c[j]*sign(Θ[j]'*x) # voting (+1 or -1 * c[j] weight) end s end @@ -24,7 +27,7 @@ function trainer{T<:AbstractFloat}(model::VotedPerceptron{T}, history = [] nerrors,nlast_errors = Inf,0 epochs = 0 - k,Θ,c,α = 0,Dict(0=>rand(m+1)),Dict(0=>0),model.α + k,Θ,c,α = 1,Dict(1=>rand(m+1)),Dict(1=>0),model.α while nerrors>0 && epochs < max_epochs # stops when error is equal to zero or grater than last_error or reached max iterations # shuffle dataset @@ -37,7 +40,7 @@ function trainer{T<:AbstractFloat}(model::VotedPerceptron{T}, # weight updates for all samples for i=1:n xi = x[i,:] - ξ = h(Θ[k],xi) - y[i] + ξ = sinal(Θ[k]'*xi) - y[i] if ξ==0 c[k] += 1 else @@ -72,7 +75,8 @@ function predictor{T<:AbstractFloat}(model::VotedPerceptron{T}, y = zeros(Real,n) X = hcat(X,ones(n,1)) # adding bias for i=1:n - y[i] = sinal(vote(Θ[i],X[i,:],c,k)) + y[i] = sinal(vote(Θ,X[i,:],c,k)) + println("y[i] = ",y[i]) end y From 0f6504046da5f0cc6583d9794a08477e8471a30d Mon Sep 17 00:00:00 2001 From: Leandro Alvim Date: Sun, 26 Nov 2017 20:37:25 -0200 Subject: [PATCH 14/16] fix --- src/utils.jl | 2 -- src/voted_perceptron.jl | 4 ++-- test/runtests.jl | 4 ++-- test/voted_perceptron_test.jl | 30 +++++++++++++++++++++++------- 4 files changed, 27 insertions(+), 13 deletions(-) diff --git a/src/utils.jl b/src/utils.jl index b7a79e3..2caaae8 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -9,8 +9,6 @@ export acc @inline sinal(x) = (x>=0 ? 1.0 : 0.0) # used in kernel perceptron @inline sign(val) = (val >=0 ? 1.0: -1.0 ) -# use in linear and voted perceptron -@inline h(Θ,x) = sinal(Θ'*x) ## checks PLS input data and params diff --git a/src/voted_perceptron.jl b/src/voted_perceptron.jl index 7c7c803..9829a2c 100644 --- a/src/voted_perceptron.jl +++ b/src/voted_perceptron.jl @@ -28,7 +28,8 @@ function trainer{T<:AbstractFloat}(model::VotedPerceptron{T}, nerrors,nlast_errors = Inf,0 epochs = 0 k,Θ,c,α = 1,Dict(1=>rand(m+1)),Dict(1=>0),model.α - while nerrors>0 && epochs < max_epochs + #while nerrors>0 && epochs < max_epochs + while epochs < max_epochs # stops when error is equal to zero or grater than last_error or reached max iterations # shuffle dataset if shuffle_epoch @@ -76,7 +77,6 @@ function predictor{T<:AbstractFloat}(model::VotedPerceptron{T}, X = hcat(X,ones(n,1)) # adding bias for i=1:n y[i] = sinal(vote(Θ,X[i,:],c,k)) - println("y[i] = ",y[i]) end y diff --git a/test/runtests.jl b/test/runtests.jl index 0fa72a8..48b8dcc 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,7 +1,7 @@ using Perceptrons using Base.Test -include("utils_test.jl") +#include("utils_test.jl") #include("linear_perceptron_test.jl") #include("kernel_perceptron_test.jl") -#include("voted_perceptron_test.jl") +include("voted_perceptron_test.jl") diff --git a/test/voted_perceptron_test.jl b/test/voted_perceptron_test.jl index b383903..3baf517 100644 --- a/test/voted_perceptron_test.jl +++ b/test/voted_perceptron_test.jl @@ -1,6 +1,22 @@ using Perceptrons +@testset "OR function" begin + + X = [1.0 1.0; 0.0 1.0; 1.0 0.0; 0.0 0.0] + Y = [1.0 ; 1.0; 1.0; 0.0] + + model = Perceptrons.fit(X,Y,centralize=false,mode="voted",max_epochs=100) + pred = Perceptrons.predict(model,X) + + @test all(pred .== Y) + + model = Perceptrons.fit(X,Y,centralize=true,mode="voted",max_epochs=100) + pred = Perceptrons.predict(model,X) + + @test all(pred .== Y) +end + @testset "Voted Perceptron Tests (in sample)" begin @@ -9,12 +25,12 @@ using Perceptrons X = [1.0 1.0; 0.0 1.0; 1.0 0.0; 0.0 0.0] Y = [1.0 ; 1.0; 1.0; 0.0] - model = Perceptrons.fit(X,Y,centralize=false,mode="voted") + model = Perceptrons.fit(X,Y,centralize=false,mode="voted",max_epochs=100) pred = Perceptrons.predict(model,X) @test all(pred .== Y) - model = Perceptrons.fit(X,Y,centralize=true,mode="voted") + model = Perceptrons.fit(X,Y,centralize=true,mode="voted",max_epochs=100) pred = Perceptrons.predict(model,X) @test all(pred .== Y) @@ -26,12 +42,12 @@ using Perceptrons X = [1.0 1.0; 0.0 1.0; 1.0 0.0; 0.0 0.0] Y = [1.0 ; 0.0; 0.0; 0.0] - model = Perceptrons.fit(X,Y,centralize=false,mode="voted") + model = Perceptrons.fit(X,Y,centralize=false,mode="voted",max_epochs=100) pred = Perceptrons.predict(model,X) @test all(pred .== Y) - model = Perceptrons.fit(X,Y,centralize=true,mode="voted") + model = Perceptrons.fit(X,Y,centralize=true,mode="voted",max_epochs=100) pred = Perceptrons.predict(model,X) @test all(pred .== Y) @@ -49,7 +65,7 @@ end X = [1.0 1.0; 0.0 1.0; 1.0 0.0; 0.0 0.0] Y = [1.0 ; 1.0; 1.0; 0.0] Xt = X .+ .3 - model = Perceptrons.fit(X,Y,centralize=true,mode="voted") + model = Perceptrons.fit(X,Y,centralize=true,mode="voted",max_epochs=100) pred = Perceptrons.predict(model,Xt) @test all(pred .== Y) @@ -61,7 +77,7 @@ end Y = [1.0 ; 0.0; 0.0; 0.0] Xt = X .+ .03 - model = Perceptrons.fit(X,Y,centralize=true,alpha=1.0,mode="voted") + model = Perceptrons.fit(X,Y,centralize=true,alpha=1.0,mode="voted",max_epochs=100) pred = Perceptrons.predict(model,Xt) end @@ -73,6 +89,6 @@ end X = [1.0 1.0; 0.0 1.0; 1.0 0.0; 0.0 0.0] Y = [1.0 ; -1; 0.0; 0.0] - try model = Perceptrons.fit(X,Y,mode="voted") catch @test true end + try model = Perceptrons.fit(X,Y,mode="voted",max_epochs=100) catch @test true end end From a1e443142dea77872acfcf5142902b362468b766 Mon Sep 17 00:00:00 2001 From: Leandro Alvim Date: Sun, 26 Nov 2017 20:38:24 -0200 Subject: [PATCH 15/16] ok --- test/runtests.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index 48b8dcc..a3f5936 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,7 +1,7 @@ using Perceptrons using Base.Test -#include("utils_test.jl") -#include("linear_perceptron_test.jl") -#include("kernel_perceptron_test.jl") +include("utils_test.jl") +include("linear_perceptron_test.jl") +include("kernel_perceptron_test.jl") include("voted_perceptron_test.jl") From bcdf4b11ed304df861cfd9a47b3250d558f43b20 Mon Sep 17 00:00:00 2001 From: Leandro Alvim Date: Sun, 26 Nov 2017 20:45:16 -0200 Subject: [PATCH 16/16] Improvement --- README.md | 4 ++-- experiments/readme_example.jl | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index e7c0d48..6ee6c95 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ Perceptrons.jl ====== -A package with several types of Perceptron classifiers. Perceptrons are fast classifiers and can be used even for big data. Up to now, this package contains a linear perceptron and a Kernel perceptron for binary classification problems. This project will have the following perceptron classifiers: Multiclass, Kernel, Structured, Voted, Average and Sparse. Some state-of-the-art must be included after these. +A package with several types of Perceptron classifiers. Perceptrons are fast classifiers and can be used even for big data. Up to now, this package contains a linear perceptron, voted perceptron and a Kernel perceptron for binary classification problems. This project will have the following perceptron classifiers: Multiclass, Kernel, Structured, Voted, Average and Sparse. Some state-of-the-art must be included after these. [![Build Status](https://travis-ci.org/lalvim/Perceptrons.jl.svg?branch=master)](https://travis-ci.org/lalvim/Perceptrons.jl) @@ -44,7 +44,7 @@ Examples # training a kernel perceptron (XOR) X_train = [1.0 1.0; 0.0 1.0; 1.0 0.0; 0.0 0.0] Y_train = [0.0 ; 1.0; 1.0; 0.0] - X_test = X .+ .03 # adding noise + X_test = X_train .+ .03 # adding noise model = Perceptrons.fit(X_train,Y_train,centralize=true,mode="kernel",kernel="rbf",width=.01) Y_pred = Perceptrons.predict(model,X_test) diff --git a/experiments/readme_example.jl b/experiments/readme_example.jl index 1bcbb6d..aa9f34b 100644 --- a/experiments/readme_example.jl +++ b/experiments/readme_example.jl @@ -20,7 +20,7 @@ println("[Voted Perceptron] accuracy : $(acc(Y_train,Y_pred))") # training a kernel perceptron (XOR) X_train = [1.0 1.0; 0.0 1.0; 1.0 0.0; 0.0 0.0] Y_train = [0.0 ; 1.0; 1.0; 0.0] -X_test = X .+ .03 # adding noise +X_test = X_train .+ .03 # adding noise model = Perceptrons.fit(X_train,Y_train,centralize=true,mode="kernel",kernel="rbf",width=.01) Y_pred = Perceptrons.predict(model,X_test)