数据处理

using MLJ
using TableView # for showtable

using RDatasets
boston = dataset("MASS", "Boston");
y, X = unpack(boston, col -> col == :MedV, col -> col != :MedV) # MedV 平均房价特征

models(matching(X,y))
models(matching(X))

julia> info("RidgeRegressor", pkg="MLJLinearModels")
...
# 输入数据的科学类型
 input_scitype = Table{_s23} where _s23<:(AbstractArray{_s25,1} where _s25<:Continuous),
 # 输出数据的科学类型
 target_scitype = AbstractArray{Continuous,1},
...

measures(matching(y))

julia> info(l1)
absolute deviations; aliases: `l1`.
...
# 虽然也是target，但这个其实是输入的数据类型
 target_scitype = Union{AbstractArray{Continuous,1}, AbstractArray{Count,1}},
...

X = (col_1 = [1,2,3],
	 col_2 = [1,2,3],
	 col_3 = ["High", "Low", "High"],
	 col_4 = ["John", "Maria", "Mike"],
	 col_5 = [1.5, 1.5, -2.3, -2.3])
schema(X)	 

Xhat = coerce(X, :col_1 => OrderedFactor, # 这里的分类数据用OrderedFactor来做个例子好了
	      :col_2 => Count, # 可有可无
		  :col_3 => OrderedFactor,
		  :col_4 => Multiclass,
		  :col_5 => Multiclass) 
schema(Xhat)

Dict{Symbol,Type} with 5 entries:
  :col_5 => OrderedFactor
  :col_2 => OrderedFactor
  :col_3 => Multiclass
  :col_4 => Multiclass
  :col_1 => OrderedFactor

Dict{Symbol,Type} with 2 entries:
  :col_2 => Continuous
  :col_1 => Continuous

Dict{Symbol,Type} with 2 entries:
  :col_3 => Multiclass
  :col_4 => Multiclass

Dict{Symbol,Type} with 5 entries:
  :col_5 => OrderedFactor
  :col_2 => OrderedFactor
  :col_3 => Multiclass
  :col_4 => Multiclass
  :col_1 => OrderedFactor

coerce(X, autotype(X, :string_to_multiclass)) |> schema

julia> x1 = coerce([1,2,3], OrderedFactor)
3-element CategoricalArrays.CategoricalArray{Int64,1,UInt32}:
 1
 2
 3

julia> x2 = categorical([1,2,3], ordered=true)
3-element CategoricalArrays.CategoricalArray{Int64,1,UInt32}:
 1
 2
 3

julia> levels(x1)
3-element Array{Int64,1}:
 1
 2
 3

julia> levels!(x1, [3,2,1])
3-element CategoricalArrays.CategoricalArray{Int64,1,UInt32}:
 1
 2
 3

julia> levels(x1)
3-element Array{Int64,1}:
 3
 2
 1

info("DecisionTreeClassifier").prediction_type == :probabilistic # true
info("SVMClassifier", pkg="ScikitLearn").prediction_type == :deterministic # true

import RDatasets
iris = RDatasets.dataset("datasets", "iris")
y, X = unpack(iris, ==(:Species), colname -> true)
@load DecisionTreeClassifier
tree_model = DecisionTreeClassifier()

tree = machine(tree_model, X, y)
train, test = partition(eachindex(y), 0.7, shuffle = true)
fit!(tree, rows=train)
yhat = predict(tree, rows=test)

 UnivariateFinite{Multiclass{3}}(setosa=>1.0, versicolor=>0.0, virginica=>0.0)
 UnivariateFinite{Multiclass{3}}(setosa=>0.0, versicolor=>1.0, virginica=>0.0)
 UnivariateFinite{Multiclass{3}}(setosa=>0.0, versicolor=>0.0, virginica=>1.0)
...

mode.(yhat)

setosa
versicolor
virginica
...

@load SVMClassifier pkg=ScikitLearn
clf = fit!(machine(SVMClassifier(), X, y))
yhat = predict(clf, X)

 "setosa"
 "setosa"
 "setosa"
...

describe(X, :nmissing) # 每一列有missing的数量
13×2 DataFrame

describe(X, :min, :max, :mean, :std) # 每一列的最小值，最大值，平均值，标准差，他们会跳过missing

desribe(X, :symbol => fn) # fn作用于整个列

desribe(X, :symbol => sum) 

model = FeatureSelector([:Crim]) # 选择Crim的特征字段
mach = fit!(machine(model, X))
MLJ.transform(mach, X) |> df -> first(df, 5) # 这里的transform会与DataFrame的transform冲突，要指定模块为MLJ

FillImputer(
   features        = [],
   continuous_fill = e -> skipmissing(e) |> median
   count_fill      = e -> skipmissing(e) |> (f -> round(eltype(f), median(f)))
   finite_fill     = e -> skipmissing(e) |> mode)

df = coerce((x1 = 1:3, x2 = [missing, 1, 2]), :x2 => Continuous)
schema(df)

model = FillImputer(continuous_fill = e -> skipmissing(e) |> mean)
mach = fit!(machine(model, df))
w = MLJ.transform(mach, df)	
schema(w)

julia> w = MLJ.transform(mach, df)
(x1 = 1:3,
 x2 = [1.5, 1.0, 2.0],)

X = (ordinal1 = [1, 2, 3],
              ordinal2 = categorical([:x, :y, :x], ordered=true),
              ordinal3 = [10.0, 20.0, 30.0],
              ordinal4 = [-20.0, -30.0, -40.0],
              nominal = categorical(["Your father", "he", "is"]));

schema(X)

model = Standardizer()
mach = fit!(machine(model, X))
transform(mach, X)

(ordinal1 = [1, 2, 3],
 ordinal2 = CategoricalArrays.CategoricalValue{Symbol,UInt32}[:x, :y, :x],
 ordinal3 = [-1.0, 0.0, 1.0],
 ordinal4 = [1.0, 0.0, -1.0],
 nominal = CategoricalArrays.CategoricalValue{String,UInt32}["Your father", "he", "is"],)

# 先将X的ordinal2提取出来
temp = X.ordered2
nums = coerce(temp, Count)
# 3-element Array{Int64,1}:
#  1
#  2
#  1

model = UnivariateStandardizer() # UnivariateStandardizer 和 Standardizer 类似, UnivariateStandardizer不能用在命名元组DataFrame上，另外UnivariateStandardizer没有参数，不会忽略Count类型
mach = fit!(machine(model, nums)
transform(mach, nums)```

```julia-repl
 -0.5773502691896256
  1.1547005383792517
 -0.5773502691896256

model = Standardizer(ordered_factor = true)
mach = fit!(machine(model, X))
transform(mach, X)

(ordinal1 = [-1.0, 0.0, 1.0],
 ordinal2 = [-0.5773502691896256, 1.1547005383792517, -0.5773502691896256],
 ordinal3 = [-1.0, 0.0, 1.0],
 ordinal4 = [1.0, 0.0, -1.0],
 nominal = CategoricalArrays.CategoricalValue{String,UInt32}["Your father", "he", "is"],)

  UnivariateDiscretizer(n_classes=512)

  Returns an MLJModel for for discretizing any continuous vector v
  (scitype(v) <: AbstractVector{Continuous}), where n_classes describes
  the resolution of the discretization.

data = coerce(1:100, Continuous)
t = UnivariateDiscretizer(n_classes = 10)
discretizer = fit!(machine(t, data))
v = rand(1:100, 10)
w = transform(discretizer, v)