From 34ce9621a6ec17421575a888c778002bdb8c2518 Mon Sep 17 00:00:00 2001 From: Juhyun Kim Date: Thu, 3 Jan 2019 21:35:03 -0800 Subject: [PATCH 01/56] fixed deprecation warnings after running test --- src/VarianceComponentModels.jl | 56 ++++++++++----------- src/multivariate_calculus.jl | 32 ++++++------ src/two_variance_component.jl | 18 +++---- test/multivariate_calculus_test.jl | 40 +++++++-------- test/two_variance_component_test.jl | 70 +++++++++++++------------- test/variance_component_models_test.jl | 46 ++++++++--------- 6 files changed, 131 insertions(+), 131 deletions(-) diff --git a/src/VarianceComponentModels.jl b/src/VarianceComponentModels.jl index 1093260..bf25ef5 100644 --- a/src/VarianceComponentModels.jl +++ b/src/VarianceComponentModels.jl @@ -3,12 +3,17 @@ module VarianceComponentModels using Compat import Compat.view -using MathProgBase, Ipopt#, KNITRO#, Mosek#, Gurobi -import Base: eltype, length, size, mean, mean!, cov, + +using LinearAlgebra, Statistics, MathProgBase, Ipopt#, KNITRO#, Mosek#, Gurobi +import Base: eltype, length, size, + export VarianceComponentModel, VarianceComponentVariate, TwoVarCompModelRotate, TwoVarCompVariateRotate, VarianceComponentAuxData, - residual, residual!, nvarcomps, nmeanparams, nvarparams, nparams, - mean!, mean, cov!, cov + residual, residual!, nvarcomps, nmeanparams, nvarparams, nparams + +# import Base: eltype, length, size, mean, mean!, cov, + +# export VarianceComponentModel, VarianceComponentVariate, +# TwoVarCompModelRotate, TwoVarCompVariateRotate, VarianceComponentAuxData, +# residual, residual!, nvarcomps, nmeanparams, nvarparams, nparams, +# mean!, mean, cov!, cov """ `VarianceComponentModel` stores the model parameters of a variance component @@ -46,7 +51,7 @@ function VarianceComponentModel( B::AbstractVecOrMat, Σ::NTuple{M, AbstractMatrix}, A::Matrix = zeros(eltype(B), 0, length(B)), - sense::Union{Char, Vector{Char}} = Array{Char}(0), + sense::Union{Char, Vector{Char}} = Array{Char}(undef, 0), b::Union{Number, Vector} = zeros(eltype(B), 0), lb::Union{Number, Vector} = convert(eltype(B), -Inf), ub::Union{Number, Vector} = convert(eltype(B), Inf), @@ -103,19 +108,18 @@ end Constructor of a [`TwoVarCompModelRotate`](@ref) instance from a [`VarianceComponentModel`](@ref) instance. """ -function TwoVarCompModelRotate{T, BT, ΣT}( - vcm::VarianceComponentModel{T, 2, BT, ΣT} - ) +function TwoVarCompModelRotate(vcm::VarianceComponentModel{T, 2, BT, ΣT}) where {T, BT, ΣT} # generalized eigenvalue decomposition of (Σ1, Σ2) - F = eigfact(Symmetric(vcm.Σ[1]), Symmetric(vcm.Σ[2])) # output is of BlasFloat type + F = eigen(Symmetric(vcm.Σ[1]), Symmetric(vcm.Σ[2])) # output is of BlasFloat type λ = convert(Vector{T}, F.values) Φ = convert(Matrix{T}, F.vectors) # correct negative eigenvalues due to roundoff @compat map!(x -> max(x, zero(T)), λ, λ) - Brot = isempty(vcm.B) ? Array{T}(size(vcm.B)) : vcm.B * Φ + Brot = isempty(vcm.B) ? Array{T}(undef, size(vcm.B)) : vcm.B * Φ logdetΣ2 = convert(T, logdet(vcm.Σ[2])) TwoVarCompModelRotate{T, BT}(Brot, λ, Φ, logdetΣ2) + #TwoVarCompModelRotate(Brot, λ, Φ, logdetΣ2) where {T, BT} end """ @@ -207,20 +211,18 @@ end Constructor of a [`TwoVarCompVariateRotate`](@ref) instance from a [`VarianceComponentVariate`](@ref) instance. """ -function TwoVarCompVariateRotate{T <: AbstractFloat}( - vcobs::VarianceComponentVariate{T, 2} - ) +function TwoVarCompVariateRotate(vcobs::VarianceComponentVariate{T, 2}) where T <: AbstractFloat zeroT = zero(T) # (generalized)-eigendecomposition of (V1, V2) if isa(vcobs.V[2], UniformScaling) || - (isdiag(vcobs.V[2]) && vecnorm(diag(vcobs.V[2]) - one(T)) < 1.0e-8) - F = eigfact(Symmetric(vcobs.V[1])) + (isdiag(vcobs.V[2]) && norm(diag(vcobs.V[2]) .- one(T)) < 1.0e-8) + F = eigen(Symmetric(vcobs.V[1])) deval = convert(Vector{T}, F.values) U = convert(Matrix{T}, F.vectors) logdetV2 = zero(T) else - F = eigfact(Symmetric(vcobs.V[1]), Symmetric(vcobs.V[2])) + F = eigen(Symmetric(vcobs.V[1]), Symmetric(vcobs.V[2])) deval = convert(Vector{T}, F.values) U = convert(Matrix{T}, F.vectors) logdetV2 = convert(T, logdet(vcobs.V[2])) @@ -230,13 +232,13 @@ function TwoVarCompVariateRotate{T <: AbstractFloat}( deval[i] = deval[i] > zeroT ? deval[i] : zeroT end # rotate responses - Yrot = At_mul_B(U, vcobs.Y) - Xrot = isempty(vcobs.X) ? Array{T}(size(Yrot, 1), 0) : At_mul_B(U, vcobs.X) + Yrot = transpose(U) * vcobs.Y + Xrot = isempty(vcobs.X) ? Array{T}(size(Yrot, 1), 0) : transpose(U) * vcobs.X # output TwoVarCompVariateRotate(Yrot, Xrot, deval, U, logdetV2) end -function TwoVarCompVariateRotate{T <: VarianceComponentVariate}(vcdata::Array{T}) +function TwoVarCompVariateRotate(vcdata::Array{T}) where T <: VarianceComponentVariate map(x -> TwoVarCompVariateRotate(x), vcdata) end @@ -251,7 +253,7 @@ function VarianceComponentModel(vcobs::VarianceComponentVariate{T, M}) where {T, p, d, m = size(vcobs.X, 2), size(vcobs.Y, 2), length(vcobs.V) B = zeros(T, p, d) - Σ = ntuple(x -> eye(T, d), m)::NTuple{M, Matrix{T}} + Σ = ntuple(x -> Matrix(I, T, d), m)::NTuple{M, Matrix{T}} VarianceComponentModel(B, Σ) end @@ -266,7 +268,7 @@ function VarianceComponentModel(vcobsrot::TwoVarCompVariateRotate) p, d = size(vcobsrot.Xrot, 2), size(vcobsrot.Yrot, 2) T = eltype(vcobsrot) B = zeros(T, p, d) - Σ = (eye(T, d), eye(T, d)) + Σ = (Matrix(I, T, d), Matrix(I, T, d)) VarianceComponentModel(B, Σ) end @@ -305,10 +307,8 @@ function VarianceComponentAuxData(vcobsrot::TwoVarCompVariateRotate) Xwork, ywork, obswt) end -function VarianceComponentAuxData{ - T <: Union{VarianceComponentVariate, TwoVarCompVariateRotate}}( - vcdata::Array{T} - ) +function VarianceComponentAuxData(vcdata::Array{T}) where + T <: Union{VarianceComponentVariate, TwoVarCompVariateRotate} map(x -> VarianceComponentAuxData(x), vcdata) end @@ -461,7 +461,7 @@ end Residual of [`VarianceComponentVariate`](@ref) under [`VarianceComponentModel`](@ref). """ function residual(vcm::VarianceComponentModel, vcobs::VarianceComponentVariate) - return isempty(vcobs.X)? vcobs.Y : vcobs.Y - vcobs.X * vcm.B + return isempty(vcobs.X) ? vcobs.Y : vcobs.Y - vcobs.X * vcm.B end function residual( @@ -479,10 +479,10 @@ function residual!( ) if isempty(vcobs.X) - copy!(resid, vcobs.Y) + copyto!(resid, vcobs.Y) else oneT = one(eltype(vcm)) - copy!(resid, vcobs.Y) + copyto!(resid, vcobs.Y) LinAlg.BLAS.gemm!('N', 'N', -oneT, vcobs.X, vcm.B, oneT, resid) end resid diff --git a/src/multivariate_calculus.jl b/src/multivariate_calculus.jl index 7874246..6eaf242 100644 --- a/src/multivariate_calculus.jl +++ b/src/multivariate_calculus.jl @@ -1,5 +1,5 @@ # module MultivariateCalculus - +using SparseArrays export vech, trilind, triuind, commutation, spcommutation, duplication, spduplication, @@ -16,7 +16,7 @@ function vech(a::Union{Number, AbstractVecOrMat}) ooffset, aoffset = 1, 1 for j = 1:n len = m - j + 1 # no. elements to copy in column j - copy!(out, ooffset, a, aoffset, len) + copyto!(out, ooffset, a, aoffset, len) ooffset += m - j + 1 aoffset += m + 1 end @@ -34,7 +34,7 @@ Create the `mn x mn` commutation matrix `K`, defined by function commutation(t::Type, m::Integer, n::Integer) ((m < 0) || (n < 0)) && throw(ArgumentError("invalid Array dimensions")) mn = m * n - reshape(kron(vec(eye(t, m)), eye(t, n)), mn, mn) + reshape(kron(vec(Matrix{t}(I, m, m)), Matrix{t}(I, n, n)), mn, mn) end commutation(m::Integer, n::Integer) = commutation(Float64, m, n) commutation(t::Type, m::Integer) = commutation(t, m, m) @@ -50,7 +50,7 @@ Create the sparse `mn`-by-`mn` commutation matrix `K`, defined by function spcommutation(t::Type, m::Integer, n::Integer) ((m < 0) || (n < 0)) && throw(ArgumentError("invalid Array dimensions")) mn = m * n - reshape(kron(vec(speye(t, m)), speye(t, n)), mn, mn) + reshape(kron(vec(sparse(t(1)I, m, m)), sparse(t(1)I, n, n)), mn, mn) end spcommutation(m::Integer, n::Integer) = spcommutation(Float64, m, n) spcommutation(t::Type, m::Integer) = spcommutation(t, m, m) @@ -63,13 +63,13 @@ spcommutation(M::AbstractMatrix) = spcommutation(eltype(M), size(M, 1), size(M, Linear indices of the lower triangular part of an `m x n` array. """ function trilind(m::Integer, n::Integer, k::Integer) - find(tril(trues(m, n), k)) + (LinearIndices(tril(trues(m, n), k)))[findall(tril(trues(m, n), k))] end function trilind(m::Integer, n::Integer) - find(tril(trues(m, n))) + (LinearIndices(tril(trues(m, n))))[findall(tril(trues(m, n)))] end function trilind(m::Integer) - find(tril(trues(m, m))) + (LinearIndices(tril(trues(m, m))))[findall(tril(trues(m, m)))] end trilind(M::AbstractArray) = trilind(size(M, 1), size(M, 2)) trilind(M::AbstractArray, k::Integer) = trilind(size(M, 1), size(M, 2), k) @@ -80,13 +80,13 @@ trilind(M::AbstractArray, k::Integer) = trilind(size(M, 1), size(M, 2), k) Linear indices of the upper triangular part of an `m x n` array. """ function triuind(m::Integer, n::Integer, k::Integer) - find(triu(trues(m, n), k)) + (LinearIndices(triu(trues(m, n), k)))[findall(triu(trues(m, n), k))] end function triuind(m::Integer, n::Integer) - find(triu(trues(m, n))) + (LinearIndices(triu(trues(m, n))))[findall(triu(trues(m, n)))] end function triuind(m::Integer) - find(triu(trues(m, m))) + (LinearIndices(triu(trues(m, m))))[findall(triu(trues(m, m)))] end triuind(M::AbstractArray) = triuind(size(M, 1), size(M, 2)) triuind(M::AbstractArray, k::Integer) = triuind(size(M, 1), size(M, 2), k) @@ -100,12 +100,12 @@ Create the sparse `n^2 x n(n+1)/2` duplication matrix, defined by function spduplication(t::Type, n::Integer) imatrix = zeros(typeof(n), n, n) imatrix[trilind(n, n)] = 1:binomial(n + 1, 2) - imatrix = imatrix + tril(imatrix, -1).' + imatrix = imatrix + copy(transpose(tril(imatrix, -1))) sparse(1:n^2, vec(imatrix), one(t)) end spduplication(n::Integer) = spduplication(Float64, n) spduplication(M::AbstractMatrix) = spduplication(eltype(M), size(M, 1)) -duplication(t::Type, n::Integer) = full(spduplication(t, n)) +duplication(t::Type, n::Integer) = Matrix(spduplication(t, n)) duplication(n::Integer) = duplication(Float64, n) duplication(M::AbstractMatrix) = duplication(eltype(M), size(M, 1)) @@ -118,8 +118,8 @@ Compute the gradient `d / d vec(X)` from a vector of derivatives `dM` where function kron_gradient!(g::VecOrMat{T}, dM::VecOrMat{T}, Y::Matrix{T}, n::Integer, q::Integer) where {T <: Real} p, r = size(Y) - A_mul_B!(g, kron(speye(n * q), vec(Y)'), - (kron(speye(q), spcommutation(n, r), speye(p)) * dM)) + mul!(g, kron(sparse(I, n * q, n * q), vec(Y)'), + (kron(sparse(I, q, q), spcommutation(n, r), sparse(I, p, p)) * dM)) end function kron_gradient(dM::VecOrMat{T}, Y::Matrix{T}, n::Integer, q::Integer) where {T <: Real} @@ -141,8 +141,8 @@ Compute the gradient `d / d vech(L)` from a vector of derivatives `dM` where function chol_gradient!(g::AbstractVecOrMat{T}, dM::AbstractVecOrMat{T}, L::AbstractMatrix{T}) where {T <: Real} n = size(L, 1) - At_mul_B!(g, spduplication(n), - kron(L', speye(n)) * (dM + spcommutation(n) * dM)) + mul!(g, transpose(spduplication(n)), + kron(L', sparse(1.0I, n, n)) * (dM + spcommutation(n) * dM)) end function chol_gradient(dM::AbstractVecOrMat{T}, L::AbstractMatrix{T}) where {T <: Real} n = size(L, 1) diff --git a/src/two_variance_component.jl b/src/two_variance_component.jl index c992d80..a5f5453 100644 --- a/src/two_variance_component.jl +++ b/src/two_variance_component.jl @@ -1,4 +1,4 @@ -import Base.gradient +import LinearAlgebra, Printf export heritability, logpdf, @@ -123,7 +123,7 @@ function gradient!( end #A_mul_Bt!(N, N, vcmrot.eigvec), A_mul_B!(N, vcmrot.eigvec, N) N = vcmrot.eigvec * N * vcmrot.eigvec' - copy!(∇, d^2 + 1, N, 1, d^2) + copyto!(∇, d^2 + 1, N, 1, d^2) # gradient wrt Σ[1] @inbounds for j in 1:d λj = vcmrot.eigval[j] @@ -827,7 +827,7 @@ function mle_fs!( acceptable_tol = 1.0e-5, # default is 1.0e-6 max_iter = maxiter, # default is 3000 print_frequency_iter = 5, # default is 1 - print_level = verbose? 5 : 0, + print_level = verbose ? 5 : 0, print_info_string = "yes", #nlp_scaling_method = "none", #derivative_test = "second-order", @@ -840,7 +840,7 @@ function mle_fs!( solver = MosekSolver( MSK_IPAR_INTPNT_MAX_ITERATIONS = maxiter, MSK_DPAR_INTPNT_NL_TOL_REL_GAP = 1.0e-8, - MSK_IPAR_LOG = verbose? 10 : 0, # deafult value is 10 + MSK_IPAR_LOG = verbose ? 10 : 0, # deafult value is 10 #MSK_IPAR_OPTIMIZER = MSK_OPTIMIZER_NONCONVEX, #MSK_IPAR_LOG_NONCONVEX = 20, #MSK_IPAR_NONCONVEX_MAX_ITERATIONS = 100, @@ -852,7 +852,7 @@ function mle_fs!( # see https://www.artelys.com/tools/knitro_doc/3_referenceManual/userOptions.html for Mosek options solver = KnitroSolver( KTR_PARAM_ALG = 0, - KTR_PARAM_OUTLEV = verbose? 2 : 0, + KTR_PARAM_OUTLEV = verbose ? 2 : 0, #KTR_PARAM_MAXCGIT = 5, #KTR_PARAM_SCALE = 0, #KTR_PARAM_HONORBNDS = 1, @@ -1073,7 +1073,7 @@ function mle_mm!( println(" MM Algorithm") println(" Iter Objective ") println("-------- -------------") - @printf("%8.d %13.e\n", 0, logl) + Printf.@printf("%8.d %13.e\n", 0, logl) end # MM loop @@ -1093,7 +1093,7 @@ function mle_mm!( logl = logpdf(vcm, vcdatarot, vcaux) if verbose if (iter <= 10) || (iter > 10 && iter % 10 == 0) - @printf("%8.d %13.e\n", iter, logl) + Printf.@printf("%8.d %13.e\n", iter, logl) end end if abs(logl - loglold) < funtol * (abs(logl) + one(T)) @@ -1198,10 +1198,10 @@ function fit_reml!( qs = MosekSolver(MSK_IPAR_LOG = 0) end if typeof(vcdata) <: AbstractArray - Q = kron(eye(T, d), mapreduce(x -> At_mul_B(x.X, x.X), +, vcdata)) + Q = kron(Matrix{T}(I, d, d), mapreduce(x -> At_mul_B(x.X, x.X), +, vcdata)) c = vec(mapreduce(x -> At_mul_B(x.X, x.Y), +, vcdata)) else - Q = kron(eye(T, d), At_mul_B(vcdata.X, vcdata.X)) + Q = kron(Matrix{T}(I, d, d), At_mul_B(vcdata.X, vcdata.X)) c = vec(At_mul_B(vcdata.X, vcdata.Y)) end qpsol = quadprog(-c, Q, vcmodel.A, vcmodel.sense, vcmodel.b, diff --git a/test/multivariate_calculus_test.jl b/test/multivariate_calculus_test.jl index ec38c4b..52e5b27 100644 --- a/test/multivariate_calculus_test.jl +++ b/test/multivariate_calculus_test.jl @@ -1,8 +1,8 @@ module MultivariateCalculusTest -using VarianceComponentModels, Base.Test +using VarianceComponentModels, LinearAlgebra, Random, Test -srand(123) +Random.seed!(123) # test vech @testset "vech" begin @@ -22,7 +22,7 @@ end @testset "trilind" begin n = 3 A = randn(n, n) - @test vecnorm(vech(A) - A[trilind(A)]) ≈ 0.0 + @test norm(vech(A) - A[trilind(A)]) ≈ 0.0 end # test triuind @@ -52,14 +52,14 @@ end @testset "commutation" begin m, n = 3, 2 A = randn(m, n) - @test vecnorm(commutation(m, n) * vec(A) - vec(A')) ≈ 0.0 - @test vecnorm(commutation(A) * vec(A) - vec(A')) ≈ 0.0 - @test vecnorm(spcommutation(m, n) * vec(A) - vec(A')) ≈ 0.0 - @test vecnorm(spcommutation(A) * vec(A) - vec(A')) ≈ 0.0 + @test norm(commutation(m, n) * vec(A) - vec(A')) ≈ 0.0 + @test norm(commutation(A) * vec(A) - vec(A')) ≈ 0.0 + @test norm(spcommutation(m, n) * vec(A) - vec(A')) ≈ 0.0 + @test norm(spcommutation(A) * vec(A) - vec(A')) ≈ 0.0 B = randn(m, m) - @test vecnorm(commutation(m) * vec(B) - vec(B')) ≈ 0.0 - @test vecnorm(commutation(eltype(B), m) * vec(B) - vec(B')) ≈ 0.0 - @test vecnorm(spcommutation(eltype(B), m) * vec(B) - vec(B')) ≈ 0.0 + @test norm(commutation(m) * vec(B) - vec(B')) ≈ 0.0 + @test norm(commutation(eltype(B), m) * vec(B) - vec(B')) ≈ 0.0 + @test norm(spcommutation(eltype(B), m) * vec(B) - vec(B')) ≈ 0.0 end # test duplication @@ -67,10 +67,10 @@ end n = 3 A = randn(n, n) A = 0.5(A + A') # symmetrize - @test vecnorm(duplication(n) * vech(A) - vec(A)) ≈ 0.0 - @test vecnorm(duplication(A) * vech(A) - vec(A)) ≈ 0.0 - @test vecnorm(spduplication(n) * vech(A) - vec(A)) ≈ 0.0 - @test vecnorm(spduplication(A) * vech(A) - vec(A)) ≈ 0.0 + @test norm(duplication(n) * vech(A) - vec(A)) ≈ 0.0 + @test norm(duplication(A) * vech(A) - vec(A)) ≈ 0.0 + @test norm(spduplication(n) * vech(A) - vec(A)) ≈ 0.0 + @test norm(spduplication(A) * vech(A) - vec(A)) ≈ 0.0 end # test chol_gradient @@ -82,8 +82,8 @@ end # calculate gradient wrt L using chol_gradient dL1 = chol_gradient(vec(A), L) # alternative way to calculate gradient wrt L - dL2 = 2.0vech((A * L) + L' * A' - diagm(diag(A * L))) - @test vecnorm(dL1 - dL2) ≈ 0.0 + dL2 = 2.0vech((A * L) + L' * A' - Matrix(Diagonal(A * L))) + @test norm(dL1 - dL2) ≈ 0.0 end # test kron_gradient @@ -95,13 +95,13 @@ end # calculate gradient wrt X using kron_gradient dX1 = kron_gradient(vec(M), Y, n, q) # alternative way to calculate gradient wrt X - dX2 = zeros(X) + dX2 = zero(X) for j = 1:q for i = 1:n - dX2[i, j] = vecdot(Y, M[(i-1)*p+1:i*p, (j-1)*r+1:j*r]) + dX2[i, j] = dot(Y, M[(i-1)*p+1:i*p, (j-1)*r+1:j*r]) end end - @test vecnorm(dX1 - vec(dX2)) < 1e-8 + @test norm(dX1 - vec(dX2)) < 1e-8 end # test duplication @@ -111,7 +111,7 @@ end X = randn(p, q) Y = zeros(m * p, n * q) kronaxpy!(A, X, Y) - @test vecnorm(Y - kron(A, X)) ≈ 0.0 + @test norm(Y - kron(A, X)) ≈ 0.0 end # test bump_diagonal diff --git a/test/two_variance_component_test.jl b/test/two_variance_component_test.jl index 38e315b..32c6b3b 100644 --- a/test/two_variance_component_test.jl +++ b/test/two_variance_component_test.jl @@ -1,8 +1,8 @@ module TwoVarianceComponentTest -using VarianceComponentModels, MathProgBase, Ipopt, Base.Test +using VarianceComponentModels, MathProgBase, Ipopt, Random, LinearAlgebra, Test -srand(123) +Random.seed!(123) # generate data from a d-variate response variane component model n = 100 # no. observations @@ -14,75 +14,75 @@ B = ones(p, d) Σ = ntuple(x -> zeros(d, d), m) for i in 1:m Σi = randn(d, d) - copy!(Σ[i], Σi' * Σi) + copyto!(Σ[i], Σi' * Σi) end ## make the first variance component 0 matrix #fill!(Σ[1], 0.0) V = ntuple(x -> zeros(n, n), m) for i = 1:m-1 Vi = randn(n, 50) - copy!(V[i], Vi * Vi') + copyto!(V[i], Vi * Vi') end -copy!(V[m], eye(n)) +copyto!(V[m], Matrix(I, n, n)) # form Ω Ω = zeros(n*d, n*d) for i = 1:m - Ω += kron(Σ[i], V[i]) + global Ω += kron(Σ[i], V[i]) end -Ωchol = cholfact(Ω) -Y = X * B + reshape(Ωchol[:L] * randn(n*d), n, d) +Ωchol = cholesky(Ω) +Y = X * B + reshape(Ωchol.L * randn(n*d), n, d) -info("Forming VarianceComponentModel from data") +@info "Forming VarianceComponentModel from data" @inferred VarianceComponentVariate(Y, X, V) vcdata = VarianceComponentVariate(Y, X, V) vcmodel = VarianceComponentModel(vcdata) -info("Pre-compute eigen-decomposition and rotate data") +@info "Pre-compute eigen-decomposition and rotate data" vcdatarot = TwoVarCompVariateRotate(vcdata) vcmodelrot = TwoVarCompModelRotate(vcmodel) -info("Evaluate log-pdf") +@info "Evaluate log-pdf" #@code_warntype logpdf(vcmodelrot, vcdatarot) @inferred logpdf(vcmodelrot, vcdatarot) @test logpdf(vcmodel, vcdata) == logpdf(vcmodelrot, vcdatarot) @test (logpdf(vcmodelrot, [vcdatarot vcdatarot; vcdatarot vcdatarot]) - logpdf(vcmodel, [vcdata vcdata; vcdata vcdata])) ≈ 0.0 -info("Evaluate gradient") +@info "Evaluate gradient" ∇ = zeros(2d^2) #@code_warntype gradient!(∇, vcmodelrot, vcdatarot) @inferred gradient!(∇, vcmodelrot, vcdatarot) -@test vecnorm(gradient(vcmodel, vcdata) - gradient(vcmodelrot, vcdatarot)) ≈ 0.0 -@test vecnorm(gradient(vcmodel, vcdata) - gradient(vcmodel, vcdatarot)) ≈ 0.0 -@test vecnorm(gradient(vcmodel, [vcdata vcdata]) - +@test norm(gradient(vcmodel, vcdata) - gradient(vcmodelrot, vcdatarot)) ≈ 0.0 +@test norm(gradient(vcmodel, vcdata) - gradient(vcmodel, vcdatarot)) ≈ 0.0 +@test norm(gradient(vcmodel, [vcdata vcdata]) - 2.0gradient(vcmodel, vcdata)) ≈ 0.0 -@test vecnorm(gradient(vcmodel, [vcdata vcdata]) - +@test norm(gradient(vcmodel, [vcdata vcdata]) - gradient(vcmodelrot, [vcdatarot vcdatarot])) ≈ 0.0 -info("Evaluate Fisher information matrix of Σ") +@info "Evaluate Fisher information matrix of Σ" H = zeros(2d^2, 2d^2) #@code_warntype fisher!(H, vcmodelrot, vcdatarot) @inferred fisher_Σ!(H, vcmodelrot, vcdatarot) -@test vecnorm(fisher_Σ(vcmodel, vcdata) - fisher_Σ(vcmodelrot, vcdatarot)) ≈ 0.0 -@test vecnorm(fisher_Σ(vcmodel, vcdata) - fisher_Σ(vcmodel, vcdatarot)) ≈ 0.0 -@test vecnorm(fisher_Σ(vcmodel, [vcdata vcdata]) - +@test norm(fisher_Σ(vcmodel, vcdata) - fisher_Σ(vcmodelrot, vcdatarot)) ≈ 0.0 +@test norm(fisher_Σ(vcmodel, vcdata) - fisher_Σ(vcmodel, vcdatarot)) ≈ 0.0 +@test norm(fisher_Σ(vcmodel, [vcdata vcdata]) - 2fisher_Σ(vcmodel, vcdata)) ≈ 0.0 -@test vecnorm(fisher_Σ(vcmodel, [vcdata vcdata]) - +@test norm(fisher_Σ(vcmodel, [vcdata vcdata]) - fisher_Σ(vcmodelrot, [vcdatarot vcdatarot])) ≈ 0.0 -info("Evaluate Fisher information matrix of B") +@info "Evaluate Fisher information matrix of B" H = zeros(p * d, p * d) #@code_warntype fisher_B!(H, vcmodelrot, vcdatarot) @inferred fisher_B!(H, vcmodelrot, vcdatarot) -@test vecnorm(fisher_B(vcmodel, vcdata) - fisher_B(vcmodelrot, vcdatarot)) ≈ 0.0 -@test vecnorm(fisher_B(vcmodel, vcdata) - fisher_B(vcmodel, vcdatarot)) ≈ 0.0 -@test vecnorm(fisher_B(vcmodel, [vcdata vcdata]) - +@test norm(fisher_B(vcmodel, vcdata) - fisher_B(vcmodelrot, vcdatarot)) ≈ 0.0 +@test norm(fisher_B(vcmodel, vcdata) - fisher_B(vcmodel, vcdatarot)) ≈ 0.0 +@test norm(fisher_B(vcmodel, [vcdata vcdata]) - 2.0fisher_B(vcmodel, vcdata)) ≈ 0.0 -@test vecnorm(fisher_B(vcmodel, [vcdata vcdata]) - +@test norm(fisher_B(vcmodel, [vcdata vcdata]) - fisher_B(vcmodelrot, [vcdatarot vcdatarot])) ≈ 0.0 -info("Find MLE using Fisher scoring") +@info "Find MLE using Fisher scoring" vcmfs = deepcopy(vcmodel) #@code_warntype mle_fs!(vcmfs, vcdatarot; solver = :Ipopt) #@inferred mle_fs!(vcmfs, vcdatarot; solver = :Ipopt) @@ -92,7 +92,7 @@ logl_fs_array, = mle_fs!(vcmfs, [vcdatarot vcdatarot]; solver = :Ipopt) @show Bse_fs @show B -info("Find MLE using MM algorithm") +@info "Find MLE using MM algorithm" vcmmm = deepcopy(vcmodel) #@code_warntype mle_mm!(vcmm, vcdatarot) #@inferred mle_mm!(vcmmm, vcdatarot) @@ -105,7 +105,7 @@ vcmmm = deepcopy(vcmodel) logl_mm_array, = mle_mm!(vcmmm, [vcdatarot vcdatarot]) @test abs(logl_fs_array - logl_mm_array) / (abs(logl_fs_array) + 1.0) < 1.0e-4 -info("Find MLE using Fisher scoring (linear equality + box constraints)") +@info "Find MLE using Fisher scoring (linear equality + box constraints)" vcmfs = deepcopy(vcmodel) vcmfs.A = [1.0 -1.0 zeros(1, p*d-2)] vcmfs.sense = '=' @@ -118,7 +118,7 @@ logl_fs, _, _, Σcov_fs, Bse_fs, = mle_fs!(vcmfs, vcdatarot; solver = :Ipopt, qp @test all(vcmfs.B .≥ 0.0) @test all(vcmfs.B .≤ 1.0) -info("Find MLE using MM algorithm (linear equality + box constraints)") +@info "Find MLE using MM algorithm (linear equality + box constraints)" vcmm = deepcopy(vcmodel) vcmm.A = [1.0 -1.0 zeros(1, p*d-2)] vcmm.sense = '=' @@ -132,29 +132,29 @@ logl_mm, _, _, Σcov_mm = mle_mm!(vcmm, vcdatarot; qpsolver = :Ipopt) @test all(vcmm.B .≤ 1.0) @test abs(logl_fs - logl_mm) / (abs(logl_fs) + 1.0) < 1.0e-4 -info("Heritability estimation") +@info "Heritability estimation" h, h_se = heritability(vcmfs.Σ, Σcov_fs) @show h, h_se @test all(0.0 .≤ h .≤ 1.0) -info("test fit_mle (FS)") +@info "test fit_mle (FS)" vcmmle = deepcopy(vcmodel) logl_mle, _, _, Σcov_mle, Bse_mle, = fit_mle!(vcmmle, vcdata; algo = :FS) @show vcmmle.B, Bse_mle, B vcmle = deepcopy(vcmodel) fit_mle!(vcmmle, [vcdata vcdata]; algo = :FS) -info("test fit_mle (MM)") +@info "test fit_mle (MM)" vcmmle = deepcopy(vcmodel) logl_mle, _, _, Σcov_mle, Bse_mle, = fit_mle!(vcmmle, vcdata; algo = :MM) @show vcmmle.B, Bse_mle, B -info("test fit_reml (FS)") +@info "test fit_reml (FS)" vcmreml = deepcopy(vcmodel) logl_reml, _, _, Σcov_reml, Bse_reml, = fit_reml!(vcmreml, vcdata; algo = :FS) @show vcmreml.B, Bse_reml, B -info("test fit_reml (MM)") +@info "test fit_reml (MM)" vcmreml = deepcopy(vcmodel) logl_reml, _, _, Σcov_reml, Bse_reml, = fit_reml!(vcmreml, vcdata; algo = :MM) @show vcmreml.B, Bse_reml, B diff --git a/test/variance_component_models_test.jl b/test/variance_component_models_test.jl index d12477b..b55204c 100644 --- a/test/variance_component_models_test.jl +++ b/test/variance_component_models_test.jl @@ -1,8 +1,8 @@ module VarianceComponentTypeTest -using VarianceComponentModels, MathProgBase, Ipopt, Base.Test +using VarianceComponentModels, MathProgBase, Ipopt, LinearAlgebra, Random, Test -srand(123) +Random.seed!(123) # generate data from a d-variate response variane component model n = 100 # no. observations @@ -14,25 +14,25 @@ B = ones(p, d) Σ = ntuple(x -> zeros(d, d), m) for i in 1:m Σi = randn(d, d) - copy!(Σ[i], Σi' * Σi) + copyto!(Σ[i], Σi' * Σi) end ## make the first variance component 0 matrix #fill!(Σ[1], 0.0) V = ntuple(x -> zeros(n, n), m) for i = 1:m-1 Vi = randn(n, 50) - copy!(V[i], Vi * Vi') + copyto!(V[i], Vi * Vi') end -copy!(V[m], eye(n)) +copyto!(V[m], Matrix(I, n, n)) # form Ω Ω = zeros(n*d, n*d) for i = 1:m - Ω += kron(Σ[i], V[i]) + global Ω += kron(Σ[i], V[i]) end -Ωchol = cholfact(Ω) -Y = X * B + reshape(Ωchol[:L] * randn(n*d), n, d) +Ωchol = cholesky(Ω) +Y = X * B + reshape(Ωchol.L * randn(n*d), n, d) -info("VarianceComponentModel type") +@info "VarianceComponentModel type" #@code_warntype VarianceComponentModel(B, Σ) @inferred VarianceComponentModel(B, Σ) #@code_warntype VarianceComponentModel(Σ) @@ -40,18 +40,18 @@ info("VarianceComponentModel type") vcmodel = VarianceComponentModel(Σ) @test isempty(vcmodel.B) -info("TwoVarCompModelRotate type") -F = eigfact(Σ[1], Σ[2]) +@info "TwoVarCompModelRotate type" +F = eigen(Σ[1], Σ[2]) #@code_warntype TwoVarCompModelRotate(B * F.vectors, F.values, F.vectors, logdet(Σ[2])) @inferred TwoVarCompModelRotate(B * F.vectors, F.values, F.vectors, logdet(Σ[2])) -info("VarianceComponentVariate type") +@info "VarianceComponentVariate type" #@code_warntype VarianceComponentVariate(Y, V) @inferred VarianceComponentVariate(Y, V) vcdata = VarianceComponentVariate(Y, V) @test isempty(vcdata.X) -info("Forming VarianceComponentModel from data") +@info "Forming VarianceComponentModel from data" vcdata = VarianceComponentVariate(Y, X, V) #@code_warntype VarianceComponentVariate(Y, X, V) @inferred VarianceComponentVariate(Y, X, V) @@ -59,11 +59,11 @@ vcdata = VarianceComponentVariate(Y, X, V) #@code_warntype VarianceComponentModel(vcdata) @inferred VarianceComponentModel(vcdata) vcmodel = VarianceComponentModel(vcdata) -@test vecnorm(vcmodel.B) ≈ 0.0 -@test vecnorm(vcmodel.Σ[1] - eye(d)) ≈ 0.0 -@test vecnorm(vcmodel.Σ[2] - eye(d)) ≈ 0.0 +@test norm(vcmodel.B) ≈ 0.0 +@test norm(vcmodel.Σ[1] - Matrix(I, d, d)) ≈ 0.0 +@test norm(vcmodel.Σ[2] - Matrix(I, d, d)) ≈ 0.0 -info("Pre-compute eigen-decomposition and rotate data") +@info "Pre-compute eigen-decomposition and rotate data" #@code_warntype TwoVarCompVariateRotate(vcdata) @inferred TwoVarCompVariateRotate(vcdata) vcdatarot = TwoVarCompVariateRotate(vcdata) @@ -71,20 +71,20 @@ vcdatarot = TwoVarCompVariateRotate(vcdata) @inferred TwoVarCompModelRotate(vcmodel) vcmodelrot = TwoVarCompModelRotate(vcmodel) -info("Test generalized eigen-decomposition") +@info "Test generalized eigen-decomposition" vcdatacopy = deepcopy(vcdata) -scale!(vcdatacopy.V[2], 2.0) # V[2] = 2I +rmul!(vcdatacopy.V[2], 2.0) # V[2] = 2I vcdatacopyrot = TwoVarCompVariateRotate(vcdatacopy) @test vcdatacopyrot.logdetV2 ≈ n * log(2.0) # det(V[2]) = 2^n -info("VarianceComponentModel from TwoVarCompVariateRotate") +@info "VarianceComponentModel from TwoVarCompVariateRotate" @inferred VarianceComponentModel(vcdatarot) vcmodelfromrot = VarianceComponentModel(vcdatarot) @test size(vcmodelfromrot.B, 1) == p @test size(vcmodelfromrot.B, 2) == d @test length(vcmodelfromrot.Σ) == m -info("VarianceComponentAuxData from VarianceComponentVariate") +@info "VarianceComponentAuxData from VarianceComponentVariate" vcobsaux1 = VarianceComponentAuxData(vcdata) vcobsaux2 = VarianceComponentAuxData(vcdatarot) @test size(vcobsaux1.res) == size(vcobsaux2.res) @@ -92,7 +92,7 @@ vcobsaux2 = VarianceComponentAuxData(vcdatarot) @test size(vcobsaux1.ywork) == size(vcobsaux2.ywork) @test size(vcobsaux1.obswt) == size(vcobsaux2.obswt) -info("Query functions") +@info "Query functions" @test eltype(vcmodel) == eltype(B) @test eltype(vcdata) == eltype(Y) @test eltype(vcmodelrot) == eltype(vcmodel) @@ -116,7 +116,7 @@ info("Query functions") @test nparams(vcmodel) == p * d + m * binomial(d + 1, 2) @test nparams(vcmodelrot) == p * d + m * binomial(d + 1, 2) -info("Mean, covariance, and residual of model") +@info "Mean, covariance, and residual of model" #@code_warntype mean(vcmodel, vcdata) @inferred mean(vcmodel, vcdata) μ = mean(vcmodel, vcdata) From 6c42ce53f8eccccb44b27d46896fbe297975e9fe Mon Sep 17 00:00:00 2001 From: Juhyun Kim Date: Fri, 4 Jan 2019 13:48:06 -0800 Subject: [PATCH 02/56] replace A_mul_B! with mul! --- src/two_variance_component.jl | 96 +++++++++++++++++------------------ 1 file changed, 48 insertions(+), 48 deletions(-) diff --git a/src/two_variance_component.jl b/src/two_variance_component.jl index a5f5453..6379698 100644 --- a/src/two_variance_component.jl +++ b/src/two_variance_component.jl @@ -117,7 +117,7 @@ function gradient!( dg[j] += tmp end end - N = At_mul_B(vcaux.res, vcaux.res) + N = transpose(vcaux.res) * vcaux.res @inbounds for j in 1:d N[j, j] -= dg[j] end @@ -134,15 +134,15 @@ function gradient!( dg[j] += tmp / (tmp * λj + oneT) end end - At_mul_B!(N, vcaux.res, vcaux.res) + mul!(N, transpose(vcaux.res), vcaux.res) @inbounds for j in 1:d N[j, j] -= dg[j] end #A_mul_Bt!(N, N, vcmrot.eigvec), A_mul_B!(N, vcmrot.eigvec, N) N = vcmrot.eigvec * N * vcmrot.eigvec' - copy!(∇, N) + copyto!(∇, N) # scale by 0.5 - scale!(∇, 1//2) + rmul!(∇, 1//2) end # function gradient! """ @@ -233,7 +233,7 @@ function gradient!( T2 <: Union{VarianceComponentVariate, TwoVarCompVariateRotate}, T3 <: VarianceComponentAuxData} - copy!(∇, gradient(vcm, vcdata, vcaux)) + copyto!(∇, gradient(vcm, vcdata, vcaux)) end @@ -290,7 +290,7 @@ function fisher_Σ!( end end LinAlg.copytri!(C, 'L') - A_mul_Bt!(view(H, 1:d^2, 1:d^2), Φ2 * Diagonal(vec(C)), Φ2) + mul!(view(H, 1:d^2, 1:d^2), Φ2 * Diagonal(vec(C)), transpose(Φ2)) # (2, 1) block fill!(C, zeroT) @inbounds for j in 1:d, i in j:d @@ -301,7 +301,7 @@ function fisher_Σ!( end end LinAlg.copytri!(C, 'L') - A_mul_Bt!(view(H, (d^2+1):(2d^2), 1:d^2), Φ2 * Diagonal(vec(C)), Φ2) + mul!(view(H, (d^2+1):(2d^2), 1:d^2), Φ2 * Diagonal(vec(C)), transpose(Φ2)) # d-by-d (2, 2) block fill!(C, zeroT) @inbounds for j in 1:d, i in j:d @@ -312,10 +312,10 @@ function fisher_Σ!( end end LinAlg.copytri!(C, 'L') - A_mul_Bt!(view(H, (d^2+1):(2d^2), (d^2+1):(2d^2)), Φ2 * Diagonal(vec(C)), Φ2) + mul!(view(H, (d^2+1):(2d^2), (d^2+1):(2d^2)), Φ2 * Diagonal(vec(C)), transpose(Φ2)) # copy to upper triangular part LinAlg.copytri!(H, 'L') - scale!(H, convert(T, 0.5)) + rmul!(H, convert(T, 0.5)) # output return H end # function fisher! @@ -355,7 +355,7 @@ function fisher_Σ!( T1 <: Union{VarianceComponentModel, TwoVarCompModelRotate}, T2 <: Union{VarianceComponentVariate, TwoVarCompVariateRotate}} - copy!(H, fisher_Σ(vcm, vcobs)) + copyto!(H, fisher_Σ(vcm, vcobs)) end function fisher_Σ( @@ -406,8 +406,8 @@ function fisher_B!( vcaux.obswt[i] = oneT / √(vcaux.obswt[i] + oneT) end # weighted least squares - scale!(vcaux.obswt, vcaux.Xwork) - At_mul_B!(H, vcaux.Xwork, vcaux.Xwork) + rmul!(vcaux.obswt, vcaux.Xwork) + mul!(H, transpose(vcaux.Xwork), vcaux.Xwork) end function fisher_B( @@ -453,7 +453,7 @@ function fisher_B!( T2 <: Union{VarianceComponentVariate, TwoVarCompVariateRotate}, T3 <: VarianceComponentAuxData} - copy!(H, fisher_B(vcm, vcdata, vcaux)) + copyto!(H, fisher_B(vcm, vcdata, vcaux)) end function fisher_B( @@ -491,14 +491,14 @@ function suffstats_for_B( # working X fill!(vcaux.Xwork, zeroT) kronaxpy!(vcmrot.eigvec', vcobsrot.Xrot, vcaux.Xwork) - scale!(vcaux.obswt, vcaux.Xwork) + rmul!(vcaux.obswt, vcaux.Xwork) # working y - A_mul_B!(vcaux.res, vcobsrot.Yrot, vcmrot.eigvec) + mul!(vcaux.res, vcobsrot.Yrot, vcmrot.eigvec) @inbounds @simd for i in eachindex(vcaux.ywork) vcaux.ywork[i] = vcaux.res[i] * vcaux.obswt[i] end # output - return At_mul_B(vcaux.Xwork, vcaux.Xwork), At_mul_B(vcaux.Xwork, vcaux.ywork) + return transpose(vcaux.Xwork) * vcaux.Xwork, transpose(vcaux.Xwork) * vcaux.ywork end function suffstats_for_B( @@ -557,7 +557,7 @@ function update_meanparam!( if qpsol.status ≠ :Optimal println("Error in quadratic programming $(qpsol.status)") end - copy!(vcm.B, qpsol.sol) + copyto!(vcm.B, qpsol.sol) return vcm.B end @@ -651,8 +651,8 @@ function optimparam_to_vcparam!(dd::TwoVarCompOptProb, x::Vector{T}) where {T} dd.L[2][i, i] = exp(dd.L[2][i, i]) end # Σi = Li Li' - A_mul_Bt!(dd.vcmodel.Σ[1], dd.L[1], dd.L[1]) - A_mul_Bt!(dd.vcmodel.Σ[2], dd.L[2], dd.L[2]) + mul!(dd.vcmodel.Σ[1], dd.L[1], transpose(dd.L[1])) + mul!(dd.vcmodel.Σ[2], dd.L[2], transpose(dd.L[2])) # make sure the last variance component is pos. def. ϵ = convert(T, 1e-8) clamp_diagonal!(dd.vcmodel.Σ[2], ϵ, T(Inf)) @@ -761,8 +761,8 @@ function MathProgBase.eval_hesslag(dd::TwoVarCompOptProb, H::Vector{T}, end end # output - copy!(H, vech(dd.HL)) - scale!(H, -σ) + copyto!(H, vech(dd.HL)) + rmul!(H, -σ) end """ @@ -880,8 +880,8 @@ function mle_fs!( ub = zeros(nvar); fill!(ub, T(Inf)) MathProgBase.loadproblem!(m, nvar, 0, lb, ub, T[], T[], :Max, dd) # start point - copy!(dd.L[1], cholfact(vcmodel.Σ[1], :L, Val{true})[:L].data) - copy!(dd.L[2], cholfact(vcmodel.Σ[2], :L, Val{true})[:L].data) + copyto!(dd.L[1], cholfact(vcmodel.Σ[1], :L, Val{true})[:L].data) + copyto!(dd.L[2], cholfact(vcmodel.Σ[2], :L, Val{true})[:L].data) # reparameterize diagonal entries to exponential @inbounds @simd for i in 1:d dd.L[1][i, i] = log(dd.L[1][i, i] + convert(T, 1e-8)) @@ -904,11 +904,11 @@ function mle_fs!( Bcov = zeros(T, nmean, nmean) Bcov = pinv(fisher_B(vcmodel, vcdatarot, dd.vcaux)) Bse = similar(vcmodel.B) - copy!(Bse, sqrt.(diag(Bcov))) + copyto!(Bse, sqrt.(diag(Bcov))) Σcov = pinv(fisher_Σ(vcmodel, vcdatarot)) Σse = (zeros(T, d, d), zeros(T, d, d)) - copy!(Σse[1], sqrt.(diag(view(Σcov, 1:d^2, 1:d^2)))) - copy!(Σse[2], sqrt.(diag(view(Σcov, d^2+1:2d^2, d^2+1:2d^2)))) + copyto!(Σse[1], sqrt.(diag(view(Σcov, 1:d^2, 1:d^2)))) + copyto!(Σse[2], sqrt.(diag(view(Σcov, d^2+1:2d^2, d^2+1:2d^2)))) # output maxlogl, vcmodel, Σse, Σcov, Bse, Bcov @@ -941,7 +941,7 @@ function suffstats_for_Σ!( b2[j] += tmp end end - At_mul_B!(A2, vcaux.res, vcaux.res) + mul!(A2, transpose(vcaux.res), vcaux.res) # sufficient statistics for Σ1 @inbounds for j in 1:d λj = vcmrot.eigval[j] @@ -951,7 +951,7 @@ function suffstats_for_Σ!( b1[j] += tmp / (tmp * λj + oneT) end end - At_mul_B!(A1, vcaux.res, vcaux.res) + mul!(A1, transpose(vcaux.res), vcaux.res) # output return A1, b1, A2, b2 end @@ -997,25 +997,25 @@ function mm_update_Σ!( end Φinv = inv(vcmrot.eigvec) # update Σ1 - scale!(b1, A1), scale!(A1, b1) + rmul!(b1, A1), rmul!(A1, b1) storage = eigfact!(Symmetric(A1)) @inbounds for i in 1:d storage.values[i] = storage.values[i] > zeroT ? √√storage.values[i] : zeroT end - scale!(storage.vectors, storage.values) - scale!(oneT ./ b1, storage.vectors) - At_mul_B!(vcm.Σ[1], Φinv, storage.vectors) - copy!(vcm.Σ[1], A_mul_Bt(vcm.Σ[1], vcm.Σ[1])) + rmul!(storage.vectors, storage.values) + rmul!(oneT ./ b1, storage.vectors) + mul!(vcm.Σ[1], transpose(Φinv), storage.vectors) + copyto!(vcm.Σ[1], vcm.Σ[1] * transpose(vcm.Σ[1])) # update Σ2 - scale!(b2, A2), scale!(A2, b2) + rmul!(b2, A2), rmul!(A2, b2) storage = eigfact!(Symmetric(A2)) @inbounds for i in 1:d storage.values[i] = storage.values[i] > zeroT ? √√storage.values[i] : zeroT end - scale!(storage.vectors, storage.values) - scale!(oneT ./ b2, storage.vectors) - At_mul_B!(vcm.Σ[2], Φinv, storage.vectors) - copy!(vcm.Σ[2], A_mul_Bt(vcm.Σ[2], vcm.Σ[2])) + rmul!(storage.vectors, storage.values) + rmul!(oneT ./ b2, storage.vectors) + mul!(vcm.Σ[2], transpose(Φinv), storage.vectors) + copyto!(vcm.Σ[2], vcm.Σ[2] * transpose(vcm.Σ[2])) end """ @@ -1105,11 +1105,11 @@ function mle_mm!( # standard errors Bcov = pinv(fisher_B(vcm, vcdatarot, vcaux)) Bse = similar(vcm.B) - copy!(Bse, sqrt.(diag(Bcov))) + copyto!(Bse, sqrt.(diag(Bcov))) Σcov = pinv(fisher_Σ(vcm, vcdatarot)) Σse = (zeros(T, d, d), zeros(T, d, d)) - copy!(Σse[1], sqrt.(diag(view(Σcov, 1:d^2, 1:d^2)))) - copy!(Σse[2], sqrt.(diag(view(Σcov, d^2+1:2d^2, d^2+1:2d^2)))) + copyto!(Σse[1], sqrt.(diag(view(Σcov, 1:d^2, 1:d^2)))) + copyto!(Σse[2], sqrt.(diag(view(Σcov, d^2+1:2d^2, d^2+1:2d^2)))) # output logl, vcm, Σse, Σcov, Bse, Bcov @@ -1198,18 +1198,18 @@ function fit_reml!( qs = MosekSolver(MSK_IPAR_LOG = 0) end if typeof(vcdata) <: AbstractArray - Q = kron(Matrix{T}(I, d, d), mapreduce(x -> At_mul_B(x.X, x.X), +, vcdata)) - c = vec(mapreduce(x -> At_mul_B(x.X, x.Y), +, vcdata)) + Q = kron(Matrix{T}(I, d, d), mapreduce(x -> transpose(x.X) * x.X, +, vcdata)) + c = vec(mapreduce(x -> transpose(x.X) * x.Y, +, vcdata)) else - Q = kron(Matrix{T}(I, d, d), At_mul_B(vcdata.X, vcdata.X)) - c = vec(At_mul_B(vcdata.X, vcdata.Y)) + Q = kron(Matrix{T}(I, d, d), transpose(vcdata.X) * vcdata.X) + c = vec(transpose(vcdata.X) * vcdata.Y) end qpsol = quadprog(-c, Q, vcmodel.A, vcmodel.sense, vcmodel.b, vcmodel.lb, vcmodel.ub, qs) if qpsol.status ≠ :Optimal println("Error in quadratic programming $(qpsol.status)") end - copy!(vcmodel.B, qpsol.sol) + copyto!(vcmodel.B, qpsol.sol) # use residuals as responses resdata = deepcopy(vcdata) if typeof(vcdata) <: AbstractArray @@ -1231,14 +1231,14 @@ function fit_reml!( end # estimate mean parameters from covariance estimate - copy!(vcmodel.Σ[1], resmodel.Σ[1]), copy!(vcmodel.Σ[2], resmodel.Σ[2]) + copyto!(vcmodel.Σ[1], resmodel.Σ[1]), copyto!(vcmodel.Σ[2], resmodel.Σ[2]) vcdatarot = TwoVarCompVariateRotate(vcdata) update_meanparam!(vcmodel, vcdatarot, qs, vcaux) # standard errors and covariance of mean parameters Bcov = inv(fisher_B(vcmodel, vcdatarot, vcaux)) Bse = similar(vcmodel.B) - copy!(Bse, sqrt.(diag(Bcov))) + copyto!(Bse, sqrt.(diag(Bcov))) # output logpdf(vcmodel, vcdatarot), vcmodel, Σse, Σcov, Bse, Bcov From b3eb78314beafc314f5b1f3269a951ed4f4a1516 Mon Sep 17 00:00:00 2001 From: Juhyun Kim Date: Sat, 5 Jan 2019 17:34:18 -0800 Subject: [PATCH 03/56] replace A_mul_B! and eye(n) --- src/VarianceComponentModels.jl | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/VarianceComponentModels.jl b/src/VarianceComponentModels.jl index bf25ef5..91c2dee 100644 --- a/src/VarianceComponentModels.jl +++ b/src/VarianceComponentModels.jl @@ -4,10 +4,11 @@ using Compat import Compat.view using LinearAlgebra, Statistics, MathProgBase, Ipopt#, KNITRO#, Mosek#, Gurobi -import Base: eltype, length, size, + +import Base: eltype, length, size, + export VarianceComponentModel, VarianceComponentVariate, TwoVarCompModelRotate, TwoVarCompVariateRotate, VarianceComponentAuxData, - residual, residual!, nvarcomps, nmeanparams, nvarparams, nparams + residual, residual!, nvarcomps, nmeanparams, nvarparams, nparams, + mean!, mean, cov!, cov # import Base: eltype, length, size, mean, mean!, cov, + # export VarianceComponentModel, VarianceComponentVariate, @@ -239,9 +240,11 @@ function TwoVarCompVariateRotate(vcobs::VarianceComponentVariate{T, 2}) where T end function TwoVarCompVariateRotate(vcdata::Array{T}) where T <: VarianceComponentVariate + map(x -> TwoVarCompVariateRotate(x), vcdata) end + """ VarianceComponentModel(vcobs) @@ -253,7 +256,7 @@ function VarianceComponentModel(vcobs::VarianceComponentVariate{T, M}) where {T, p, d, m = size(vcobs.X, 2), size(vcobs.Y, 2), length(vcobs.V) B = zeros(T, p, d) - Σ = ntuple(x -> Matrix(I, T, d), m)::NTuple{M, Matrix{T}} + Σ = ntuple(x -> Matrix{T}(I, d, d), m)::NTuple{M, Matrix{T}} VarianceComponentModel(B, Σ) end @@ -268,7 +271,7 @@ function VarianceComponentModel(vcobsrot::TwoVarCompVariateRotate) p, d = size(vcobsrot.Xrot, 2), size(vcobsrot.Yrot, 2) T = eltype(vcobsrot) B = zeros(T, p, d) - Σ = (Matrix(I, T, d), Matrix(I, T, d)) + Σ = (Matrix{T}(I, d, d), Matrix{T}(I, d, d)) VarianceComponentModel(B, Σ) end @@ -441,7 +444,7 @@ function mean!( vcobs::VarianceComponentVariate ) - A_mul_B!(μ, vcobs.X, vcm.B) + mul!(μ, vcobs.X, vcm.B) end """ @@ -525,11 +528,11 @@ function residual!( ) if isempty(vcmrot.Brot) - A_mul_B!(resid, vcobsrot.Yrot, vcmrot.eigvec) + mul!(resid, vcobsrot.Yrot, vcmrot.eigvec) else #copy!(resid, vcobsrot.Yrot * vcmrot.eigvec - vcobsrot.Xrot * vcmrot.Brot) oneT = one(eltype(vcmrot)) - A_mul_B!(resid, vcobsrot.Yrot, vcmrot.eigvec) + mul!(resid, vcobsrot.Yrot, vcmrot.eigvec) LinAlg.BLAS.gemm!('N', 'N', -oneT, vcobsrot.Xrot, vcmrot.Brot, oneT, resid) end resid From cdaa15ba1cfd6ce7ed280d8f8d3c5693bc042c2a Mon Sep 17 00:00:00 2001 From: Juhyun Kim Date: Sat, 5 Jan 2019 17:43:55 -0800 Subject: [PATCH 04/56] replace some rmul! with lmul! --- src/two_variance_component.jl | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/src/two_variance_component.jl b/src/two_variance_component.jl index 6379698..8258bd3 100644 --- a/src/two_variance_component.jl +++ b/src/two_variance_component.jl @@ -1,4 +1,5 @@ -import LinearAlgebra, Printf +using LinearAlgebra +import Printf export heritability, logpdf, @@ -343,7 +344,7 @@ function fisher_Σ( vcm::VarianceComponentModel{T, 2}, vcobsrot::Union{TwoVarCompVariateRotate{T}, VarianceComponentVariate{T, 2}} ) where {T <: AbstractFloat} - + fisher_Σ(TwoVarCompModelRotate(vcm), TwoVarCompVariateRotate(vcobsrot)) end @@ -406,7 +407,7 @@ function fisher_B!( vcaux.obswt[i] = oneT / √(vcaux.obswt[i] + oneT) end # weighted least squares - rmul!(vcaux.obswt, vcaux.Xwork) + lmul!(Diagonal(vcaux.obswt), vcaux.Xwork) mul!(H, transpose(vcaux.Xwork), vcaux.Xwork) end @@ -491,7 +492,7 @@ function suffstats_for_B( # working X fill!(vcaux.Xwork, zeroT) kronaxpy!(vcmrot.eigvec', vcobsrot.Xrot, vcaux.Xwork) - rmul!(vcaux.obswt, vcaux.Xwork) + lmul!(Diagonal(vcaux.obswt), vcaux.Xwork) # working y mul!(vcaux.res, vcobsrot.Yrot, vcmrot.eigvec) @inbounds @simd for i in eachindex(vcaux.ywork) @@ -716,7 +717,8 @@ end # function MathProgBase.eval_grad_f function MathProgBase.hesslag_structure(dd::TwoVarCompOptProb) nvar = nvarparams(dd.vcmodel) # linear indices for variance parameters - ind2sub((nvar, nvar), trilind(nvar)) + ## ind2sub((nvar, nvar), trilind(nvar)) + Tuple(CartesianIndices((nvar, nvar))[trilind(nvar)]) end # function MathProgBase.hesslag_structure function MathProgBase.eval_hesslag(dd::TwoVarCompOptProb, H::Vector{T}, @@ -880,8 +882,8 @@ function mle_fs!( ub = zeros(nvar); fill!(ub, T(Inf)) MathProgBase.loadproblem!(m, nvar, 0, lb, ub, T[], T[], :Max, dd) # start point - copyto!(dd.L[1], cholfact(vcmodel.Σ[1], :L, Val{true})[:L].data) - copyto!(dd.L[2], cholfact(vcmodel.Σ[2], :L, Val{true})[:L].data) + copyto!(dd.L[1], cholesky(vcmodel.Σ[1], Val(true)).L.data) + copyto!(dd.L[2], cholesky(vcmodel.Σ[2], Val(true)).L.data) # reparameterize diagonal entries to exponential @inbounds @simd for i in 1:d dd.L[1][i, i] = log(dd.L[1][i, i] + convert(T, 1e-8)) @@ -997,23 +999,23 @@ function mm_update_Σ!( end Φinv = inv(vcmrot.eigvec) # update Σ1 - rmul!(b1, A1), rmul!(A1, b1) + lmul!(Diagonal(b1), A1), rmul!(A1, b1) storage = eigfact!(Symmetric(A1)) @inbounds for i in 1:d storage.values[i] = storage.values[i] > zeroT ? √√storage.values[i] : zeroT end - rmul!(storage.vectors, storage.values) - rmul!(oneT ./ b1, storage.vectors) + scale!(storage.vectors, storage.values) + scale!(oneT ./ b1, storage.vectors) mul!(vcm.Σ[1], transpose(Φinv), storage.vectors) copyto!(vcm.Σ[1], vcm.Σ[1] * transpose(vcm.Σ[1])) # update Σ2 - rmul!(b2, A2), rmul!(A2, b2) + lmul!(Diagonal(b2), A2), rmul!(A2, b2) storage = eigfact!(Symmetric(A2)) @inbounds for i in 1:d storage.values[i] = storage.values[i] > zeroT ? √√storage.values[i] : zeroT end - rmul!(storage.vectors, storage.values) - rmul!(oneT ./ b2, storage.vectors) + scale!(storage.vectors, storage.values) + scale!(oneT ./ b2, storage.vectors) mul!(vcm.Σ[2], transpose(Φinv), storage.vectors) copyto!(vcm.Σ[2], vcm.Σ[2] * transpose(vcm.Σ[2])) end From 58bb81e1a6977fa0e671fb678ed63cb9fe44de25 Mon Sep 17 00:00:00 2001 From: Juhyun Kim Date: Sat, 5 Jan 2019 17:46:20 -0800 Subject: [PATCH 05/56] avoid conflict with gradient function in LinearAlgebra --- test/two_variance_component_test.jl | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/test/two_variance_component_test.jl b/test/two_variance_component_test.jl index 32c6b3b..3ba742a 100644 --- a/test/two_variance_component_test.jl +++ b/test/two_variance_component_test.jl @@ -1,6 +1,6 @@ module TwoVarianceComponentTest -using VarianceComponentModels, MathProgBase, Ipopt, Random, LinearAlgebra, Test +using VarianceComponentModels, MathProgBase, Ipopt, Random, Test, LinearAlgebra Random.seed!(123) @@ -48,16 +48,27 @@ vcmodelrot = TwoVarCompModelRotate(vcmodel) @test (logpdf(vcmodelrot, [vcdatarot vcdatarot; vcdatarot vcdatarot]) - logpdf(vcmodel, [vcdata vcdata; vcdata vcdata])) ≈ 0.0 +# @info "Evaluate gradient" +# ∇ = zeros(2d^2) +# #@code_warntype gradient!(∇, vcmodelrot, vcdatarot) +# @inferred gradient!(∇, vcmodelrot, vcdatarot) +# @test norm(gradient(vcmodel, vcdata) - gradient(vcmodelrot, vcdatarot)) ≈ 0.0 +# @test norm(gradient(vcmodel, vcdata) - gradient(vcmodel, vcdatarot)) ≈ 0.0 +# @test norm(gradient(vcmodel, [vcdata vcdata]) - +# 2.0gradient(vcmodel, vcdata)) ≈ 0.0 +# @test norm(gradient(vcmodel, [vcdata vcdata]) - +# gradient(vcmodelrot, [vcdatarot vcdatarot])) ≈ 0.0 + @info "Evaluate gradient" ∇ = zeros(2d^2) #@code_warntype gradient!(∇, vcmodelrot, vcdatarot) -@inferred gradient!(∇, vcmodelrot, vcdatarot) -@test norm(gradient(vcmodel, vcdata) - gradient(vcmodelrot, vcdatarot)) ≈ 0.0 -@test norm(gradient(vcmodel, vcdata) - gradient(vcmodel, vcdatarot)) ≈ 0.0 -@test norm(gradient(vcmodel, [vcdata vcdata]) - - 2.0gradient(vcmodel, vcdata)) ≈ 0.0 -@test norm(gradient(vcmodel, [vcdata vcdata]) - - gradient(vcmodelrot, [vcdatarot vcdatarot])) ≈ 0.0 +@inferred VarianceComponentModels.gradient!(∇, vcmodelrot, vcdatarot) +@test norm(VarianceComponentModels.gradient(vcmodel, vcdata) - VarianceComponentModels.gradient(vcmodelrot, vcdatarot)) ≈ 0.0 +@test norm(VarianceComponentModels.gradient(vcmodel, vcdata) - VarianceComponentModels.gradient(vcmodel, vcdatarot)) ≈ 0.0 +@test norm(VarianceComponentModels.gradient(vcmodel, [vcdata vcdata]) - + 2.0VarianceComponentModels.gradient(vcmodel, vcdata)) ≈ 0.0 +@test norm(VarianceComponentModels.gradient(vcmodel, [vcdata vcdata]) - +VarianceComponentModels.gradient(vcmodelrot, [vcdatarot vcdatarot])) ≈ 0.0 @info "Evaluate Fisher information matrix of Σ" H = zeros(2d^2, 2d^2) From 166d20bf19e5ff8e53b5d9b71402f57ad96a3fbf Mon Sep 17 00:00:00 2001 From: Juhyun Kim Date: Sat, 5 Jan 2019 18:13:48 -0800 Subject: [PATCH 06/56] minor edits --- test/variance_component_models_test.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/variance_component_models_test.jl b/test/variance_component_models_test.jl index b55204c..a3bf864 100644 --- a/test/variance_component_models_test.jl +++ b/test/variance_component_models_test.jl @@ -58,7 +58,7 @@ vcdata = VarianceComponentVariate(Y, X, V) vcdata = VarianceComponentVariate(Y, X, V) #@code_warntype VarianceComponentModel(vcdata) @inferred VarianceComponentModel(vcdata) -vcmodel = VarianceComponentModel(vcdata) +vcmodel = VarianceComponentModel(vcdata) @test norm(vcmodel.B) ≈ 0.0 @test norm(vcmodel.Σ[1] - Matrix(I, d, d)) ≈ 0.0 @test norm(vcmodel.Σ[2] - Matrix(I, d, d)) ≈ 0.0 From 03d494f65e2229b359d9ed4f2a806d2b313e5049 Mon Sep 17 00:00:00 2001 From: Juhyun Kim Date: Mon, 7 Jan 2019 20:41:29 -0800 Subject: [PATCH 07/56] replace LinAlg with LinearAlgebra; remove Compat --- src/VarianceComponentModels.jl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/VarianceComponentModels.jl b/src/VarianceComponentModels.jl index 91c2dee..ea2453a 100644 --- a/src/VarianceComponentModels.jl +++ b/src/VarianceComponentModels.jl @@ -1,7 +1,7 @@ module VarianceComponentModels -using Compat -import Compat.view +#using Compat +#import Compat.view using LinearAlgebra, Statistics, MathProgBase, Ipopt#, KNITRO#, Mosek#, Gurobi import Base: eltype, length, size, + @@ -116,7 +116,7 @@ function TwoVarCompModelRotate(vcm::VarianceComponentModel{T, 2, BT, ΣT}) where λ = convert(Vector{T}, F.values) Φ = convert(Matrix{T}, F.vectors) # correct negative eigenvalues due to roundoff - @compat map!(x -> max(x, zero(T)), λ, λ) + map!(x -> max(x, zero(T)), λ, λ) Brot = isempty(vcm.B) ? Array{T}(undef, size(vcm.B)) : vcm.B * Φ logdetΣ2 = convert(T, logdet(vcm.Σ[2])) TwoVarCompModelRotate{T, BT}(Brot, λ, Φ, logdetΣ2) @@ -486,7 +486,7 @@ function residual!( else oneT = one(eltype(vcm)) copyto!(resid, vcobs.Y) - LinAlg.BLAS.gemm!('N', 'N', -oneT, vcobs.X, vcm.B, oneT, resid) + LinearAlgebra.BLAS.gemm!('N', 'N', -oneT, vcobs.X, vcm.B, oneT, resid) end resid end @@ -533,7 +533,7 @@ function residual!( #copy!(resid, vcobsrot.Yrot * vcmrot.eigvec - vcobsrot.Xrot * vcmrot.Brot) oneT = one(eltype(vcmrot)) mul!(resid, vcobsrot.Yrot, vcmrot.eigvec) - LinAlg.BLAS.gemm!('N', 'N', -oneT, vcobsrot.Xrot, vcmrot.Brot, oneT, resid) + LinearAlgebra.BLAS.gemm!('N', 'N', -oneT, vcobsrot.Xrot, vcmrot.Brot, oneT, resid) end resid end From e3db83a7a8dd52fddf27e3371157363190aeda16 Mon Sep 17 00:00:00 2001 From: Juhyun Kim Date: Mon, 7 Jan 2019 20:48:54 -0800 Subject: [PATCH 08/56] avoid calling TwoVarCompVariateRotate for its own type by spliting Union of types --- src/two_variance_component.jl | 160 ++++++++++++++++++++++++++++++---- 1 file changed, 145 insertions(+), 15 deletions(-) diff --git a/src/two_variance_component.jl b/src/two_variance_component.jl index 8258bd3..07c7efd 100644 --- a/src/two_variance_component.jl +++ b/src/two_variance_component.jl @@ -1,4 +1,3 @@ -using LinearAlgebra import Printf export heritability, @@ -57,7 +56,19 @@ function logpdf( vcaux::T3 = VarianceComponentAuxData(vcobs) ) where { T1 <: VarianceComponentModel, - T2 <: Union{TwoVarCompVariateRotate, VarianceComponentVariate}, + T2 <: TwoVarCompVariateRotate, + T3 <: VarianceComponentAuxData} + + logpdf(TwoVarCompModelRotate(vcm), vcobs, vcaux) +end + +function logpdf( + vcm::T1, + vcobs::T2, + vcaux::T3 = VarianceComponentAuxData(vcobs) + ) where { + T1 <: VarianceComponentModel, + T2 <: VarianceComponentVariate, T3 <: VarianceComponentAuxData} logpdf(TwoVarCompModelRotate(vcm), TwoVarCompVariateRotate(vcobs), vcaux) @@ -192,7 +203,17 @@ Evaluate gradient of `Σ` at `vcmrot.Σ` and overwrite `∇`. function gradient!( ∇::AbstractVector{T}, vcm::VarianceComponentModel{T, 2}, - vcobs::Union{TwoVarCompVariateRotate{T}, VarianceComponentVariate{T, 2}}, + vcobs::TwoVarCompVariateRotate{T}, + vcaux::VarianceComponentAuxData = VarianceComponentAuxData(vcobs) + ) where {T <: AbstractFloat} + + gradient!(∇, TwoVarCompModelRotate(vcm), vcobs, vcaux) +end + +function gradient!( + ∇::AbstractVector{T}, + vcm::VarianceComponentModel{T, 2}, + vcobs::VarianceComponentVariate{T, 2}, vcaux::VarianceComponentAuxData = VarianceComponentAuxData(vcobs) ) where {T <: AbstractFloat} @@ -217,7 +238,16 @@ Evaluate gradient of `Σ` at `vcmrot.Σ` and overwrite `∇`. """ function gradient( vcm::VarianceComponentModel{T, 2}, - vcobs::Union{TwoVarCompVariateRotate{T}, VarianceComponentVariate{T, 2}}, + vcobs::TwoVarCompVariateRotate, + vcaux::VarianceComponentAuxData = VarianceComponentAuxData(vcobs) + ) where {T <: AbstractFloat} + + gradient(TwoVarCompModelRotate(vcm), vcobs, vcaux) +end + +function gradient( + vcm::VarianceComponentModel{T, 2}, + vcobs::VarianceComponentVariate{T, 2}, vcaux::VarianceComponentAuxData = VarianceComponentAuxData(vcobs) ) where {T <: AbstractFloat} @@ -290,7 +320,7 @@ function fisher_Σ!( C[i, j] += deval * deval / (λi * deval + oneT) / (λj * deval + oneT) end end - LinAlg.copytri!(C, 'L') + LinearAlgebra.copytri!(C, 'L') mul!(view(H, 1:d^2, 1:d^2), Φ2 * Diagonal(vec(C)), transpose(Φ2)) # (2, 1) block fill!(C, zeroT) @@ -301,7 +331,7 @@ function fisher_Σ!( C[i, j] += deval / (λi * deval + oneT) / (λj * deval + oneT) end end - LinAlg.copytri!(C, 'L') + LinearAlgebra.copytri!(C, 'L') mul!(view(H, (d^2+1):(2d^2), 1:d^2), Φ2 * Diagonal(vec(C)), transpose(Φ2)) # d-by-d (2, 2) block fill!(C, zeroT) @@ -312,10 +342,10 @@ function fisher_Σ!( C[i, j] += oneT / (λi * deval + oneT) / (λj * deval + oneT) end end - LinAlg.copytri!(C, 'L') + LinearAlgebra.copytri!(C, 'L') mul!(view(H, (d^2+1):(2d^2), (d^2+1):(2d^2)), Φ2 * Diagonal(vec(C)), transpose(Φ2)) # copy to upper triangular part - LinAlg.copytri!(H, 'L') + LinearAlgebra.copytri!(H, 'L') rmul!(H, convert(T, 0.5)) # output return H @@ -334,7 +364,16 @@ end function fisher_Σ!( H::AbstractMatrix{T}, vcm::VarianceComponentModel{T, 2}, - vcobsrot::Union{TwoVarCompVariateRotate{T}, VarianceComponentVariate{T, 2}} + vcobsrot::TwoVarCompVariateRotate{T} + ) where {T <: AbstractFloat} + + fisher_Σ!(H, TwoVarCompModelRotate(vcm), vcobsrot) +end + +function fisher_Σ!( + H::AbstractMatrix{T}, + vcm::VarianceComponentModel{T, 2}, + vcobsrot::VarianceComponentVariate{T, 2} ) where {T <: AbstractFloat} fisher_Σ!(H, TwoVarCompModelRotate(vcm), TwoVarCompVariateRotate(vcobsrot)) @@ -342,12 +381,22 @@ end function fisher_Σ( vcm::VarianceComponentModel{T, 2}, - vcobsrot::Union{TwoVarCompVariateRotate{T}, VarianceComponentVariate{T, 2}} + vcobsrot::VarianceComponentVariate{T, 2} ) where {T <: AbstractFloat} - + fisher_Σ(TwoVarCompModelRotate(vcm), TwoVarCompVariateRotate(vcobsrot)) end +function fisher_Σ( + vcm::VarianceComponentModel{T, 2}, + vcobsrot::TwoVarCompVariateRotate{T} + ) where {T <: AbstractFloat} + + fisher_Σ(TwoVarCompModelRotate(vcm), vcobsrot) +end + + + function fisher_Σ!( H::AbstractMatrix, vcm::T1, @@ -420,30 +469,111 @@ function fisher_B( fisher_B!(H, vcmrot, vcobsrot) end +# function fisher_B!( +# H::AbstractMatrix{T}, +# vcm::Union{VarianceComponentModel{T, 2}, TwoVarCompVariateRotate{T}}, +# vcobs::Union{TwoVarCompVariateRotate{T}, VarianceComponentVariate{T, 2}}, +# vcaux::VarianceComponentAuxData = VarianceComponentAuxData(vcobs) +# ) where {T <: AbstractFloat} + +# fisher_B!(H, TwoVarCompModelRotate(vcm), TwoVarCompVariateRotate(vcobs), vcaux) +# end function fisher_B!( H::AbstractMatrix{T}, - vcm::Union{VarianceComponentModel{T, 2}, TwoVarCompVariateRotate{T}}, - vcobs::Union{TwoVarCompVariateRotate{T}, VarianceComponentVariate{T, 2}}, + vcm::VarianceComponentModel{T, 2}, + vcobs::VarianceComponentVariate{T, 2}, vcaux::VarianceComponentAuxData = VarianceComponentAuxData(vcobs) ) where {T <: AbstractFloat} + println("this one?") fisher_B!(H, TwoVarCompModelRotate(vcm), TwoVarCompVariateRotate(vcobs), vcaux) end +function fisher_B!( + H::AbstractMatrix{T}, + vcm::TwoVarCompVariateRotate{T}, + vcobs::VarianceComponentVariate{T, 2}, + vcaux::VarianceComponentAuxData = VarianceComponentAuxData(vcobs) + ) where {T <: AbstractFloat} + + println("this one?") + fisher_B!(H, vcm, TwoVarCompVariateRotate(vcobs), vcaux) +end + +function fisher_B!( + H::AbstractMatrix{T}, + vcm::VarianceComponentModel{T, 2}, + vcobs::TwoVarCompVariateRotate{T}, + vcaux::VarianceComponentAuxData = VarianceComponentAuxData(vcobs) + ) where {T <: AbstractFloat} + println("this two?") + fisher_B!(H, TwoVarCompModelRotate(vcm), vcobs, vcaux) +end + +function fisher_B!( + H::AbstractMatrix{T}, + vcm::TwoVarCompVariateRotate{T}, + vcobs::TwoVarCompVariateRotate{T}, + vcaux::VarianceComponentAuxData = VarianceComponentAuxData(vcobs) + ) where {T <: AbstractFloat} + println("this two?") + fisher_B!(H, vcm, vcobs, vcaux) +end + function fisher_B( vcm::T1, vcobs::T2, vcaux::T3 = VarianceComponentAuxData(vcobs) ) where { - T1 <: Union{VarianceComponentModel, TwoVarCompModelRotate}, - T2 <: Union{VarianceComponentVariate, TwoVarCompVariateRotate}, + T1 <: VarianceComponentModel, + T2 <: VarianceComponentVariate, T3 <: VarianceComponentAuxData} H = zeros(eltype(vcm), nmeanparams(vcm), nmeanparams(vcm)) fisher_B!(H, TwoVarCompModelRotate(vcm), TwoVarCompVariateRotate(vcobs), vcaux) end +function fisher_B( + vcm::T1, + vcobs::T2, + vcaux::T3 = VarianceComponentAuxData(vcobs) + ) where { + T1 <: TwoVarCompModelRotate, + T2 <: VarianceComponentVariate, + T3 <: VarianceComponentAuxData} + + H = zeros(eltype(vcm), nmeanparams(vcm), nmeanparams(vcm)) + fisher_B!(H, vcm, TwoVarCompVariateRotate(vcobs), vcaux) +end + +function fisher_B( + vcm::T1, + vcobs::T2, + vcaux::T3 = VarianceComponentAuxData(vcobs) + ) where { + T1 <: VarianceComponentModel, + T2 <: TwoVarCompVariateRotate, + T3 <: VarianceComponentAuxData} + + H = zeros(eltype(vcm), nmeanparams(vcm), nmeanparams(vcm)) + fisher_B!(H, TwoVarCompModelRotate(vcm), vcobs, vcaux) +end + +function fisher_B( + vcm::T1, + vcobs::T2, + vcaux::T3 = VarianceComponentAuxData(vcobs) + ) where { + T1 <: TwoVarCompModelRotate, + T2 <: TwoVarCompVariateRotate, + T3 <: VarianceComponentAuxData} + + H = zeros(eltype(vcm), nmeanparams(vcm), nmeanparams(vcm)) + fisher_B!(H, vcm, vcobs, vcaux) +end + + function fisher_B!( H::AbstractMatrix, vcm::T1, From 95b37bb0d8e0e98052f7be1364f81e2423ff38bf Mon Sep 17 00:00:00 2001 From: Juhyun Kim Date: Tue, 8 Jan 2019 21:33:29 -0800 Subject: [PATCH 09/56] replace Tuple to match with ind2sub output --- src/two_variance_component.jl | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/src/two_variance_component.jl b/src/two_variance_component.jl index 07c7efd..4258ccb 100644 --- a/src/two_variance_component.jl +++ b/src/two_variance_component.jl @@ -486,7 +486,6 @@ function fisher_B!( vcaux::VarianceComponentAuxData = VarianceComponentAuxData(vcobs) ) where {T <: AbstractFloat} - println("this one?") fisher_B!(H, TwoVarCompModelRotate(vcm), TwoVarCompVariateRotate(vcobs), vcaux) end @@ -497,7 +496,6 @@ function fisher_B!( vcaux::VarianceComponentAuxData = VarianceComponentAuxData(vcobs) ) where {T <: AbstractFloat} - println("this one?") fisher_B!(H, vcm, TwoVarCompVariateRotate(vcobs), vcaux) end @@ -507,7 +505,7 @@ function fisher_B!( vcobs::TwoVarCompVariateRotate{T}, vcaux::VarianceComponentAuxData = VarianceComponentAuxData(vcobs) ) where {T <: AbstractFloat} - println("this two?") + fisher_B!(H, TwoVarCompModelRotate(vcm), vcobs, vcaux) end @@ -517,7 +515,7 @@ function fisher_B!( vcobs::TwoVarCompVariateRotate{T}, vcaux::VarianceComponentAuxData = VarianceComponentAuxData(vcobs) ) where {T <: AbstractFloat} - println("this two?") + fisher_B!(H, vcm, vcobs, vcaux) end @@ -746,7 +744,7 @@ function TwoVarCompOptProb( ) where { T1 <: VarianceComponentModel, T2 <: TwoVarCompVariateRotate} - + T = eltype(vcm) d, pd = length(vcm), nmeanparams(vcm) # number of optimization parameters in variance @@ -757,6 +755,7 @@ function TwoVarCompOptProb( HΣ = zeros(T, 2d^2, 2d^2) # Hessian wrt (Σ1, Σ2) HL = zeros(T, nvar, nvar) # Hessian wrt Ls vcaux = VarianceComponentAuxData(vcdatarot) + # constructor TwoVarCompOptProb{typeof(vcm), typeof(vcdatarot), typeof(HΣ), typeof(∇Σ), typeof(vcaux)}(vcm, vcdatarot, qpsolver, L, ∇Σ, HΣ, HL, vcaux) @@ -845,10 +844,17 @@ function MathProgBase.eval_grad_f( end # function MathProgBase.eval_grad_f function MathProgBase.hesslag_structure(dd::TwoVarCompOptProb) + nvar = nvarparams(dd.vcmodel) # linear indices for variance parameters ## ind2sub((nvar, nvar), trilind(nvar)) - Tuple(CartesianIndices((nvar, nvar))[trilind(nvar)]) + #Tuple(CartesianIndices((nvar, nvar))[trilind(nvar)]) + arr1, arr2 = Int64[], Int64[] + for i in 1:nvar + arr1 = vcat(arr1, i:nvar) + arr2 = vcat(arr2, fill(i, nvar-i+1)) + end + return (arr1, arr2) end # function MathProgBase.hesslag_structure function MathProgBase.eval_hesslag(dd::TwoVarCompOptProb, H::Vector{T}, @@ -895,6 +901,7 @@ function MathProgBase.eval_hesslag(dd::TwoVarCompOptProb, H::Vector{T}, # output copyto!(H, vech(dd.HL)) rmul!(H, -σ) + end """ @@ -949,7 +956,6 @@ function mle_fs!( qs = MosekSolver(MSK_IPAR_LOG = 0) end dd = TwoVarCompOptProb(vcmodel, vcdatarot, qs) - # set up MathProgBase interface if solver == :Ipopt # see http://www.coin-or.org/Ipopt/documentation/documentation.html for IPOPT @@ -1129,12 +1135,13 @@ function mm_update_Σ!( end Φinv = inv(vcmrot.eigvec) # update Σ1 - lmul!(Diagonal(b1), A1), rmul!(A1, b1) - storage = eigfact!(Symmetric(A1)) + lmul!(Diagonal(b1), A1), rmul!(A1, Diagonal(b1)) + storage = eigen!(Symmetric(A1)) @inbounds for i in 1:d storage.values[i] = storage.values[i] > zeroT ? √√storage.values[i] : zeroT end - scale!(storage.vectors, storage.values) + #scale!(storage.vectors, storage.values) + rmul!(storage.vectors, Diagonal(storage.values)) scale!(oneT ./ b1, storage.vectors) mul!(vcm.Σ[1], transpose(Φinv), storage.vectors) copyto!(vcm.Σ[1], vcm.Σ[1] * transpose(vcm.Σ[1])) From 1cd5d0f5dd1c8d3b26598c3fd6fcb6b0337b32e4 Mon Sep 17 00:00:00 2001 From: Juhyun Kim Date: Wed, 9 Jan 2019 08:14:22 -0800 Subject: [PATCH 10/56] replace scale! and eigfact! --- src/two_variance_component.jl | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/two_variance_component.jl b/src/two_variance_component.jl index 4258ccb..5f854bb 100644 --- a/src/two_variance_component.jl +++ b/src/two_variance_component.jl @@ -1140,19 +1140,18 @@ function mm_update_Σ!( @inbounds for i in 1:d storage.values[i] = storage.values[i] > zeroT ? √√storage.values[i] : zeroT end - #scale!(storage.vectors, storage.values) - rmul!(storage.vectors, Diagonal(storage.values)) - scale!(oneT ./ b1, storage.vectors) + rmul!(storage.vectors, Diagonal(storage.values)) #scale!(storage.vectors, storage.values) + lmul!(Diagonal(oneT ./ b1), storage.vectors) #scale!(storage.vectors, storage.values) mul!(vcm.Σ[1], transpose(Φinv), storage.vectors) copyto!(vcm.Σ[1], vcm.Σ[1] * transpose(vcm.Σ[1])) # update Σ2 - lmul!(Diagonal(b2), A2), rmul!(A2, b2) - storage = eigfact!(Symmetric(A2)) + lmul!(Diagonal(b2), A2), rmul!(A2, Diagonal(b2)) + storage = eigen!(Symmetric(A2)) @inbounds for i in 1:d storage.values[i] = storage.values[i] > zeroT ? √√storage.values[i] : zeroT end - scale!(storage.vectors, storage.values) - scale!(oneT ./ b2, storage.vectors) + rmul!(storage.vectors, Diagonal(storage.values)) #scale!(storage.vectors, storage.values) + lmul!(Diagonal(oneT ./ b2), storage.vectors) #scale!(storage.vectors, storage.values) mul!(vcm.Σ[2], transpose(Φinv), storage.vectors) copyto!(vcm.Σ[2], vcm.Σ[2] * transpose(vcm.Σ[2])) end From 6e1a2ac7d98adbc49cf3b38205c5a1c1e6ca67d7 Mon Sep 17 00:00:00 2001 From: Juhyun Kim Date: Wed, 9 Jan 2019 17:22:21 -0800 Subject: [PATCH 11/56] Create Project.toml describing metadata about the project --- Project.toml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 Project.toml diff --git a/Project.toml b/Project.toml new file mode 100644 index 0000000..7b471f2 --- /dev/null +++ b/Project.toml @@ -0,0 +1,19 @@ +name = "VarianceComponentModels" +uuid = "813005db-34b4-5f71-be9e-1bbf0a1d8f1c" +keywords = ["variance components model", "GWAS"] +author = ["Hua Zhou ", "OpenMendel Team"] +version = "0.1.0" + +[deps] +LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" +SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" +MathProgBase = "fdba3010-5040-5b88-9595-932c9decdf73" +Ipopt = "b6b21f68-93f8-5de0-b562-5493be1d77c9" +Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" +Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" + +[extras] +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[targets] +test = ["Test"] \ No newline at end of file From a8afd5be5766ecb3f936bd0f664af40419b8a327 Mon Sep 17 00:00:00 2001 From: Juhyun Kim Date: Wed, 9 Jan 2019 17:22:47 -0800 Subject: [PATCH 12/56] delete REQUIRE for updating to 0.7 --- REQUIRE | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 REQUIRE diff --git a/REQUIRE b/REQUIRE deleted file mode 100644 index d826093..0000000 --- a/REQUIRE +++ /dev/null @@ -1,5 +0,0 @@ -julia 0.5 - -Compat -MathProgBase -Ipopt From c81c85f3da45efb35bd8029cee1d71fb9ec48b57 Mon Sep 17 00:00:00 2001 From: Juhyun Kim Date: Wed, 9 Jan 2019 17:30:17 -0800 Subject: [PATCH 13/56] switch to html format --- docs/make.jl | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/docs/make.jl b/docs/make.jl index 2ab76df..1c78efe 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -1,10 +1,14 @@ using Documenter, VarianceComponentModels ENV["DOCUMENTER_DEBUG"] = "true" -makedocs() + +makedocs( + format = :html, + sitename = "VarianceComponentModels", + modules = [VarianceComponentModels] +) + deploydocs( - deps = Deps.pip("pygments", "mkdocs", "mkdocs-material", "python-markdown-math"), - repo = "github.com:OpenMendel/VarianceComponentModels.jl.git", - julia = "0.6", - osname = "linux" - ) + repo = "github.com/OpenMendel/VarianceComponentModels.jl.git", + target = "build" +) From 36f793885d70edf472df3f4c1ef9a81fd4c44b4d Mon Sep 17 00:00:00 2001 From: Juhyun Kim Date: Wed, 9 Jan 2019 17:32:32 -0800 Subject: [PATCH 14/56] remove mkdocs.yml --- docs/mkdocs.yml | 39 --------------------------------------- 1 file changed, 39 deletions(-) delete mode 100644 docs/mkdocs.yml diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml deleted file mode 100644 index 9755b32..0000000 --- a/docs/mkdocs.yml +++ /dev/null @@ -1,39 +0,0 @@ -site_name: VarianceComponentModels.jl -repo_url: https://github.com/OpenMendel/VarianceComponentModels.jl -site_description: Utilities for fitting and testing variance component models -site_author: Hua Zhou - -theme: material - -extra: - palette: - primary: 'indigo' - accent: 'blue' - -extra_css: - - assets/Documenter.css - -extra_javascript: - - https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML - - assets/mathjaxhelper.js - -markdown_extensions: - - codehilite - - extra - - tables - - fenced_code - - mdx_math - - admonition - -extra_javascript: - - https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML - - assets/mathjaxhelper.js - -docs_dir: 'build' - -pages: -- Home: index.md -- Manual: - - MLE/REML: man/mle_reml.md - - Heritability: man/heritability.md -- API: man/api.md From 6305f86cc6751f44c9ab8170345d9141dd212d19 Mon Sep 17 00:00:00 2001 From: Juhyun Kim Date: Wed, 9 Jan 2019 17:38:13 -0800 Subject: [PATCH 15/56] update travis to allow julia 0.7, 1.0 --- .travis.yml | 36 +++++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/.travis.yml b/.travis.yml index a5067ba..e2dd166 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,8 @@ os: - linux - osx julia: - - 0.6 + - 0.7 + - 1.0 - nightly matrix: allow_failures: @@ -12,16 +13,25 @@ matrix: addons: apt_packages: - gfortran -# uncomment the following lines to override the default test script -script: - - if [[ -a .git/shallow ]]; then git fetch --unshallow; fi - - julia --check-bounds=yes --depwarn=no -e 'Pkg.clone(pwd()); Pkg.build("VarianceComponentModels"); Pkg.test("VarianceComponentModels"; coverage=true)' -after_success: - - julia -e 'Pkg.add("Coverage"); cd(Pkg.dir("VarianceComponentModels")); using Coverage; Coveralls.submit(Coveralls.process_folder()); Codecov.submit(process_folder())' - - julia -e 'Pkg.add("Documenter"); cd(Pkg.dir("VarianceComponentModels")); include(joinpath("docs", "make.jl"))' notifications: - email: - on_success: change - on_failure: always - recipients: - - huazhou@ucla.edu + email: false +git: + depth: 99999999 + +before_script: # install dependent unregistered packages + - if [[ -a .git/shallow ]]; then git fetch --unshallow; fi + - julia -e 'using Pkg; Pkg.add([PackageSpec(url="https://github.com/OpenMendel/VarianceComponentModels.jl.git", rev="juliav0.7"), PackageSpec(url="https://github.com/OpenMendel/VarianceComponentModels.jl.git", rev="master")]);' +after_script: # TODO: change to after_success once https://github.com/JuliaLang/julia/issues/28306 is fixed + # push coverage results to Codecov and Coveralls + - julia -e 'using Pkg, VarianceComponentModels; cd(joinpath(dirname(pathof(VarianceComponentModels)), "..")); Pkg.add("Coverage"); using Coverage; Codecov.submit(process_folder()); Coveralls.submit(process_folder())' + +jobs: + include: + - stage: "Documentation" + julia: 1.0 + os: osx + script: + - julia -e 'using Pkg; Pkg.add("Documenter")' + - julia -e 'using VarianceComponentModels; include(joinpath(dirname(pathof(VarianceComponentModels)), "..", "docs", "make.jl"))' + after_script: skip + after_success: skip \ No newline at end of file From 29f7b1db4162035739937469e7b2462b8dab772a Mon Sep 17 00:00:00 2001 From: Juhyun Kim Date: Thu, 10 Jan 2019 09:27:29 -0800 Subject: [PATCH 16/56] currently fit_reml! returning 4 values; otherwise segmentation fault --- src/two_variance_component.jl | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/two_variance_component.jl b/src/two_variance_component.jl index 5f854bb..c251143 100644 --- a/src/two_variance_component.jl +++ b/src/two_variance_component.jl @@ -1316,7 +1316,7 @@ Find restricted MLE (REML) of variane component model. - `Bse`: standard errors of estimate `B` - `Bcov`: covariance of estimate `B` """ -function fit_reml!( +function fit_reml!( vcmodel::T1, vcdata::Union{T2, Array{T2}}; algo::Symbol = :FS, @@ -1378,8 +1378,9 @@ function fit_reml!( Bse = similar(vcmodel.B) copyto!(Bse, sqrt.(diag(Bcov))) - # output - logpdf(vcmodel, vcdatarot), vcmodel, Σse, Σcov, Bse, Bcov + ## output + # logpdf(vcmodel, vcdatarot), vcmodel, Σse, Σcov, Bse, Bcov + logpdf(vcmodel, vcdatarot), Σcov, Bse, Bcov end #---------------------------------------------------------------------------# From ac80972d6b3e6415dd31ea9d6f9151b17ee1cd6e Mon Sep 17 00:00:00 2001 From: Juhyun Kim Date: Thu, 10 Jan 2019 09:28:12 -0800 Subject: [PATCH 17/56] fit_reml! returns 4 values instead of 6 --- test/two_variance_component_test.jl | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/test/two_variance_component_test.jl b/test/two_variance_component_test.jl index 3ba742a..e06e1ac 100644 --- a/test/two_variance_component_test.jl +++ b/test/two_variance_component_test.jl @@ -162,12 +162,20 @@ logl_mle, _, _, Σcov_mle, Bse_mle, = fit_mle!(vcmmle, vcdata; algo = :MM) @info "test fit_reml (FS)" vcmreml = deepcopy(vcmodel) -logl_reml, _, _, Σcov_reml, Bse_reml, = fit_reml!(vcmreml, vcdata; algo = :FS) +println("vcmreml = $vcmreml") +logl_reml, Σcov_reml, Bse_reml, = fit_reml!(vcmreml, vcdata; algo = :FS) +# logl_reml, _, _, Σcov_reml, Bse_reml, = fit_reml!(vcmreml, vcdata; algo = :FS) @show vcmreml.B, Bse_reml, B @info "test fit_reml (MM)" vcmreml = deepcopy(vcmodel) -logl_reml, _, _, Σcov_reml, Bse_reml, = fit_reml!(vcmreml, vcdata; algo = :MM) +#logl_reml, _, _, Σcov_reml, Bse_reml, = fit_reml!(vcmreml, vcdata; algo = :MM) +logl_reml, _, Σcov_reml, Bse_reml, = fit_reml!(vcmreml, vcdata; algo = :MM) @show vcmreml.B, Bse_reml, B +## NOTE: +## fit_reml! currently returns logpdf(vcmodel, vcdatarot), Σcov, Bse, Bcov +## as opposed to logpdf(vcmodel, vcdatarot), vcmodel, Σse, Σcov, Bse, Bcov. +## returning more than 4 values causes segmentation fault for some reason. + end # module VarianceComponentTypeTest From 5860e53a3523d4a8070408983f2c41da9eea2a39 Mon Sep 17 00:00:00 2001 From: Juhyun Kim Date: Thu, 10 Jan 2019 09:35:16 -0800 Subject: [PATCH 18/56] fix deprecation warning --- src/VarianceComponentModels.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/VarianceComponentModels.jl b/src/VarianceComponentModels.jl index ea2453a..03782a4 100644 --- a/src/VarianceComponentModels.jl +++ b/src/VarianceComponentModels.jl @@ -234,7 +234,7 @@ function TwoVarCompVariateRotate(vcobs::VarianceComponentVariate{T, 2}) where T end # rotate responses Yrot = transpose(U) * vcobs.Y - Xrot = isempty(vcobs.X) ? Array{T}(size(Yrot, 1), 0) : transpose(U) * vcobs.X + Xrot = isempty(vcobs.X) ? Array{T}(undef, size(Yrot, 1), 0) : transpose(U) * vcobs.X # output TwoVarCompVariateRotate(Yrot, Xrot, deval, U, logdetV2) end From a3f1e5318de2c480d9920b9ed7b10b5030a71410 Mon Sep 17 00:00:00 2001 From: Juhyun Kim Date: Thu, 10 Jan 2019 09:35:32 -0800 Subject: [PATCH 19/56] remove println --- test/two_variance_component_test.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/test/two_variance_component_test.jl b/test/two_variance_component_test.jl index e06e1ac..dd73058 100644 --- a/test/two_variance_component_test.jl +++ b/test/two_variance_component_test.jl @@ -162,7 +162,6 @@ logl_mle, _, _, Σcov_mle, Bse_mle, = fit_mle!(vcmmle, vcdata; algo = :MM) @info "test fit_reml (FS)" vcmreml = deepcopy(vcmodel) -println("vcmreml = $vcmreml") logl_reml, Σcov_reml, Bse_reml, = fit_reml!(vcmreml, vcdata; algo = :FS) # logl_reml, _, _, Σcov_reml, Bse_reml, = fit_reml!(vcmreml, vcdata; algo = :FS) @show vcmreml.B, Bse_reml, B From d1bdb475680b8efde3e6ca92bfed3f907908bb32 Mon Sep 17 00:00:00 2001 From: Juhyun Kim Date: Thu, 10 Jan 2019 23:30:46 -0800 Subject: [PATCH 20/56] fix fit_reml! to return 6 values as before --- src/two_variance_component.jl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/two_variance_component.jl b/src/two_variance_component.jl index c251143..fb77866 100644 --- a/src/two_variance_component.jl +++ b/src/two_variance_component.jl @@ -1379,8 +1379,7 @@ function fit_reml!( copyto!(Bse, sqrt.(diag(Bcov))) ## output - # logpdf(vcmodel, vcdatarot), vcmodel, Σse, Σcov, Bse, Bcov - logpdf(vcmodel, vcdatarot), Σcov, Bse, Bcov + logpdf(vcmodel, vcdatarot), vcmodel, (Σse[1], Σse[2]), Σcov, Bse, Bcov end #---------------------------------------------------------------------------# From b0b0412dd0eefa0591fd1a9d3d0fb656f30131cb Mon Sep 17 00:00:00 2001 From: Juhyun Kim Date: Thu, 10 Jan 2019 23:31:56 -0800 Subject: [PATCH 21/56] fix fit_reml to return 6 values as before --- test/two_variance_component_test.jl | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/test/two_variance_component_test.jl b/test/two_variance_component_test.jl index dd73058..32a442d 100644 --- a/test/two_variance_component_test.jl +++ b/test/two_variance_component_test.jl @@ -46,7 +46,7 @@ vcmodelrot = TwoVarCompModelRotate(vcmodel) @inferred logpdf(vcmodelrot, vcdatarot) @test logpdf(vcmodel, vcdata) == logpdf(vcmodelrot, vcdatarot) @test (logpdf(vcmodelrot, [vcdatarot vcdatarot; vcdatarot vcdatarot]) - - logpdf(vcmodel, [vcdata vcdata; vcdata vcdata])) ≈ 0.0 +logpdf(vcmodel, [vcdata vcdata; vcdata vcdata])) ≈ 0.0 # @info "Evaluate gradient" # ∇ = zeros(2d^2) @@ -66,7 +66,7 @@ vcmodelrot = TwoVarCompModelRotate(vcmodel) @test norm(VarianceComponentModels.gradient(vcmodel, vcdata) - VarianceComponentModels.gradient(vcmodelrot, vcdatarot)) ≈ 0.0 @test norm(VarianceComponentModels.gradient(vcmodel, vcdata) - VarianceComponentModels.gradient(vcmodel, vcdatarot)) ≈ 0.0 @test norm(VarianceComponentModels.gradient(vcmodel, [vcdata vcdata]) - - 2.0VarianceComponentModels.gradient(vcmodel, vcdata)) ≈ 0.0 +2.0VarianceComponentModels.gradient(vcmodel, vcdata)) ≈ 0.0 @test norm(VarianceComponentModels.gradient(vcmodel, [vcdata vcdata]) - VarianceComponentModels.gradient(vcmodelrot, [vcdatarot vcdatarot])) ≈ 0.0 @@ -77,9 +77,9 @@ H = zeros(2d^2, 2d^2) @test norm(fisher_Σ(vcmodel, vcdata) - fisher_Σ(vcmodelrot, vcdatarot)) ≈ 0.0 @test norm(fisher_Σ(vcmodel, vcdata) - fisher_Σ(vcmodel, vcdatarot)) ≈ 0.0 @test norm(fisher_Σ(vcmodel, [vcdata vcdata]) - - 2fisher_Σ(vcmodel, vcdata)) ≈ 0.0 +2fisher_Σ(vcmodel, vcdata)) ≈ 0.0 @test norm(fisher_Σ(vcmodel, [vcdata vcdata]) - - fisher_Σ(vcmodelrot, [vcdatarot vcdatarot])) ≈ 0.0 +fisher_Σ(vcmodelrot, [vcdatarot vcdatarot])) ≈ 0.0 @info "Evaluate Fisher information matrix of B" @@ -89,9 +89,9 @@ H = zeros(p * d, p * d) @test norm(fisher_B(vcmodel, vcdata) - fisher_B(vcmodelrot, vcdatarot)) ≈ 0.0 @test norm(fisher_B(vcmodel, vcdata) - fisher_B(vcmodel, vcdatarot)) ≈ 0.0 @test norm(fisher_B(vcmodel, [vcdata vcdata]) - - 2.0fisher_B(vcmodel, vcdata)) ≈ 0.0 +2.0fisher_B(vcmodel, vcdata)) ≈ 0.0 @test norm(fisher_B(vcmodel, [vcdata vcdata]) - - fisher_B(vcmodelrot, [vcdatarot vcdatarot])) ≈ 0.0 +fisher_B(vcmodelrot, [vcdatarot vcdatarot])) ≈ 0.0 @info "Find MLE using Fisher scoring" vcmfs = deepcopy(vcmodel) @@ -162,14 +162,12 @@ logl_mle, _, _, Σcov_mle, Bse_mle, = fit_mle!(vcmmle, vcdata; algo = :MM) @info "test fit_reml (FS)" vcmreml = deepcopy(vcmodel) -logl_reml, Σcov_reml, Bse_reml, = fit_reml!(vcmreml, vcdata; algo = :FS) -# logl_reml, _, _, Σcov_reml, Bse_reml, = fit_reml!(vcmreml, vcdata; algo = :FS) +logl_reml, _, _, Σcov_reml, Bse_reml, = fit_reml!(vcmreml, vcdata; algo = :FS) @show vcmreml.B, Bse_reml, B @info "test fit_reml (MM)" vcmreml = deepcopy(vcmodel) -#logl_reml, _, _, Σcov_reml, Bse_reml, = fit_reml!(vcmreml, vcdata; algo = :MM) -logl_reml, _, Σcov_reml, Bse_reml, = fit_reml!(vcmreml, vcdata; algo = :MM) +logl_reml, _, _, Σcov_reml, Bse_reml, = fit_reml!(vcmreml, vcdata; algo = :MM) @show vcmreml.B, Bse_reml, B ## NOTE: From 437922102afce6a86adac323e4484813cbb57f1c Mon Sep 17 00:00:00 2001 From: Juhyun Kim Date: Fri, 11 Jan 2019 00:03:17 -0800 Subject: [PATCH 22/56] delete REQUIRE from test folder --- test/REQUIRE | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 test/REQUIRE diff --git a/test/REQUIRE b/test/REQUIRE deleted file mode 100644 index e69de29..0000000 From 5eab6660b80800e039eb7f7dbc29f4aa1b4b3d00 Mon Sep 17 00:00:00 2001 From: Juhyun Kim Date: Fri, 11 Jan 2019 00:07:25 -0800 Subject: [PATCH 23/56] use packages only at testing time but not enforce a dependency on them --- Project.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 7b471f2..454c967 100644 --- a/Project.toml +++ b/Project.toml @@ -14,6 +14,7 @@ Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" [extras] Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" [targets] -test = ["Test"] \ No newline at end of file +test = ["Test", "Random"] \ No newline at end of file From 659076654dc9e35b3444f1ea2ec792e5d7d470e4 Mon Sep 17 00:00:00 2001 From: Juhyun Kim Date: Fri, 11 Jan 2019 07:14:50 -0800 Subject: [PATCH 24/56] remove comments --- src/VarianceComponentModels.jl | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/VarianceComponentModels.jl b/src/VarianceComponentModels.jl index 03782a4..25e1e6c 100644 --- a/src/VarianceComponentModels.jl +++ b/src/VarianceComponentModels.jl @@ -1,8 +1,5 @@ module VarianceComponentModels -#using Compat -#import Compat.view - using LinearAlgebra, Statistics, MathProgBase, Ipopt#, KNITRO#, Mosek#, Gurobi import Base: eltype, length, size, + export VarianceComponentModel, VarianceComponentVariate, @@ -10,12 +7,6 @@ export VarianceComponentModel, VarianceComponentVariate, residual, residual!, nvarcomps, nmeanparams, nvarparams, nparams, mean!, mean, cov!, cov -# import Base: eltype, length, size, mean, mean!, cov, + -# export VarianceComponentModel, VarianceComponentVariate, -# TwoVarCompModelRotate, TwoVarCompVariateRotate, VarianceComponentAuxData, -# residual, residual!, nvarcomps, nmeanparams, nvarparams, nparams, -# mean!, mean, cov!, cov - """ `VarianceComponentModel` stores the model parameters of a variance component model. From 40e2db743f7fd70313dcecb5210938ad400fda55 Mon Sep 17 00:00:00 2001 From: Juhyun Kim Date: Fri, 11 Jan 2019 07:15:10 -0800 Subject: [PATCH 25/56] remove comments --- src/two_variance_component.jl | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/src/two_variance_component.jl b/src/two_variance_component.jl index fb77866..417fedc 100644 --- a/src/two_variance_component.jl +++ b/src/two_variance_component.jl @@ -395,8 +395,6 @@ function fisher_Σ( fisher_Σ(TwoVarCompModelRotate(vcm), vcobsrot) end - - function fisher_Σ!( H::AbstractMatrix, vcm::T1, @@ -469,16 +467,6 @@ function fisher_B( fisher_B!(H, vcmrot, vcobsrot) end -# function fisher_B!( -# H::AbstractMatrix{T}, -# vcm::Union{VarianceComponentModel{T, 2}, TwoVarCompVariateRotate{T}}, -# vcobs::Union{TwoVarCompVariateRotate{T}, VarianceComponentVariate{T, 2}}, -# vcaux::VarianceComponentAuxData = VarianceComponentAuxData(vcobs) -# ) where {T <: AbstractFloat} - -# fisher_B!(H, TwoVarCompModelRotate(vcm), TwoVarCompVariateRotate(vcobs), vcaux) -# end - function fisher_B!( H::AbstractMatrix{T}, vcm::VarianceComponentModel{T, 2}, @@ -571,7 +559,6 @@ function fisher_B( fisher_B!(H, vcm, vcobs, vcaux) end - function fisher_B!( H::AbstractMatrix, vcm::T1, From 6f5e0611efcabbd2aee05a442283db05f69c8884 Mon Sep 17 00:00:00 2001 From: Juhyun Kim Date: Fri, 11 Jan 2019 07:15:22 -0800 Subject: [PATCH 26/56] remove comments --- test/two_variance_component_test.jl | 40 ++++++++++------------------- 1 file changed, 13 insertions(+), 27 deletions(-) diff --git a/test/two_variance_component_test.jl b/test/two_variance_component_test.jl index 32a442d..1d9d803 100644 --- a/test/two_variance_component_test.jl +++ b/test/two_variance_component_test.jl @@ -46,29 +46,20 @@ vcmodelrot = TwoVarCompModelRotate(vcmodel) @inferred logpdf(vcmodelrot, vcdatarot) @test logpdf(vcmodel, vcdata) == logpdf(vcmodelrot, vcdatarot) @test (logpdf(vcmodelrot, [vcdatarot vcdatarot; vcdatarot vcdatarot]) - -logpdf(vcmodel, [vcdata vcdata; vcdata vcdata])) ≈ 0.0 - -# @info "Evaluate gradient" -# ∇ = zeros(2d^2) -# #@code_warntype gradient!(∇, vcmodelrot, vcdatarot) -# @inferred gradient!(∇, vcmodelrot, vcdatarot) -# @test norm(gradient(vcmodel, vcdata) - gradient(vcmodelrot, vcdatarot)) ≈ 0.0 -# @test norm(gradient(vcmodel, vcdata) - gradient(vcmodel, vcdatarot)) ≈ 0.0 -# @test norm(gradient(vcmodel, [vcdata vcdata]) - -# 2.0gradient(vcmodel, vcdata)) ≈ 0.0 -# @test norm(gradient(vcmodel, [vcdata vcdata]) - -# gradient(vcmodelrot, [vcdatarot vcdatarot])) ≈ 0.0 + logpdf(vcmodel, [vcdata vcdata; vcdata vcdata])) ≈ 0.0 @info "Evaluate gradient" ∇ = zeros(2d^2) #@code_warntype gradient!(∇, vcmodelrot, vcdatarot) @inferred VarianceComponentModels.gradient!(∇, vcmodelrot, vcdatarot) -@test norm(VarianceComponentModels.gradient(vcmodel, vcdata) - VarianceComponentModels.gradient(vcmodelrot, vcdatarot)) ≈ 0.0 -@test norm(VarianceComponentModels.gradient(vcmodel, vcdata) - VarianceComponentModels.gradient(vcmodel, vcdatarot)) ≈ 0.0 +@test norm(VarianceComponentModels.gradient(vcmodel, vcdata) - + VarianceComponentModels.gradient(vcmodelrot, vcdatarot)) ≈ 0.0 +@test norm(VarianceComponentModels.gradient(vcmodel, vcdata) - + VarianceComponentModels.gradient(vcmodel, vcdatarot)) ≈ 0.0 @test norm(VarianceComponentModels.gradient(vcmodel, [vcdata vcdata]) - -2.0VarianceComponentModels.gradient(vcmodel, vcdata)) ≈ 0.0 + 2.0VarianceComponentModels.gradient(vcmodel, vcdata)) ≈ 0.0 @test norm(VarianceComponentModels.gradient(vcmodel, [vcdata vcdata]) - -VarianceComponentModels.gradient(vcmodelrot, [vcdatarot vcdatarot])) ≈ 0.0 + VarianceComponentModels.gradient(vcmodelrot, [vcdatarot vcdatarot])) ≈ 0.0 @info "Evaluate Fisher information matrix of Σ" H = zeros(2d^2, 2d^2) @@ -77,10 +68,9 @@ H = zeros(2d^2, 2d^2) @test norm(fisher_Σ(vcmodel, vcdata) - fisher_Σ(vcmodelrot, vcdatarot)) ≈ 0.0 @test norm(fisher_Σ(vcmodel, vcdata) - fisher_Σ(vcmodel, vcdatarot)) ≈ 0.0 @test norm(fisher_Σ(vcmodel, [vcdata vcdata]) - -2fisher_Σ(vcmodel, vcdata)) ≈ 0.0 + 2fisher_Σ(vcmodel, vcdata)) ≈ 0.0 @test norm(fisher_Σ(vcmodel, [vcdata vcdata]) - -fisher_Σ(vcmodelrot, [vcdatarot vcdatarot])) ≈ 0.0 - + fisher_Σ(vcmodelrot, [vcdatarot vcdatarot])) ≈ 0.0 @info "Evaluate Fisher information matrix of B" H = zeros(p * d, p * d) @@ -89,9 +79,9 @@ H = zeros(p * d, p * d) @test norm(fisher_B(vcmodel, vcdata) - fisher_B(vcmodelrot, vcdatarot)) ≈ 0.0 @test norm(fisher_B(vcmodel, vcdata) - fisher_B(vcmodel, vcdatarot)) ≈ 0.0 @test norm(fisher_B(vcmodel, [vcdata vcdata]) - -2.0fisher_B(vcmodel, vcdata)) ≈ 0.0 + 2.0fisher_B(vcmodel, vcdata)) ≈ 0.0 @test norm(fisher_B(vcmodel, [vcdata vcdata]) - -fisher_B(vcmodelrot, [vcdatarot vcdatarot])) ≈ 0.0 + fisher_B(vcmodelrot, [vcdatarot vcdatarot])) ≈ 0.0 @info "Find MLE using Fisher scoring" vcmfs = deepcopy(vcmodel) @@ -123,7 +113,8 @@ vcmfs.sense = '=' vcmfs.b = 0.0 vcmfs.lb = 0.0 vcmfs.ub = 1.0 -logl_fs, _, _, Σcov_fs, Bse_fs, = mle_fs!(vcmfs, vcdatarot; solver = :Ipopt, qpsolver = :Ipopt) +logl_fs, _, _, Σcov_fs, Bse_fs, = mle_fs!(vcmfs, vcdatarot; + solver = :Ipopt, qpsolver = :Ipopt) @show vcmfs.B @test vcmfs.B[1] ≈ vcmfs.B[2] @test all(vcmfs.B .≥ 0.0) @@ -170,9 +161,4 @@ vcmreml = deepcopy(vcmodel) logl_reml, _, _, Σcov_reml, Bse_reml, = fit_reml!(vcmreml, vcdata; algo = :MM) @show vcmreml.B, Bse_reml, B -## NOTE: -## fit_reml! currently returns logpdf(vcmodel, vcdatarot), Σcov, Bse, Bcov -## as opposed to logpdf(vcmodel, vcdatarot), vcmodel, Σse, Σcov, Bse, Bcov. -## returning more than 4 values causes segmentation fault for some reason. - end # module VarianceComponentTypeTest From 7d92f862eb4f5972aa0665fd3342de4e64cf055c Mon Sep 17 00:00:00 2001 From: Juhyun Kim Date: Fri, 11 Jan 2019 07:16:48 -0800 Subject: [PATCH 27/56] update version number --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5d55cfd..eec6db5 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ Within Julia, use the package manager to install VarianceComponentModels: Pkg.clone("https://github.com/OpenMendel/VarianceComponentModels.jl.git") -This package supports Julia v0.5 and v0.6. +This package supports Julia v0.7 and v1.0. ## Citation From 7493c394f7f6e8e5504fed81edb496afe291d553 Mon Sep 17 00:00:00 2001 From: Juhyun Kim Date: Fri, 11 Jan 2019 07:42:43 -0800 Subject: [PATCH 28/56] add pages --- docs/make.jl | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/docs/make.jl b/docs/make.jl index 1c78efe..d02903c 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -4,11 +4,22 @@ ENV["DOCUMENTER_DEBUG"] = "true" makedocs( format = :html, - sitename = "VarianceComponentModels", - modules = [VarianceComponentModels] + sitename = "VarianceComponentModels.jl", + modules = [VarianceComponentModels], + pages = Any[ + "Home" => "index.md", + "Manual" => Any[ + "man/mle_reml.md", + "man/heritability.md" + ], + "API" => "man/api.md" + ] ) deploydocs( repo = "github.com/OpenMendel/VarianceComponentModels.jl.git", - target = "build" -) + target = "build", + osname = "linux", + julia = "1.0", + deps = nothing, + make = nothing) From dbcd11ad1f331315bd1ebc6418b502c406cc9ae4 Mon Sep 17 00:00:00 2001 From: Juhyun Kim Date: Wed, 16 Jan 2019 09:19:52 -0800 Subject: [PATCH 29/56] comment out before_script --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index e2dd166..9c4be02 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,9 +18,9 @@ notifications: git: depth: 99999999 -before_script: # install dependent unregistered packages - - if [[ -a .git/shallow ]]; then git fetch --unshallow; fi - - julia -e 'using Pkg; Pkg.add([PackageSpec(url="https://github.com/OpenMendel/VarianceComponentModels.jl.git", rev="juliav0.7"), PackageSpec(url="https://github.com/OpenMendel/VarianceComponentModels.jl.git", rev="master")]);' +# before_script: # install dependent unregistered packages +# - if [[ -a .git/shallow ]]; then git fetch --unshallow; fi +# - julia -e 'using Pkg; Pkg.add([PackageSpec(url="https://github.com/OpenMendel/VarianceComponentModels.jl.git", rev="juliav0.7"), PackageSpec(url="https://github.com/OpenMendel/VarianceComponentModels.jl.git", rev="master")]);' after_script: # TODO: change to after_success once https://github.com/JuliaLang/julia/issues/28306 is fixed # push coverage results to Codecov and Coveralls - julia -e 'using Pkg, VarianceComponentModels; cd(joinpath(dirname(pathof(VarianceComponentModels)), "..")); Pkg.add("Coverage"); using Coverage; Codecov.submit(process_folder()); Coveralls.submit(process_folder())' From 637bb16e14049f377225906246b31b276d4cc701 Mon Sep 17 00:00:00 2001 From: Juhyun Kim Date: Fri, 18 Jan 2019 11:02:56 -0800 Subject: [PATCH 30/56] add more tests for fisher_B! --- test/two_variance_component_test.jl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/two_variance_component_test.jl b/test/two_variance_component_test.jl index 1d9d803..01b6e79 100644 --- a/test/two_variance_component_test.jl +++ b/test/two_variance_component_test.jl @@ -75,7 +75,11 @@ H = zeros(2d^2, 2d^2) @info "Evaluate Fisher information matrix of B" H = zeros(p * d, p * d) #@code_warntype fisher_B!(H, vcmodelrot, vcdatarot) + @inferred fisher_B!(H, vcmodelrot, vcdatarot) +@inferred fisher_B!(H, vcdatarot, vcdata) +@inferred fisher_B!(H, vcmodel, vcdatarot) +@inferred fisher_B!(H, vcdatarot, vcdatarot) @test norm(fisher_B(vcmodel, vcdata) - fisher_B(vcmodelrot, vcdatarot)) ≈ 0.0 @test norm(fisher_B(vcmodel, vcdata) - fisher_B(vcmodel, vcdatarot)) ≈ 0.0 @test norm(fisher_B(vcmodel, [vcdata vcdata]) - From 56844f3ec5505f57196d264bad19c35d1fc28203 Mon Sep 17 00:00:00 2001 From: Juhyun Kim Date: Fri, 18 Jan 2019 11:13:05 -0800 Subject: [PATCH 31/56] minor edits on fisher_B testing --- test/two_variance_component_test.jl | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/two_variance_component_test.jl b/test/two_variance_component_test.jl index 01b6e79..9f13492 100644 --- a/test/two_variance_component_test.jl +++ b/test/two_variance_component_test.jl @@ -77,9 +77,7 @@ H = zeros(p * d, p * d) #@code_warntype fisher_B!(H, vcmodelrot, vcdatarot) @inferred fisher_B!(H, vcmodelrot, vcdatarot) -@inferred fisher_B!(H, vcdatarot, vcdata) @inferred fisher_B!(H, vcmodel, vcdatarot) -@inferred fisher_B!(H, vcdatarot, vcdatarot) @test norm(fisher_B(vcmodel, vcdata) - fisher_B(vcmodelrot, vcdatarot)) ≈ 0.0 @test norm(fisher_B(vcmodel, vcdata) - fisher_B(vcmodel, vcdatarot)) ≈ 0.0 @test norm(fisher_B(vcmodel, [vcdata vcdata]) - From fdd238c8eaa8a7f77ab2b34acfa34639b856a19d Mon Sep 17 00:00:00 2001 From: Juhyun Kim Date: Fri, 18 Jan 2019 11:22:22 -0800 Subject: [PATCH 32/56] update appveyor --- appveyor.yml | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 87981cb..c4a1d7a 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,9 +1,12 @@ environment: matrix: - - JULIAVERSION: "julialang/bin/winnt/x86/0.6/julia-0.6-latest-win32.exe" - - JULIAVERSION: "julialang/bin/winnt/x64/0.6/julia-0.6-latest-win64.exe" - - JULIAVERSION: "julianightlies/bin/winnt/x86/julia-latest-win32.exe" - - JULIAVERSION: "julianightlies/bin/winnt/x64/julia-latest-win64.exe" + - julia_version: 0.7 + - julia_version: 1 + - julia_version: nightly + +platform: + - x86 # 32-bit + - x64 # 64-bit branches: only: @@ -22,12 +25,7 @@ notifications: on_build_status_changed: false install: -# Download most recent Julia Windows binary - - ps: (new-object net.webclient).DownloadFile( - $("http://s3.amazonaws.com/"+$env:JULIAVERSION), - "C:\projects\julia-binary.exe") -# Run installer silently, output to C:\projects\julia - - C:\projects\julia-binary.exe /S /D=C:\projects\julia + - ps: iex ((new-object net.webclient).DownloadString("https://raw.githubusercontent.com/JuliaCI/Appveyor.jl/version-1/bin/install.ps1")) build_script: # Need to convert from shallow to complete for Pkg.clone to work From d7d40d33ccc014911f0867ad0f6b8c5499f1baed Mon Sep 17 00:00:00 2001 From: Juhyun Kim Date: Fri, 18 Jan 2019 14:05:07 -0800 Subject: [PATCH 33/56] edit build_script --- appveyor.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index c4a1d7a..912db81 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -13,11 +13,6 @@ branches: - master - /release-.*/ -matrix: - allow_failures: - - JULIAVERSION: "julianightlies/bin/winnt/x86/julia-latest-win32.exe" - - JULIAVERSION: "julianightlies/bin/winnt/x64/julia-latest-win64.exe" - notifications: - provider: Email on_build_success: false @@ -28,10 +23,15 @@ install: - ps: iex ((new-object net.webclient).DownloadString("https://raw.githubusercontent.com/JuliaCI/Appveyor.jl/version-1/bin/install.ps1")) build_script: -# Need to convert from shallow to complete for Pkg.clone to work - - IF EXIST .git\shallow (git fetch --unshallow) - - C:\projects\julia\bin\julia -e "versioninfo(); - Pkg.clone(pwd(), \"VarianceComponentModels\"); Pkg.build(\"VarianceComponentModels\")" + - echo "%JL_BUILD_SCRIPT%" + - C:\julia\bin\julia -e "%JL_BUILD_SCRIPT%" test_script: - - C:\projects\julia\bin\julia --check-bounds=yes -e "Pkg.test(\"VarianceComponentModels\")" + - echo "%JL_TEST_SCRIPT%" + - C:\julia\bin\julia -e "%JL_TEST_SCRIPT%" + +# # Uncomment to support code coverage upload. Should only be enabled for packages +# # which would have coverage gaps without running on Windows +# on_success: +# - echo "%JL_CODECOV_SCRIPT%" +# - C:\julia\bin\julia -e "%JL_CODECOV_SCRIPT%" \ No newline at end of file From b8df3d6ea3d1ec6624b2bc89357b9e07e52216dd Mon Sep 17 00:00:00 2001 From: Juhyun Kim Date: Fri, 18 Jan 2019 15:31:11 -0800 Subject: [PATCH 34/56] add test functions for gradient! and fisher_B! --- test/two_variance_component_test.jl | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/two_variance_component_test.jl b/test/two_variance_component_test.jl index 9f13492..55e650e 100644 --- a/test/two_variance_component_test.jl +++ b/test/two_variance_component_test.jl @@ -52,6 +52,8 @@ vcmodelrot = TwoVarCompModelRotate(vcmodel) ∇ = zeros(2d^2) #@code_warntype gradient!(∇, vcmodelrot, vcdatarot) @inferred VarianceComponentModels.gradient!(∇, vcmodelrot, vcdatarot) +@inferred VarianceComponentModels.gradient!(∇, vcmodel, vcdatarot) +@inferred VarianceComponentModels.gradient!(∇, vcmodel, vcdata) @test norm(VarianceComponentModels.gradient(vcmodel, vcdata) - VarianceComponentModels.gradient(vcmodelrot, vcdatarot)) ≈ 0.0 @test norm(VarianceComponentModels.gradient(vcmodel, vcdata) - @@ -75,9 +77,17 @@ H = zeros(2d^2, 2d^2) @info "Evaluate Fisher information matrix of B" H = zeros(p * d, p * d) #@code_warntype fisher_B!(H, vcmodelrot, vcdatarot) +# @inferred VarianceComponentVariate(Y, X, V) +# vcdata = VarianceComponentVariate(Y, X, V) +# vcmodel = VarianceComponentModel(vcdata) + +# @info "Pre-compute eigen-decomposition and rotate data" +# vcdatarot = TwoVarCompVariateRotate(vcdata) +# vcmodelrot = TwoVarCompModelRotate(vcmodel) @inferred fisher_B!(H, vcmodelrot, vcdatarot) @inferred fisher_B!(H, vcmodel, vcdatarot) +@inferred fisher_B!(H, vcmodel, vcdata) @test norm(fisher_B(vcmodel, vcdata) - fisher_B(vcmodelrot, vcdatarot)) ≈ 0.0 @test norm(fisher_B(vcmodel, vcdata) - fisher_B(vcmodel, vcdatarot)) ≈ 0.0 @test norm(fisher_B(vcmodel, [vcdata vcdata]) - From a2325f91eb4302cc2a14af3809778b004205c9b8 Mon Sep 17 00:00:00 2001 From: Juhyun Kim Date: Fri, 18 Jan 2019 17:54:09 -0800 Subject: [PATCH 35/56] =?UTF-8?q?add=20extra=20testing=20functions=20for?= =?UTF-8?q?=20fisher=5F=CE=A3!?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/two_variance_component.jl | 2 +- test/two_variance_component_test.jl | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/two_variance_component.jl b/src/two_variance_component.jl index 417fedc..f6d12fd 100644 --- a/src/two_variance_component.jl +++ b/src/two_variance_component.jl @@ -258,7 +258,7 @@ function gradient!( ∇::AbstractVector, vcm::T1, vcdata::Array{T2}, - vcaux::Array{T3} = map(x -> VarianceComponentAuxData(x), vcdata) + vcaux::Array{T3} = VarianceComponentAuxData(vcdata)# map(x -> VarianceComponentAuxData(x), vcdata) ) where { T1 <: Union{VarianceComponentModel, TwoVarCompModelRotate}, T2 <: Union{VarianceComponentVariate, TwoVarCompVariateRotate}, diff --git a/test/two_variance_component_test.jl b/test/two_variance_component_test.jl index 55e650e..3c718f1 100644 --- a/test/two_variance_component_test.jl +++ b/test/two_variance_component_test.jl @@ -67,6 +67,7 @@ vcmodelrot = TwoVarCompModelRotate(vcmodel) H = zeros(2d^2, 2d^2) #@code_warntype fisher!(H, vcmodelrot, vcdatarot) @inferred fisher_Σ!(H, vcmodelrot, vcdatarot) +@inferred fisher_Σ!(H, vcmodel, vcdata) @test norm(fisher_Σ(vcmodel, vcdata) - fisher_Σ(vcmodelrot, vcdatarot)) ≈ 0.0 @test norm(fisher_Σ(vcmodel, vcdata) - fisher_Σ(vcmodel, vcdatarot)) ≈ 0.0 @test norm(fisher_Σ(vcmodel, [vcdata vcdata]) - @@ -88,6 +89,7 @@ H = zeros(p * d, p * d) @inferred fisher_B!(H, vcmodelrot, vcdatarot) @inferred fisher_B!(H, vcmodel, vcdatarot) @inferred fisher_B!(H, vcmodel, vcdata) + @test norm(fisher_B(vcmodel, vcdata) - fisher_B(vcmodelrot, vcdatarot)) ≈ 0.0 @test norm(fisher_B(vcmodel, vcdata) - fisher_B(vcmodel, vcdatarot)) ≈ 0.0 @test norm(fisher_B(vcmodel, [vcdata vcdata]) - From cc6e70d19112fec30ee74ca2d6442214971d916a Mon Sep 17 00:00:00 2001 From: Juhyun Kim Date: Fri, 18 Jan 2019 21:14:54 -0800 Subject: [PATCH 36/56] create extra constructors --- src/VarianceComponentModels.jl | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/VarianceComponentModels.jl b/src/VarianceComponentModels.jl index 25e1e6c..5930f07 100644 --- a/src/VarianceComponentModels.jl +++ b/src/VarianceComponentModels.jl @@ -94,6 +94,19 @@ function TwoVarCompModelRotate( TwoVarCompModelRotate{eltype(Brot), typeof(Brot)}(Brot, eigval, eigvec, logdetΣ2) end +""" + TwoVarCompModelRotate(twovarcomp) + +Constructor of [`TwoVarCompModelRotate`](@ref) instance from a +[`TwoVarCompModelRotate`](@ref) instance itself. +""" +function TwoVarCompModelRotate( + twovarcomp::TwoVarCompModelRotate{T}) where T <: AbstractFloat + + ## JK ### + twovarcomp +end + """ TwoVarCompModelRotate(vcmodel) @@ -197,6 +210,19 @@ function TwoVarCompVariateRotate( eigval, eigvec, logdetV2) end +""" + TwoVarCompVariateRotate(twovarcomp) + +Constructor of a [`TwoVarCompVariateRotate`](@ref) instance from a +[`TwoVarCompVariateRotate`](@ref) instance itself. +""" +function TwoVarCompVariateRotate( + twovarcomp::TwoVarCompVariateRotate{T}) where T <: AbstractFloat + + ## JK ### + twovarcomp +end + """ TwoVarCompVariateRotate(vcobs) From 56862c71797c50f2928cd66ed27167b6fdf6b209 Mon Sep 17 00:00:00 2001 From: Juhyun Kim Date: Fri, 18 Jan 2019 21:15:28 -0800 Subject: [PATCH 37/56] condense fisher_B! and fisher_B --- src/two_variance_component.jl | 77 ++--------------------------------- 1 file changed, 4 insertions(+), 73 deletions(-) diff --git a/src/two_variance_component.jl b/src/two_variance_component.jl index f6d12fd..485c6de 100644 --- a/src/two_variance_component.jl +++ b/src/two_variance_component.jl @@ -469,96 +469,27 @@ end function fisher_B!( H::AbstractMatrix{T}, - vcm::VarianceComponentModel{T, 2}, - vcobs::VarianceComponentVariate{T, 2}, + vcm::Union{VarianceComponentModel{T, 2}, TwoVarCompVariateRotate{T}}, + vcobs::Union{TwoVarCompVariateRotate{T}, VarianceComponentVariate{T, 2}}, vcaux::VarianceComponentAuxData = VarianceComponentAuxData(vcobs) ) where {T <: AbstractFloat} fisher_B!(H, TwoVarCompModelRotate(vcm), TwoVarCompVariateRotate(vcobs), vcaux) end -function fisher_B!( - H::AbstractMatrix{T}, - vcm::TwoVarCompVariateRotate{T}, - vcobs::VarianceComponentVariate{T, 2}, - vcaux::VarianceComponentAuxData = VarianceComponentAuxData(vcobs) - ) where {T <: AbstractFloat} - - fisher_B!(H, vcm, TwoVarCompVariateRotate(vcobs), vcaux) -end - -function fisher_B!( - H::AbstractMatrix{T}, - vcm::VarianceComponentModel{T, 2}, - vcobs::TwoVarCompVariateRotate{T}, - vcaux::VarianceComponentAuxData = VarianceComponentAuxData(vcobs) - ) where {T <: AbstractFloat} - - fisher_B!(H, TwoVarCompModelRotate(vcm), vcobs, vcaux) -end - -function fisher_B!( - H::AbstractMatrix{T}, - vcm::TwoVarCompVariateRotate{T}, - vcobs::TwoVarCompVariateRotate{T}, - vcaux::VarianceComponentAuxData = VarianceComponentAuxData(vcobs) - ) where {T <: AbstractFloat} - - fisher_B!(H, vcm, vcobs, vcaux) -end - function fisher_B( vcm::T1, vcobs::T2, vcaux::T3 = VarianceComponentAuxData(vcobs) ) where { - T1 <: VarianceComponentModel, - T2 <: VarianceComponentVariate, + T1 <: Union{VarianceComponentModel, TwoVarCompModelRotate}, + T2 <: Union{VarianceComponentVariate, TwoVarCompVariateRotate}, T3 <: VarianceComponentAuxData} H = zeros(eltype(vcm), nmeanparams(vcm), nmeanparams(vcm)) fisher_B!(H, TwoVarCompModelRotate(vcm), TwoVarCompVariateRotate(vcobs), vcaux) end -function fisher_B( - vcm::T1, - vcobs::T2, - vcaux::T3 = VarianceComponentAuxData(vcobs) - ) where { - T1 <: TwoVarCompModelRotate, - T2 <: VarianceComponentVariate, - T3 <: VarianceComponentAuxData} - - H = zeros(eltype(vcm), nmeanparams(vcm), nmeanparams(vcm)) - fisher_B!(H, vcm, TwoVarCompVariateRotate(vcobs), vcaux) -end - -function fisher_B( - vcm::T1, - vcobs::T2, - vcaux::T3 = VarianceComponentAuxData(vcobs) - ) where { - T1 <: VarianceComponentModel, - T2 <: TwoVarCompVariateRotate, - T3 <: VarianceComponentAuxData} - - H = zeros(eltype(vcm), nmeanparams(vcm), nmeanparams(vcm)) - fisher_B!(H, TwoVarCompModelRotate(vcm), vcobs, vcaux) -end - -function fisher_B( - vcm::T1, - vcobs::T2, - vcaux::T3 = VarianceComponentAuxData(vcobs) - ) where { - T1 <: TwoVarCompModelRotate, - T2 <: TwoVarCompVariateRotate, - T3 <: VarianceComponentAuxData} - - H = zeros(eltype(vcm), nmeanparams(vcm), nmeanparams(vcm)) - fisher_B!(H, vcm, vcobs, vcaux) -end - function fisher_B!( H::AbstractMatrix, vcm::T1, From 61470b8d0f86c52968a8a247c5eae5374264b614 Mon Sep 17 00:00:00 2001 From: Juhyun Kim Date: Fri, 18 Jan 2019 21:15:45 -0800 Subject: [PATCH 38/56] delete comments --- test/two_variance_component_test.jl | 9 --------- 1 file changed, 9 deletions(-) diff --git a/test/two_variance_component_test.jl b/test/two_variance_component_test.jl index 3c718f1..5225c56 100644 --- a/test/two_variance_component_test.jl +++ b/test/two_variance_component_test.jl @@ -78,18 +78,9 @@ H = zeros(2d^2, 2d^2) @info "Evaluate Fisher information matrix of B" H = zeros(p * d, p * d) #@code_warntype fisher_B!(H, vcmodelrot, vcdatarot) -# @inferred VarianceComponentVariate(Y, X, V) -# vcdata = VarianceComponentVariate(Y, X, V) -# vcmodel = VarianceComponentModel(vcdata) - -# @info "Pre-compute eigen-decomposition and rotate data" -# vcdatarot = TwoVarCompVariateRotate(vcdata) -# vcmodelrot = TwoVarCompModelRotate(vcmodel) - @inferred fisher_B!(H, vcmodelrot, vcdatarot) @inferred fisher_B!(H, vcmodel, vcdatarot) @inferred fisher_B!(H, vcmodel, vcdata) - @test norm(fisher_B(vcmodel, vcdata) - fisher_B(vcmodelrot, vcdatarot)) ≈ 0.0 @test norm(fisher_B(vcmodel, vcdata) - fisher_B(vcmodel, vcdatarot)) ≈ 0.0 @test norm(fisher_B(vcmodel, [vcdata vcdata]) - From 692607d51c545fae3a6409e2002a754f6307c461 Mon Sep 17 00:00:00 2001 From: Juhyun Kim Date: Sat, 19 Jan 2019 11:16:49 -0800 Subject: [PATCH 39/56] add testing for trilind --- test/multivariate_calculus_test.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/test/multivariate_calculus_test.jl b/test/multivariate_calculus_test.jl index 52e5b27..fc35abd 100644 --- a/test/multivariate_calculus_test.jl +++ b/test/multivariate_calculus_test.jl @@ -23,6 +23,7 @@ end n = 3 A = randn(n, n) @test norm(vech(A) - A[trilind(A)]) ≈ 0.0 + @test norm([2, 3, 6] - trilind(A, -1)) ≈ 0.0 end # test triuind From 8ca14132740fa7896b717714d62186c0588510e4 Mon Sep 17 00:00:00 2001 From: Juhyun Kim Date: Mon, 21 Jan 2019 20:55:41 -0800 Subject: [PATCH 40/56] update installation instruction --- README.md | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index eec6db5..4f89a7e 100644 --- a/README.md +++ b/README.md @@ -8,15 +8,20 @@ This [Julia](http://julialang.org/) package provides computational routines for ## Installation -Within Julia, use the package manager to install VarianceComponentModels: +This package requires Julia v0.7.0 or later, which can be obtained from +https://julialang.org/downloads/ or by building Julia from the sources in the +https://github.com/JuliaLang/julia repository. - Pkg.clone("https://github.com/OpenMendel/VarianceComponentModels.jl.git") - -This package supports Julia v0.7 and v1.0. +The package has not yet been registered and must be installed using the repository location. +Start julia and use the `]` key to switch to the package manager REPL +```julia +(v0.7) pkg> add https://github.com/OpenMendel/VarianceComponentModels.jl.git#julia-0.7 +``` +Use the backspace key to return to the Julia REPL. ## Citation -If you use this analysis package in your research, please cite the following reference in the resulting publications: +If you use [OpenMendel](https://openmendel.github.io) analysis packages in your research, please cite the following reference in the resulting publications: *Lange K, Papp JC, Sinsheimer JS, Sripracha R, Zhou H, Sobel EM (2013) Mendel: The Swiss army knife of genetic analysis programs. Bioinformatics 29:1568-1570.* From dbaec64b38c82d477feb9ccbd182f54a58d5b803 Mon Sep 17 00:00:00 2001 From: Juhyun Kim Date: Mon, 21 Jan 2019 21:32:37 -0800 Subject: [PATCH 41/56] minor edit --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4f89a7e..92968b8 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ https://github.com/JuliaLang/julia repository. The package has not yet been registered and must be installed using the repository location. Start julia and use the `]` key to switch to the package manager REPL ```julia -(v0.7) pkg> add https://github.com/OpenMendel/VarianceComponentModels.jl.git#julia-0.7 +(v0.7) pkg> add https://github.com/OpenMendel/VarianceComponentModels.jl.git#Julia-0.7 ``` Use the backspace key to return to the Julia REPL. From 8c724300c88757e1adfa5977498592cc5964efcc Mon Sep 17 00:00:00 2001 From: Juhyun Kim Date: Tue, 22 Jan 2019 09:22:34 -0800 Subject: [PATCH 42/56] update installation instruction --- docs/src/index.md | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/docs/src/index.md b/docs/src/index.md index 3739daa..3901bfe 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -28,11 +28,16 @@ and **parameters** are ## Installation -Use the Julia package manager to install VarianceComponentModels.jl. +This package requires Julia v0.7.0 or later, which can be obtained from +https://julialang.org/downloads/ or by building Julia from the sources in the +https://github.com/JuliaLang/julia repository. + +The package has not yet been registered and must be installed using the repository location. +Start julia and use the `]` key to switch to the package manager REPL ```julia -Pkg.clone("https://github.com/OpenMendel/VarianceComponentModels.jl.git") +(v0.7) pkg> add https://github.com/OpenMendel/VarianceComponentModels.jl.git#Julia-0.7 ``` -This package supports Julia `0.6`. +Use the backspace key to return to the Julia REPL. ## Manual Outline From dea0194aa4481450fd3092fb24e7312f099a3d8b Mon Sep 17 00:00:00 2001 From: Juhyun Kim Date: Tue, 22 Jan 2019 09:22:55 -0800 Subject: [PATCH 43/56] update to v0.7 --- docs/heritability_v0.7.ipynb | 709 ++++++++++++++++++ docs/mle_reml_v0.7.ipynb | 1312 ++++++++++++++++++++++++++++++++++ 2 files changed, 2021 insertions(+) create mode 100644 docs/heritability_v0.7.ipynb create mode 100644 docs/mle_reml_v0.7.ipynb diff --git a/docs/heritability_v0.7.ipynb b/docs/heritability_v0.7.ipynb new file mode 100644 index 0000000..4c8766a --- /dev/null +++ b/docs/heritability_v0.7.ipynb @@ -0,0 +1,709 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Heritability Analysis" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As an application of the variance component model, this note demonstrates the workflow for heritability analysis in genetics, using a sample data set `cg10k` with **6,670** individuals and **630,860** SNPs. Person IDs and phenotype names are masked for privacy. `cg10k.bed`, `cg10k.bim`, and `cg10k.fam` is a set of Plink files in binary format. `cg10k_traits.txt` contains 13 phenotypes of the 6,670 individuals." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "ls: cg10k.bed: No such file or directory\n", + "ls: cg10k.bim: No such file or directory\n", + "ls: cg10k.fam: No such file or directory\n", + "ls: cg10k_traits.txt: No such file or directory\n" + ] + } + ], + "source": [ + ";ls cg10k.bed cg10k.bim cg10k.fam cg10k_traits.txt" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Machine information:" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Julia Version 0.7.0\n", + "Commit a4cb80f3ed (2018-08-08 06:46 UTC)\n", + "Platform Info:\n", + " OS: macOS (x86_64-apple-darwin14.5.0)\n", + " CPU: Intel(R) Core(TM) i5-6267U CPU @ 2.90GHz\n", + " WORD_SIZE: 64\n", + " LIBM: libopenlibm\n", + " LLVM: libLLVM-6.0.0 (ORCJIT, skylake)\n" + ] + } + ], + "source": [ + "versioninfo()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Read in binary SNP data" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We will use the [`SnpArrays.jl`](https://github.com/OpenMendel/SnpArrays.jl) package to read in binary SNP data and compute the empirical kinship matrix. The package has not yet been registered and must be installed using the repository location. Start julia and use the `]` key to switch to the package manager REPL\n", + "```julia\n", + "(v0.7) pkg> add https://github.com/OpenMendel/SnpArrays.jl.git#juliav0.7\n", + "```\n", + "Use the backspace key to return to the Julia REPL." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "┌ Info: Precompiling SnpArrays [4e780e97-f5bf-4111-9dc4-b70aaf691b06]\n", + "└ @ Base loading.jl:1187\n" + ] + } + ], + "source": [ + "using SnpArrays" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "ename": "SystemError", + "evalue": "SystemError: opening file cg10k.fam: No such file or directory", + "output_type": "error", + "traceback": [ + "SystemError: opening file cg10k.fam: No such file or directory", + "", + "Stacktrace:", + " [1] #systemerror#39(::Nothing, ::Function, ::String, ::Bool) at ./error.jl:106", + " [2] systemerror at ./error.jl:106 [inlined]", + " [3] #open#297(::Nothing, ::Nothing, ::Nothing, ::Nothing, ::Nothing, ::Function, ::String) at ./iostream.jl:283", + " [4] open at ./iostream.jl:275 [inlined]", + " [5] #open#298(::Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}}, ::Function, ::getfield(Base, Symbol(\"##294#295\")){Char}, ::String) at ./iostream.jl:367", + " [6] open at ./iostream.jl:367 [inlined]", + " [7] #countlines#293 at ./io.jl:1041 [inlined]", + " [8] countlines at ./io.jl:1041 [inlined]", + " [9] #SnpArray#4 at /Users/juhyun-kim/.julia/packages/SnpArrays/pfwqg/src/snparray.jl:31 [inlined]", + " [10] SnpArray(::String) at /Users/juhyun-kim/.julia/packages/SnpArrays/pfwqg/src/snparray.jl:31", + " [11] top-level scope at util.jl:156", + " [12] top-level scope at In[5]:1" + ] + } + ], + "source": [ + "# read in genotype data from Plink binary file (~50 secs on my laptop)\n", + "@time cg10k = SnpArray(\"cg10k\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary statistics of SNP data" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "ename": "UndefVarError", + "evalue": "UndefVarError: cg10k not defined", + "output_type": "error", + "traceback": [ + "UndefVarError: cg10k not defined", + "", + "Stacktrace:", + " [1] top-level scope at In[6]:1" + ] + } + ], + "source": [ + "people, snps = size(cg10k)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# summary statistics (~50 secs on my laptop)\n", + "@time maf, _, missings_by_snp, = summarize(cg10k);" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# 5 number summary and average MAF (minor allele frequencies)\n", + "quantile(maf, [0.0 .25 .5 .75 1.0]), mean(maf)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# Pkg.add(\"Plots\")\n", + "# Pkg.add(\"PyPlot\")\n", + "using Plots\n", + "pyplot()\n", + "\n", + "histogram(maf, xlab = \"Minor Allele Frequency (MAF)\", label = \"MAF\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# proportion of missing genotypes\n", + "sum(missings_by_snp) / length(cg10k)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# proportion of rare SNPs with maf < 0.05\n", + "countnz(maf .< 0.05) / length(maf)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Empirical kinship matrix\n", + "\n", + "We estimate empirical kinship based on all SNPs by the genetic relation matrix (GRM). Missing genotypes are imputed on the fly by drawing according to the minor allele frequencies." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# GRM using SNPs with maf > 0.01 (default) (~10 mins on my laptop)\n", + "srand(123)\n", + "@time Φgrm = grm(cg10k; method = :GRM)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Phenotypes" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Read in the phenotype data and compute descriptive statistics." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# Pkg.add(\"DataFrames\")\n", + "using DataFrames\n", + "\n", + "cg10k_trait = readtable(\n", + " \"cg10k_traits.txt\"; \n", + " separator = ' ',\n", + " names = [:FID; :IID; :Trait1; :Trait2; :Trait3; :Trait4; :Trait5; :Trait6; \n", + " :Trait7; :Trait8; :Trait9; :Trait10; :Trait11; :Trait12; :Trait13], \n", + " eltypes = [String; String; Float64; Float64; Float64; Float64; Float64; \n", + " Float64; Float64; Float64; Float64; Float64; Float64; Float64; Float64]\n", + " )\n", + "# do not display FID and IID for privacy\n", + "cg10k_trait[:, 3:end]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "describe(cg10k_trait[:, 3:end])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "Y = convert(Matrix{Float64}, cg10k_trait[:, 3:15])\n", + "histogram(Y, layout = 13)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Pre-processing data for heritability analysis" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To prepare variance component model fitting, we form an instance of `VarianceComponentVariate`. The two variance components are $(2\\Phi, I)$." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "┌ Info: Recompiling stale cache file /Users/juhyun-kim/.julia/compiled/v0.7/VarianceComponentModels/kuJce.ji for VarianceComponentModels [813005db-34b4-5f71-be9e-1bbf0a1d8f1c]\n", + "└ @ Base loading.jl:1185\n" + ] + }, + { + "ename": "UndefVarError", + "evalue": "UndefVarError: Φgrm not defined", + "output_type": "error", + "traceback": [ + "UndefVarError: Φgrm not defined", + "", + "Stacktrace:", + " [1] top-level scope at In[7]:2" + ] + } + ], + "source": [ + "using VarianceComponentModels\n", + "\n", + "# form data as VarianceComponentVariate\n", + "cg10kdata = VarianceComponentVariate(Y, (2Φgrm, eye(size(Y, 1))))\n", + "fieldnames(cg10kdata)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "cg10kdata" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Before fitting the variance component model, we pre-compute the eigen-decomposition of $2\\Phi_{\\text{GRM}}$, the rotated responses, and the constant part in log-likelihood, and store them as a `TwoVarCompVariateRotate` instance, which is re-used in various variane component estimation procedures." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# pre-compute eigen-decomposition (~50 secs on my laptop)\n", + "@time cg10kdata_rotated = TwoVarCompVariateRotate(cg10kdata)\n", + "fieldnames(cg10kdata_rotated)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Save intermediate results\n", + "\n", + "We don't want to re-compute SnpArray and empirical kinship matrices again and again for heritibility analysis." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To load workspace" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "using SnpArrays, JLD, DataFrames, VarianceComponentModels, Plots\n", + "pyplot()\n", + "@load \"cg10k.jld\"\n", + "whos()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Heritability of single traits" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We use Fisher scoring algorithm to fit variance component model for each single trait." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# heritability from single trait analysis\n", + "hST = zeros(13)\n", + "# standard errors of estimated heritability\n", + "hST_se = zeros(13)\n", + "# additive genetic effects\n", + "σ2a = zeros(13)\n", + "# enviromental effects\n", + "σ2e = zeros(13)\n", + "\n", + "tic()\n", + "for trait in 1:13\n", + " println(names(cg10k_trait)[trait + 2])\n", + " # form data set for trait j\n", + " traitj_data = TwoVarCompVariateRotate(cg10kdata_rotated.Yrot[:, trait], cg10kdata_rotated.Xrot, \n", + " cg10kdata_rotated.eigval, cg10kdata_rotated.eigvec, cg10kdata_rotated.logdetV2)\n", + " # initialize model parameters\n", + " traitj_model = VarianceComponentModel(traitj_data)\n", + " # estimate variance components\n", + " _, _, _, Σcov, _, _ = mle_fs!(traitj_model, traitj_data; solver=:Ipopt, verbose=false)\n", + " σ2a[trait] = traitj_model.Σ[1][1]\n", + " σ2e[trait] = traitj_model.Σ[2][1]\n", + " @show σ2a[trait], σ2e[trait]\n", + " h, hse = heritability(traitj_model.Σ, Σcov)\n", + " hST[trait] = h[1]\n", + " hST_se[trait] = hse[1]\n", + "end\n", + "toc()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# heritability and standard errors\n", + "[hST'; hST_se']" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Pairwise traits\n", + "\n", + "Joint analysis of multiple traits is subject to intensive research recently. Following code snippet does joint analysis of all pairs of traits, a total of 78 bivariate variane component models." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# additive genetic effects (2x2 psd matrices) from bavariate trait analysis;\n", + "Σa = Array{Matrix{Float64}}(13, 13)\n", + "# environmental effects (2x2 psd matrices) from bavariate trait analysis;\n", + "Σe = Array{Matrix{Float64}}(13, 13)\n", + "\n", + "tic()\n", + "for i in 1:13\n", + " for j in (i+1):13\n", + " println(names(cg10k_trait)[i + 2], names(cg10k_trait)[j + 2])\n", + " # form data set for (trait1, trait2)\n", + " traitij_data = TwoVarCompVariateRotate(cg10kdata_rotated.Yrot[:, [i;j]], cg10kdata_rotated.Xrot, \n", + " cg10kdata_rotated.eigval, cg10kdata_rotated.eigvec, cg10kdata_rotated.logdetV2)\n", + " # initialize model parameters\n", + " traitij_model = VarianceComponentModel(traitij_data)\n", + " # estimate variance components\n", + " mle_fs!(traitij_model, traitij_data; solver=:Ipopt, verbose=false)\n", + " Σa[i, j] = traitij_model.Σ[1]\n", + " Σe[i, j] = traitij_model.Σ[2]\n", + " @show Σa[i, j], Σe[i, j]\n", + " end\n", + "end\n", + "toc()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 3-trait analysis\n", + "\n", + "Researchers want to jointly analyze traits 5-7. Our strategy is to try both Fisher scoring and MM algorithm with different starting point, and choose the best local optimum. We first form the data set and run Fisher scoring, which yields a final objective value -1.4700991+04." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "traitidx = 5:7\n", + "# form data set\n", + "trait57_data = TwoVarCompVariateRotate(cg10kdata_rotated.Yrot[:, traitidx], cg10kdata_rotated.Xrot, \n", + " cg10kdata_rotated.eigval, cg10kdata_rotated.eigvec, cg10kdata_rotated.logdetV2)\n", + "# initialize model parameters\n", + "trait57_model = VarianceComponentModel(trait57_data)\n", + "# estimate variance components\n", + "@time mle_fs!(trait57_model, trait57_data; solver=:Ipopt, verbose=true)\n", + "trait57_model" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We then run the MM algorithm, starting from the Fisher scoring answer. MM finds an improved solution with objective value 8.955397e+03." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "ename": "UndefVarError", + "evalue": "UndefVarError: trait57_model not defined", + "output_type": "error", + "traceback": [ + "UndefVarError: trait57_model not defined", + "", + "Stacktrace:", + " [1] top-level scope at util.jl:156", + " [2] top-level scope at In[10]:1" + ] + } + ], + "source": [ + "# trait59_model contains the fitted model by Fisher scoring now\n", + "@time mle_mm!(trait57_model, trait57_data; verbose=true)\n", + "trait57_model" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Do another run of MM algorithm from default starting point. It leads to a slightly better local optimum -1.470104e+04, slighly worse than the Fisher scoring result. Follow up anlaysis should use the Fisher scoring result." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# default starting point\n", + "trait57_model = VarianceComponentModel(trait57_data)\n", + "@time _, _, _, Σcov, = mle_mm!(trait57_model, trait57_data; verbose=true)\n", + "trait57_model" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Heritability from 3-variate estimate and their standard errors." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "h, hse = heritability(trait57_model.Σ, Σcov)\n", + "[h'; hse']" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 13-trait joint analysis\n", + "\n", + "In some situations, such as studying the genetic covariance, we need to jointly analyze 13 traits. We first try the **Fisher scoring algorithm**." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# initialize model parameters\n", + "traitall_model = VarianceComponentModel(cg10kdata_rotated)\n", + "# estimate variance components using Fisher scoring algorithm\n", + "@time mle_fs!(traitall_model, cg10kdata_rotated; solver=:Ipopt, verbose=true)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "From the output we can see the Fisher scoring algorithm ran into some numerical issues. Let's try the **MM algorithm**." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# reset model parameters\n", + "traitall_model = VarianceComponentModel(cg10kdata_rotated)\n", + "# estimate variance components using Fisher scoring algorithm\n", + "@time mle_mm!(traitall_model, cg10kdata_rotated; verbose=true)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "It converges after ~1000 iterations." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Save analysis results" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "#using JLD\n", + "#@save \"copd.jld\"\n", + "#whos()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Julia 0.7.0", + "language": "julia", + "name": "julia-0.7" + }, + "language_info": { + "file_extension": ".jl", + "mimetype": "application/julia", + "name": "julia", + "version": "0.7.0" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/mle_reml_v0.7.ipynb b/docs/mle_reml_v0.7.ipynb new file mode 100644 index 0000000..32b500f --- /dev/null +++ b/docs/mle_reml_v0.7.ipynb @@ -0,0 +1,1312 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# MLE and REML" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Machine information" + ] + }, + { + "cell_type": "code", + "execution_count": 54, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Julia Version 0.7.0\n", + "Commit a4cb80f3ed (2018-08-08 06:46 UTC)\n", + "Platform Info:\n", + " OS: macOS (x86_64-apple-darwin14.5.0)\n", + " CPU: Intel(R) Core(TM) i5-6267U CPU @ 2.90GHz\n", + " WORD_SIZE: 64\n", + " LIBM: libopenlibm\n", + " LLVM: libLLVM-6.0.0 (ORCJIT, skylake)\n" + ] + } + ], + "source": [ + "versioninfo()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Demo data" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "For demonstration, we generate a random data set." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "# generate data from a d-variate response variane component model\n", + "using Random, LinearAlgebra\n", + "Random.seed!(123)\n", + "n = 1000 # no. observations\n", + "d = 2 # dimension of responses\n", + "m = 2 # no. variance components\n", + "p = 2 # no. covariates\n", + "# n-by-p design matrix\n", + "X = randn(n, p)\n", + "# p-by-d mean component regression coefficient\n", + "B = ones(p, d) \n", + "# a tuple of m covariance matrices\n", + "V = ntuple(x -> zeros(n, n), m) \n", + "for i = 1:m-1\n", + " Vi = randn(n, 50)\n", + " copyto!(V[i], Vi * Vi')\n", + "end\n", + "copyto!(V[m], Matrix(I, n, n)) # last covarianec matrix is idendity\n", + "# a tuple of m d-by-d variance component parameters\n", + "Σ = ntuple(x -> zeros(d, d), m) \n", + "for i in 1:m\n", + " Σi = randn(d, d)\n", + " copyto!(Σ[i], Σi' * Σi)\n", + "end\n", + "# form overall nd-by-nd covariance matrix Ω\n", + "Ω = zeros(n * d, n * d)\n", + "for i = 1:m\n", + " Ω += kron(Σ[i], V[i])\n", + "end\n", + "Ωchol = cholesky(Ω)\n", + "# n-by-d responses\n", + "Y = X * B + reshape(Ωchol.L * randn(n*d), n, d);" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Maximum likelihood estimation (MLE)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To find the MLE of parameters $(B,\\Sigma_1,\\ldots,\\Sigma_m)$, we take 3 steps: \n", + "\n", + "**Step 1 (Construct data)**. Construct an instance of `VarianceComponentVariate`, which consists fields \n", + "\n", + "* `Y`: $n$-by-$d$ responses \n", + "* `X`: $n$-by-$p$ covariate matrix \n", + "* `V=(V[1],...,V[m])`: a tuple of $n$-by-$n$ covariance matrices. The last covariance matrix must be positive definite and usually is the identity matrix. " + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(:Y, :X, :V)" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "using VarianceComponentModels\n", + "vcdata = VarianceComponentVariate(Y, X, V)\n", + "fieldnames(typeof(vcdata))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In the absence of covariates $X$, we can simply initialize by `vcdata = VarianceComponentVariate(Y, V)`." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Step 2 (Construct a model)**. Construct an instance of `VarianceComponentModel`, which consists of fields \n", + "\n", + "* `B`: $n$-by-$p$ mean regression coefficients \n", + "* `Σ=(Σ[1],...,Σ[m])`: variane component parameters respectively. \n", + "\n", + "When constructed from a `VarianceComponentVariate` instance, the mean parameters $B$ are initialized to be zero and the tuple of variance component parameters $\\Sigma$ to be `(eye(d),...,eye(d))`." + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(:B, :Σ, :A, :sense, :b, :lb, :ub)" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "vcmodel = VarianceComponentModel(vcdata)\n", + "fieldnames(typeof(vcmodel))" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}([0.0 0.0; 0.0 0.0], ([1.0 0.0; 0.0 1.0], [1.0 0.0; 0.0 1.0]), Array{Float64}(0,4), Char[], Float64[], -Inf, Inf)" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "vcmodel" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The remaining fields `A`, `sense`, `b`, `lb`, `ub` specify (optional) constraints on the mean parameters `B`:\n", + "\n", + "$A * \\text{vec}(B) \\,\\, =(\\text{or } \\ge \\text{or } \\le) \\,\\, b$\n", + "\n", + "$lb \\le \\text{vec}(B) \\le ub$\n", + "\n", + "`A` is an constraint matrix with $pd$ columns, `sense` is a vector of charaters taking values `'<'`, `'='` or `'>'`, and `lb` and `ub` are the lower and upper bounds for `vec(B)`. By default, `A`, `sense`, `b` are empty, `lb` is `-Inf`, and `ub` is `Inf`. If any constraits are non-trivial, final estimates of `B` are enforced to satisfy them." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "When a better initial guess is available, we can initialize by calling `vcmodel=VarianceComponentModel(B0, Σ0)` directly." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Step 3 (Fit model)**. Call optmization routine `fit_mle!`. The keywork `algo` dictates the optimization algorithm: `:MM` (minorization-maximization algorithm) or `:FS` (Fisher scoring algorithm)." + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + " MM Algorithm\n", + " Iter Objective \n", + "-------- -------------\n", + " 0 -6.253551e+03\n", + "\n", + "******************************************************************************\n", + "This program contains Ipopt, a library for large-scale nonlinear optimization.\n", + " Ipopt is released as open source code under the Eclipse Public License (EPL).\n", + " For more information visit http://projects.coin-or.org/Ipopt\n", + "******************************************************************************\n", + "\n", + " 1 -3.881454e+03\n", + " 2 -3.853179e+03\n", + " 3 -3.846525e+03\n", + " 4 -3.844906e+03\n", + " 5 -3.844506e+03\n", + " 6 -3.844406e+03\n", + " 7 -3.844381e+03\n", + " 8 -3.844375e+03\n", + " 9 -3.844374e+03\n", + " 10 -3.844373e+03\n", + "\n", + " 4.546981 seconds (11.23 M allocations: 566.109 MiB, 5.23% gc time)\n" + ] + } + ], + "source": [ + "vcmodel_mle = deepcopy(vcmodel)\n", + "@time logl, vcmodel_mle, Σse, Σcov, Bse, Bcov = fit_mle!(vcmodel_mle, vcdata; algo = :MM);" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The output of `fit_mle!` contains \n", + "\n", + "* final log-likelihood " + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "-3844.3731814180883" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "logl" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "* fitted model" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(:B, :Σ, :A, :sense, :b, :lb, :ub)" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "fieldnames(typeof(vcmodel_mle))" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}([1.092 1.04727; 0.955346 1.01632], ([0.380637 -0.305465; -0.305465 4.51938], [1.84009 0.265569; 0.265569 2.17275]), Array{Float64}(0,4), Char[], Float64[], -Inf, Inf)" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "vcmodel_mle" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "* standard errors of the estimated varianec component parameters" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "([0.0765136 0.263047; 0.263047 0.904332], [0.0844292 0.0917441; 0.0917441 0.0996927])" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "Σse" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "* covariance matrix of the variance component parameters estimates" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "8×8 Array{Float64,2}:\n", + " 0.00585433 -0.00467019 -0.00467019 … -1.07903e-6 -1.557e-7 \n", + " -0.00467019 0.0691937 0.00372555 -1.557e-7 -1.27444e-6 \n", + " -0.00467019 0.00372555 0.0691937 -8.83212e-6 -1.27444e-6 \n", + " 0.00372555 -0.055198 -0.055198 -1.27444e-6 -1.04316e-5 \n", + " -7.4779e-6 -1.07903e-6 -1.07903e-6 0.00102878 0.000148477\n", + " -1.07903e-6 -8.83212e-6 -1.557e-7 … 0.000148477 0.00121477 \n", + " -1.07903e-6 -1.557e-7 -8.83212e-6 0.00841698 0.00121477 \n", + " -1.557e-7 -1.27444e-6 -1.27444e-6 0.00121477 0.00993864 " + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "Σcov" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "* standard errors of the estimated mean parameters" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "2×2 Array{Float64,2}:\n", + " 0.0425562 0.0483834\n", + " 0.0430596 0.0492809" + ] + }, + "execution_count": 25, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "Bse" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "* covariance matrix of the mean parameter estimates" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "4×4 Array{Float64,2}:\n", + " 0.00181103 -1.96485e-5 0.000243441 -4.38252e-6 \n", + " -1.96485e-5 0.00185413 -4.38252e-6 0.000246407\n", + " 0.000243441 -4.38252e-6 0.00234096 -5.73331e-6 \n", + " -4.38252e-6 0.000246407 -5.73331e-6 0.00242861 " + ] + }, + "execution_count": 26, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "Bcov" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Restricted maximum likelihood estimation (REML)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "[REML (restricted maximum likelihood estimation)](https://en.wikipedia.org/wiki/Restricted_maximum_likelihood) is a popular alternative to the MLE. To find the REML of a variane component model, we replace the above step 3 by " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Step 3**. Call optmization routine `fit_reml!`." + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + " MM Algorithm\n", + " Iter Objective \n", + "-------- -------------\n", + " 0 -4.215053e+03\n", + " 1 -3.925799e+03\n", + " 2 -3.865114e+03\n", + " 3 -3.851105e+03\n", + " 4 -3.847732e+03\n", + " 5 -3.846903e+03\n", + " 6 -3.846698e+03\n", + " 7 -3.846647e+03\n", + " 8 -3.846634e+03\n", + " 9 -3.846631e+03\n", + " 10 -3.846630e+03\n", + "\n", + " 0.443964 seconds (8.09 k allocations: 62.532 MiB, 2.39% gc time)\n" + ] + } + ], + "source": [ + "vcmodel_reml = deepcopy(vcmodel)\n", + "@time logl, vcmodel_reml, Σse, Σcov, Bse, Bcov = fit_reml!(vcmodel_reml, vcdata; algo = :MM);" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The output of `fit_reml!` contains" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "* the final log-likelihood at REML estimate" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "-3844.3777179025046" + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "logl" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "* REML estimates" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(:B, :Σ, :A, :sense, :b, :lb, :ub)" + ] + }, + "execution_count": 31, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "fieldnames(typeof(vcmodel_reml))" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}([1.092 1.04727; 0.955345 1.01632], ([0.380594 -0.305485; -0.305485 4.51994], [1.84285 0.261963; 0.261963 2.17842]), Array{Float64}(0,4), Char[], Float64[], -Inf, Inf)" + ] + }, + "execution_count": 32, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "vcmodel_reml" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "* standard errors of the estimated varianec component parameters" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "([0.0765055 0.26305; 0.26305 0.904446], [0.0845559 0.0919325; 0.0919325 0.0999526])" + ] + }, + "execution_count": 33, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "Σse" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "* covariance matrix of the variance component parameters estimates" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "8×8 Array{Float64,2}:\n", + " 0.0058531 -0.00467005 -0.00467005 … -1.06597e-6 -1.51499e-7 \n", + " -0.00467005 0.0691951 0.00372613 -1.51499e-7 -1.26041e-6 \n", + " -0.00467005 0.00372613 0.0691951 -8.86843e-6 -1.26041e-6 \n", + " 0.00372613 -0.0552092 -0.0552092 -1.26041e-6 -1.0486e-5 \n", + " -7.50035e-6 -1.06597e-6 -1.06597e-6 0.00101633 0.000144472\n", + " -1.06597e-6 -8.86843e-6 -1.51499e-7 … 0.000144472 0.0012014 \n", + " -1.06597e-6 -1.51499e-7 -8.86843e-6 0.00845158 0.0012014 \n", + " -1.51499e-7 -1.26041e-6 -1.26041e-6 0.0012014 0.00999052 " + ] + }, + "execution_count": 34, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "Σcov" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "* standard errors of the estimated mean parameters" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "2×2 Array{Float64,2}:\n", + " 0.0425881 0.0484485\n", + " 0.0430919 0.0493475" + ] + }, + "execution_count": 35, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "Bse" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "* covariance matrix of the mean parameter estimates" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "4×4 Array{Float64,2}:\n", + " 0.00181375 -1.96783e-5 0.000239868 -4.34611e-6 \n", + " -1.96783e-5 0.00185691 -4.34611e-6 0.000242745\n", + " 0.000239868 -4.34611e-6 0.00234726 -5.73082e-6 \n", + " -4.34611e-6 0.000242745 -5.73082e-6 0.00243518 " + ] + }, + "execution_count": 36, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "Bcov" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Optimization algorithms" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Finding the MLE or REML of variance component models is a non-trivial nonlinear optimization problem. The main complications are the non-convexity of objective function and the positive semi-definiteness constraint of variane component parameters $\\Sigma_1,\\ldots,\\Sigma_m$. In specific applications, users should try different algorithms with different starting points in order to find a better solution. Here are some tips for efficient computation. " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In general the optimization algorithm needs to invert the $nd$ by $nd$ overall covariance matrix $\\Omega = \\Sigma_1 \\otimes V_1 + \\cdots + \\Sigma_m \\otimes V_m$ in each iteration. Inverting a matrix is an expensive operation with $O(n^3 d^3)$ floating operations. When there are only **two** varianec components ($m=2$), this tedious task can be avoided by taking one (generalized) eigendecomposion of $(V_1, V_2)$ and rotating data $(Y, X)$ by the eigen-vectors. " + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(:Yrot, :Xrot, :eigval, :eigvec, :logdetV2)" + ] + }, + "execution_count": 38, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "vcdatarot = TwoVarCompVariateRotate(vcdata)\n", + "fieldnames(typeof(vcdatarot))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Two optimization algorithms are implemented: [Fisher scoring](https://books.google.com/books?id=QYqeYTftPNwC&lpg=PP1&pg=PA142#v=onepage&q&f=false) (`mle_fs!`) and the [minorization-maximization (MM) algorithm](http://arxiv.org/abs/1509.07426) (`mle_mm!`). Both take the rotated data as input. These two functions give finer control of the optimization algorithms. Generally speaking, MM algorithm is more stable while Fisher scoring (if it converges) yields more accurate answer." + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + " MM Algorithm\n", + " Iter Objective \n", + "-------- -------------\n", + " 0 -6.253551e+03\n", + " 1 -3.881454e+03\n", + " 2 -3.853179e+03\n", + " 3 -3.846525e+03\n", + " 4 -3.844906e+03\n", + " 5 -3.844506e+03\n", + " 6 -3.844406e+03\n", + " 7 -3.844381e+03\n", + " 8 -3.844375e+03\n", + " 9 -3.844374e+03\n", + " 10 -3.844373e+03\n", + "\n", + " 0.042187 seconds (21.56 k allocations: 1.366 MiB)\n" + ] + } + ], + "source": [ + "vcmodel_mm = deepcopy(vcmodel)\n", + "@time mle_mm!(vcmodel_mm, vcdatarot; maxiter=10000, funtol=1e-8, verbose = true);" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "2×2 Array{Float64,2}:\n", + " 1.092 1.04727\n", + " 0.955346 1.01632" + ] + }, + "execution_count": 40, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# MM estimates\n", + "vcmodel_mm.B" + ] + }, + { + "cell_type": "code", + "execution_count": 41, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "([0.380637 -0.305465; -0.305465 4.51938], [1.84009 0.265569; 0.265569 2.17275])" + ] + }, + "execution_count": 41, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# MM estimates\n", + "vcmodel_mm.Σ" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Fisher scoring (`mle_fs!`) uses either [Ipopt.jl](https://github.com/JuliaOpt/Ipopt.jl) (keyword `solver=:Ipopt`) or [KNITRO.jl](https://github.com/JuliaOpt/KNITRO.jl) (keyword `solver=:Knitro`) as the backend solver. Ipopt is open source and installation of [Ipopt.jl](https://github.com/JuliaOpt/Ipopt.jl) package alone is sufficient." + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "This is Ipopt version 3.12.10, running with linear solver mumps.\n", + "NOTE: Other linear solvers might be more efficient (see Ipopt documentation).\n", + "\n", + "Number of nonzeros in equality constraint Jacobian...: 0\n", + "Number of nonzeros in inequality constraint Jacobian.: 0\n", + "Number of nonzeros in Lagrangian Hessian.............: 21\n", + "\n", + "Total number of variables............................: 6\n", + " variables with only lower bounds: 0\n", + " variables with lower and upper bounds: 0\n", + " variables with only upper bounds: 0\n", + "Total number of equality constraints.................: 0\n", + "Total number of inequality constraints...............: 0\n", + " inequality constraints with only lower bounds: 0\n", + " inequality constraints with lower and upper bounds: 0\n", + " inequality constraints with only upper bounds: 0\n", + "\n", + "iter objective inf_pr inf_du lg(mu) ||d|| lg(rg) alpha_du alpha_pr ls\n", + " 0 4.2109423e+03 0.00e+00 1.00e+02 0.0 0.00e+00 - 0.00e+00 0.00e+00 0 \n", + " 5 3.8445586e+03 0.00e+00 7.87e-01 -11.0 4.94e-02 - 1.00e+00 1.00e+00f 1 MaxS\n", + " 10 3.8443870e+03 0.00e+00 2.25e-01 -11.0 1.38e-02 - 1.00e+00 1.00e+00f 1 MaxS\n", + " 15 3.8443742e+03 0.00e+00 6.23e-02 -11.0 3.78e-03 - 1.00e+00 1.00e+00f 1 MaxS\n", + " 20 3.8443733e+03 0.00e+00 1.70e-02 -11.0 1.03e-03 - 1.00e+00 1.00e+00f 1 MaxS\n", + " 25 3.8443732e+03 0.00e+00 4.61e-03 -11.0 2.79e-04 - 1.00e+00 1.00e+00f 1 MaxS\n", + " 30 3.8443732e+03 0.00e+00 1.25e-03 -11.0 7.56e-05 - 1.00e+00 1.00e+00f 1 MaxS\n", + " 35 3.8443732e+03 0.00e+00 3.39e-04 -11.0 2.05e-05 - 1.00e+00 1.00e+00f 1 MaxS\n", + " 40 3.8443732e+03 0.00e+00 9.19e-05 -11.0 5.55e-06 - 1.00e+00 1.00e+00f 1 MaxS\n", + " 45 3.8443732e+03 0.00e+00 2.49e-05 -11.0 1.51e-06 - 1.00e+00 1.00e+00f 1 MaxS\n", + "iter objective inf_pr inf_du lg(mu) ||d|| lg(rg) alpha_du alpha_pr ls\n", + " 50 3.8443732e+03 0.00e+00 6.76e-06 -11.0 4.08e-07 - 1.00e+00 1.00e+00f 1 MaxSA\n", + " 55 3.8443732e+03 0.00e+00 1.83e-06 -11.0 1.11e-07 - 1.00e+00 1.00e+00f 1 MaxSA\n", + " 60 3.8443732e+03 0.00e+00 4.97e-07 -11.0 3.00e-08 - 1.00e+00 1.00e+00f 1 MaxSA\n", + "\n", + "Number of Iterations....: 63\n", + "\n", + " (scaled) (unscaled)\n", + "Objective...............: 3.4496886481728779e+02 3.8443731733053728e+03\n", + "Dual infeasibility......: 2.2693631701157965e-07 2.5290047251948971e-06\n", + "Constraint violation....: 0.0000000000000000e+00 0.0000000000000000e+00\n", + "Complementarity.........: 0.0000000000000000e+00 0.0000000000000000e+00\n", + "Overall NLP error.......: 2.2693631701157965e-07 2.5290047251948971e-06\n", + "\n", + "\n", + "Number of objective function evaluations = 64\n", + "Number of objective gradient evaluations = 64\n", + "Number of equality constraint evaluations = 0\n", + "Number of inequality constraint evaluations = 0\n", + "Number of equality constraint Jacobian evaluations = 0\n", + "Number of inequality constraint Jacobian evaluations = 0\n", + "Number of Lagrangian Hessian evaluations = 63\n", + "Total CPU secs in IPOPT (w/o function evaluations) = 1.488\n", + "Total CPU secs in NLP function evaluations = 0.375\n", + "\n", + "EXIT: Solved To Acceptable Level.\n", + " 2.486697 seconds (4.13 M allocations: 201.947 MiB, 3.03% gc time)\n" + ] + } + ], + "source": [ + "# Fisher scoring using Ipopt\n", + "vcmodel_ipopt = deepcopy(vcmodel)\n", + "@time mle_fs!(vcmodel_ipopt, vcdatarot; solver=:Ipopt, maxiter=1000, verbose=true);" + ] + }, + { + "cell_type": "code", + "execution_count": 43, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "2×2 Array{Float64,2}:\n", + " 1.092 1.04727\n", + " 0.955346 1.01632" + ] + }, + "execution_count": 43, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Ipopt estimates\n", + "vcmodel_ipopt.B" + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "([0.380552 -0.305594; -0.305594 4.52106], [1.84008 0.265385; 0.265385 2.17287])" + ] + }, + "execution_count": 44, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Ipopt estimates\n", + "vcmodel_ipopt.Σ" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Knitro is a commercial software and users need to follow instructions at [KNITRO.jl](https://github.com/JuliaOpt/KNITRO.jl) for proper functioning. Following code invokes Knitro as the backend optimization solver.\n", + "```julia\n", + "using KNITRO\n", + "\n", + "# Fisher scoring using Knitro\n", + "vcmodel_knitro = deepcopy(vcmodel)\n", + "@time mle_fs!(vcmodel_knitro, vcdatarot; solver=:Knitro, maxiter=1000, verbose=true);\n", + "\n", + "# Knitro estimates\n", + "vcmodel_knitro.B\n", + "\n", + "# Knitro estimates\n", + "vcmodel_knitro.Σ\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Starting point" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Here are a few strategies for successful optimization. " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "* For $d>1$ (multivariate response), initialize $B, \\Sigma$ from univariate estimates. \n", + "* Use REML estimate as starting point for MLE. \n", + "* When there are only $m=2$ variance components, pre-compute `TwoVarCompVariateRotate` and use it for optimization." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Constrained estimation of `B`\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Many applications invoke constraints on the mean parameters `B`. For demonstration, we enforce `B[1,1]=B[1,2]` and all entries of `B` are within [0, 2]." + ] + }, + { + "cell_type": "code", + "execution_count": 45, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}([0.0 0.0; 0.0 0.0], ([1.0 0.0; 0.0 1.0], [1.0 0.0; 0.0 1.0]), [1.0 0.0 -1.0 0.0], '=', 0.0, 0.0, 2.0)" + ] + }, + "execution_count": 45, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# set up constraints on B\n", + "vcmodel_constr = deepcopy(vcmodel)\n", + "vcmodel_constr.A = [1.0 0.0 -1.0 0.0]\n", + "vcmodel_constr.sense = '='\n", + "vcmodel_constr.b = 0.0\n", + "vcmodel_constr.lb = 0.0\n", + "vcmodel_constr.ub = 2.0\n", + "vcmodel_constr" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We first try the MM algorithm." + ] + }, + { + "cell_type": "code", + "execution_count": 46, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + " MM Algorithm\n", + " Iter Objective \n", + "-------- -------------\n", + " 0 -6.253551e+03\n", + " 1 -3.881820e+03\n", + " 2 -3.853477e+03\n", + " 3 -3.846807e+03\n", + " 4 -3.845184e+03\n", + " 5 -3.844783e+03\n", + " 6 -3.844683e+03\n", + " 7 -3.844658e+03\n", + " 8 -3.844652e+03\n", + " 9 -3.844650e+03\n", + " 10 -3.844650e+03\n", + "\n", + " 0.170236 seconds (170.93 k allocations: 8.918 MiB)\n" + ] + } + ], + "source": [ + "# MM algorithm for constrained estimation of B\n", + "@time mle_mm!(vcmodel_constr, vcdatarot; maxiter=10000, funtol=1e-8, verbose = true);" + ] + }, + { + "cell_type": "code", + "execution_count": 48, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(:B, :Σ, :A, :sense, :b, :lb, :ub)" + ] + }, + "execution_count": 48, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "fieldnames(typeof(vcmodel_constr))" + ] + }, + { + "cell_type": "code", + "execution_count": 49, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "2×2 Array{Float64,2}:\n", + " 1.07177 1.07177\n", + " 0.955683 1.01591" + ] + }, + "execution_count": 49, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "vcmodel_constr.B" + ] + }, + { + "cell_type": "code", + "execution_count": 50, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "([0.380624 -0.305498; -0.305498 4.51948], [1.84051 0.265065; 0.265065 2.17336])" + ] + }, + "execution_count": 50, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "vcmodel_constr.Σ" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now let's try Fisher scoring." + ] + }, + { + "cell_type": "code", + "execution_count": 51, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "This is Ipopt version 3.12.10, running with linear solver mumps.\n", + "NOTE: Other linear solvers might be more efficient (see Ipopt documentation).\n", + "\n", + "Number of nonzeros in equality constraint Jacobian...: 0\n", + "Number of nonzeros in inequality constraint Jacobian.: 0\n", + "Number of nonzeros in Lagrangian Hessian.............: 21\n", + "\n", + "Total number of variables............................: 6\n", + " variables with only lower bounds: 0\n", + " variables with lower and upper bounds: 0\n", + " variables with only upper bounds: 0\n", + "Total number of equality constraints.................: 0\n", + "Total number of inequality constraints...............: 0\n", + " inequality constraints with only lower bounds: 0\n", + " inequality constraints with lower and upper bounds: 0\n", + " inequality constraints with only upper bounds: 0\n", + "\n", + "iter objective inf_pr inf_du lg(mu) ||d|| lg(rg) alpha_du alpha_pr ls\n", + " 0 4.2114270e+03 0.00e+00 1.00e+02 0.0 0.00e+00 - 0.00e+00 0.00e+00 0 \n", + " 5 3.8448353e+03 0.00e+00 7.87e-01 -11.0 4.94e-02 - 1.00e+00 1.00e+00f 1 MaxS\n", + " 10 3.8446636e+03 0.00e+00 2.25e-01 -11.0 1.38e-02 - 1.00e+00 1.00e+00f 1 MaxS\n", + " 15 3.8446509e+03 0.00e+00 6.23e-02 -11.0 3.78e-03 - 1.00e+00 1.00e+00f 1 MaxS\n", + " 20 3.8446499e+03 0.00e+00 1.70e-02 -11.0 1.03e-03 - 1.00e+00 1.00e+00f 1 MaxS\n", + " 25 3.8446498e+03 0.00e+00 4.61e-03 -11.0 2.79e-04 - 1.00e+00 1.00e+00f 1 MaxS\n", + " 30 3.8446498e+03 0.00e+00 1.25e-03 -11.0 7.56e-05 - 1.00e+00 1.00e+00f 1 MaxS\n", + " 35 3.8446498e+03 0.00e+00 3.39e-04 -11.0 2.05e-05 - 1.00e+00 1.00e+00f 1 MaxS\n", + " 40 3.8446498e+03 0.00e+00 9.19e-05 -11.0 5.56e-06 - 1.00e+00 1.00e+00f 1 MaxS\n", + " 45 3.8446498e+03 0.00e+00 2.49e-05 -11.0 1.51e-06 - 1.00e+00 1.00e+00f 1 MaxS\n", + "iter objective inf_pr inf_du lg(mu) ||d|| lg(rg) alpha_du alpha_pr ls\n", + " 50 3.8446498e+03 0.00e+00 6.76e-06 -11.0 4.08e-07 - 1.00e+00 1.00e+00f 1 MaxSA\n", + " 55 3.8446498e+03 0.00e+00 1.83e-06 -11.0 1.11e-07 - 1.00e+00 1.00e+00h 1 MaxSA\n", + " 60 3.8446498e+03 0.00e+00 4.97e-07 -11.0 3.00e-08 - 1.00e+00 1.00e+00f 1 MaxSA\n", + "\n", + "Number of Iterations....: 63\n", + "\n", + " (scaled) (unscaled)\n", + "Objective...............: 3.4484507551949679e+02 3.8446498170293380e+03\n", + "Dual infeasibility......: 2.2694405212011240e-07 2.5301808562731130e-06\n", + "Constraint violation....: 0.0000000000000000e+00 0.0000000000000000e+00\n", + "Complementarity.........: 0.0000000000000000e+00 0.0000000000000000e+00\n", + "Overall NLP error.......: 2.2694405212011240e-07 2.5301808562731130e-06\n", + "\n", + "\n", + "Number of objective function evaluations = 64\n", + "Number of objective gradient evaluations = 64\n", + "Number of equality constraint evaluations = 0\n", + "Number of inequality constraint evaluations = 0\n", + "Number of equality constraint Jacobian evaluations = 0\n", + "Number of inequality constraint Jacobian evaluations = 0\n", + "Number of Lagrangian Hessian evaluations = 63\n", + "Total CPU secs in IPOPT (w/o function evaluations) = 0.024\n", + "Total CPU secs in NLP function evaluations = 0.625\n", + "\n", + "EXIT: Solved To Acceptable Level.\n", + " 0.729483 seconds (102.83 k allocations: 8.150 MiB, 0.88% gc time)\n" + ] + } + ], + "source": [ + "# Fisher scoring using Ipopt for constrained estimation of B\n", + "vcmodel_constr = deepcopy(vcmodel)\n", + "vcmodel_constr.A = [1.0 0.0 -1.0 0.0]\n", + "vcmodel_constr.sense = '='\n", + "vcmodel_constr.b = 0.0\n", + "vcmodel_constr.lb = 0.0\n", + "vcmodel_constr.ub = 2.0\n", + "vcmodel_constr\n", + "@time mle_fs!(vcmodel_constr, vcdatarot; solver=:Ipopt, maxiter=1000, verbose=true);" + ] + }, + { + "cell_type": "code", + "execution_count": 52, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "2×2 Array{Float64,2}:\n", + " 1.07177 1.07177\n", + " 0.955683 1.01591" + ] + }, + "execution_count": 52, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "vcmodel_constr.B" + ] + }, + { + "cell_type": "code", + "execution_count": 53, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "([0.380539 -0.305626; -0.305626 4.52116], [1.8405 0.264881; 0.264881 2.17348])" + ] + }, + "execution_count": 53, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "vcmodel_constr.Σ" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Julia 0.7.0", + "language": "julia", + "name": "julia-0.7" + }, + "language_info": { + "file_extension": ".jl", + "mimetype": "application/julia", + "name": "julia", + "version": "0.7.0" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From cfb460d51f9befd5c5624c2e67c60476bc96e29a Mon Sep 17 00:00:00 2001 From: Juhyun Kim Date: Tue, 22 Jan 2019 21:24:42 -0800 Subject: [PATCH 44/56] partial update after obtaining data files --- docs/heritability_v0.7.ipynb | 446 +++++++++++++++++++++++++++-------- 1 file changed, 349 insertions(+), 97 deletions(-) diff --git a/docs/heritability_v0.7.ipynb b/docs/heritability_v0.7.ipynb index 4c8766a..02518ab 100644 --- a/docs/heritability_v0.7.ipynb +++ b/docs/heritability_v0.7.ipynb @@ -16,22 +16,32 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + ";ls cg10k.bed cg10k.bim cg10k.fam #cg10k_traits.txt" + ] + }, + { + "cell_type": "code", + "execution_count": 2, "metadata": {}, "outputs": [ { - "name": "stderr", + "name": "stdout", "output_type": "stream", "text": [ - "ls: cg10k.bed: No such file or directory\n", - "ls: cg10k.bim: No such file or directory\n", - "ls: cg10k.fam: No such file or directory\n", - "ls: cg10k_traits.txt: No such file or directory\n" + "cg10k.bed\n", + "cg10k.bim\n", + "cg10k.fam\n" ] } ], "source": [ - ";ls cg10k.bed cg10k.bim cg10k.fam cg10k_traits.txt" + ";ls cg10k.bed cg10k.bim cg10k.fam " ] }, { @@ -85,130 +95,302 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 5, "metadata": {}, + "outputs": [], + "source": [ + "using SnpArrays" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "scrolled": true + }, "outputs": [ { - "name": "stderr", + "name": "stdout", "output_type": "stream", "text": [ - "┌ Info: Precompiling SnpArrays [4e780e97-f5bf-4111-9dc4-b70aaf691b06]\n", - "└ @ Base loading.jl:1187\n" + " 0.178436 seconds (466.95 k allocations: 43.173 MiB, 5.68% gc time)\n" ] + }, + { + "data": { + "text/plain": [ + "6670×630860 SnpArray:\n", + " 0x02 0x02 0x03 0x03 0x02 0x03 … 0x02 0x03 0x03 0x03 0x03 0x03\n", + " 0x03 0x03 0x02 0x03 0x02 0x03 0x03 0x02 0x02 0x03 0x02 0x01\n", + " 0x03 0x03 0x02 0x03 0x02 0x03 0x03 0x03 0x03 0x03 0x03 0x03\n", + " 0x03 0x03 0x03 0x03 0x03 0x03 0x03 0x02 0x02 0x02 0x02 0x03\n", + " 0x03 0x03 0x03 0x03 0x03 0x03 0x03 0x03 0x03 0x03 0x03 0x02\n", + " 0x02 0x02 0x03 0x03 0x02 0x03 … 0x03 0x03 0x03 0x03 0x03 0x03\n", + " 0x00 0x00 0x03 0x03 0x00 0x03 0x03 0x02 0x03 0x03 0x03 0x03\n", + " 0x03 0x03 0x03 0x03 0x03 0x02 0x03 0x03 0x03 0x03 0x03 0x02\n", + " 0x03 0x03 0x02 0x03 0x02 0x03 0x02 0x02 0x03 0x03 0x03 0x03\n", + " 0x03 0x03 0x03 0x00 0x03 0x03 0x02 0x02 0x03 0x03 0x02 0x03\n", + " 0x03 0x03 0x02 0x03 0x02 0x02 … 0x02 0x03 0x03 0x03 0x03 0x03\n", + " 0x02 0x02 0x03 0x03 0x02 0x03 0x02 0x03 0x03 0x03 0x03 0x02\n", + " 0x03 0x03 0x03 0x00 0x03 0x03 0x03 0x03 0x03 0x03 0x03 0x02\n", + " ⋮ ⋮ ⋱ ⋮ \n", + " 0x02 0x02 0x03 0x03 0x02 0x03 0x03 0x02 0x03 0x03 0x02 0x02\n", + " 0x02 0x02 0x02 0x03 0x00 0x02 0x02 0x03 0x03 0x03 0x02 0x03\n", + " 0x03 0x03 0x02 0x02 0x02 0x03 … 0x03 0x02 0x03 0x03 0x02 0x03\n", + " 0x02 0x02 0x03 0x03 0x02 0x03 0x03 0x02 0x03 0x03 0x03 0x02\n", + " 0x03 0x03 0x03 0x02 0x03 0x02 0x03 0x03 0x03 0x03 0x02 0x03\n", + " 0x03 0x03 0x01 0x02 0x02 0x01 0x03 0x00 0x00 0x02 0x00 0x02\n", + " 0x03 0x03 0x03 0x02 0x03 0x03 0x03 0x03 0x03 0x03 0x03 0x02\n", + " 0x03 0x03 0x02 0x02 0x02 0x03 … 0x00 0x03 0x03 0x03 0x03 0x03\n", + " 0x03 0x03 0x03 0x02 0x02 0x03 0x02 0x02 0x02 0x02 0x02 0x03\n", + " 0x03 0x03 0x03 0x02 0x03 0x02 0x02 0x03 0x02 0x03 0x03 0x02\n", + " 0x02 0x02 0x03 0x03 0x02 0x03 0x03 0x03 0x01 0x03 0x03 0x03\n", + " 0x03 0x03 0x03 0x03 0x03 0x02 0x03 0x00 0x03 0x03 0x03 0x03" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ - "using SnpArrays" + "# read in genotype data from Plink binary file (~50 secs on my laptop)\n", + "@time cg10k = SnpArray(\"cg10k.bed\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary statistics of SNP data" ] }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 9, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "data": { + "text/plain": [ + "(6670, 630860)" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "people, snps = size(cg10k)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The positions of the missing data are evaluated by" + ] + }, + { + "cell_type": "code", + "execution_count": 10, "metadata": {}, "outputs": [ { - "ename": "SystemError", - "evalue": "SystemError: opening file cg10k.fam: No such file or directory", - "output_type": "error", - "traceback": [ - "SystemError: opening file cg10k.fam: No such file or directory", - "", - "Stacktrace:", - " [1] #systemerror#39(::Nothing, ::Function, ::String, ::Bool) at ./error.jl:106", - " [2] systemerror at ./error.jl:106 [inlined]", - " [3] #open#297(::Nothing, ::Nothing, ::Nothing, ::Nothing, ::Nothing, ::Function, ::String) at ./iostream.jl:283", - " [4] open at ./iostream.jl:275 [inlined]", - " [5] #open#298(::Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}}, ::Function, ::getfield(Base, Symbol(\"##294#295\")){Char}, ::String) at ./iostream.jl:367", - " [6] open at ./iostream.jl:367 [inlined]", - " [7] #countlines#293 at ./io.jl:1041 [inlined]", - " [8] countlines at ./io.jl:1041 [inlined]", - " [9] #SnpArray#4 at /Users/juhyun-kim/.julia/packages/SnpArrays/pfwqg/src/snparray.jl:31 [inlined]", - " [10] SnpArray(::String) at /Users/juhyun-kim/.julia/packages/SnpArrays/pfwqg/src/snparray.jl:31", - " [11] top-level scope at util.jl:156", - " [12] top-level scope at In[5]:1" - ] + "data": { + "text/plain": [ + "6670×630860 SparseArrays.SparseMatrixCSC{Bool,Int32} with 5524131 stored entries:\n", + " [5688 , 1] = true\n", + " [6445 , 1] = true\n", + " [109 , 3] = true\n", + " [189 , 3] = true\n", + " [216 , 3] = true\n", + " [254 , 3] = true\n", + " [331 , 3] = true\n", + " [522 , 3] = true\n", + " [525 , 3] = true\n", + " [597 , 3] = true\n", + " [619 , 3] = true\n", + " [672 , 3] = true\n", + " ⋮\n", + " [4929 , 630860] = true\n", + " [5594 , 630860] = true\n", + " [5650 , 630860] = true\n", + " [5780 , 630860] = true\n", + " [5854 , 630860] = true\n", + " [5867 , 630860] = true\n", + " [6084 , 630860] = true\n", + " [6175 , 630860] = true\n", + " [6178 , 630860] = true\n", + " [6239 , 630860] = true\n", + " [6478 , 630860] = true\n", + " [6511 , 630860] = true" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ - "# read in genotype data from Plink binary file (~50 secs on my laptop)\n", - "@time cg10k = SnpArray(\"cg10k\")" + "mp = missingpos(cg10k)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Summary statistics of SNP data" + "The number of missing data values in each column can be evaluated as" ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 11, "metadata": {}, "outputs": [ { - "ename": "UndefVarError", - "evalue": "UndefVarError: cg10k not defined", - "output_type": "error", - "traceback": [ - "UndefVarError: cg10k not defined", - "", - "Stacktrace:", - " [1] top-level scope at In[6]:1" - ] + "data": { + "text/plain": [ + "1×630860 Array{Int64,2}:\n", + " 2 0 132 77 0 27 2 2 6 27 2 … 6 4 5 11 0 0 4 29 0 5 43" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ - "people, snps = size(cg10k)" + "missings_by_snp = sum(mp, dims=1)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Minor allele frequencies (MAF) for each SNP." ] }, { "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true - }, - "outputs": [], + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "630860-element Array{Float64,1}:\n", + " 0.1699160167966407 \n", + " 0.17098950524737633 \n", + " 0.11402569593147749 \n", + " 0.2686940694676172 \n", + " 0.21926536731634183 \n", + " 0.23934969140448592 \n", + " 0.19061187762447507 \n", + " 0.20200959808038388 \n", + " 0.027160864345738278\n", + " 0.2997139846454915 \n", + " 0.24625074985003004 \n", + " 0.05555555555555558 \n", + " 0.3659067046647667 \n", + " ⋮ \n", + " 0.22547254725472543 \n", + " 0.4035864345738295 \n", + " 0.20799579957995795 \n", + " 0.44801200300075017 \n", + " 0.2954647845021775 \n", + " 0.14265367316341826 \n", + " 0.1709145427286357 \n", + " 0.2814281428142814 \n", + " 0.06113537117903933 \n", + " 0.052473763118440764\n", + " 0.13930982745686427 \n", + " 0.1324128564961521 " + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "# summary statistics (~50 secs on my laptop)\n", - "@time maf, _, missings_by_snp, = summarize(cg10k);" + "maf_cg10k = maf(cg10k)" ] }, { "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true - }, - "outputs": [], + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "([0.00841726 0.124063 … 0.364253 0.5], 0.24536516625042462)" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# 5 number summary and average MAF (minor allele frequencies)\n", - "quantile(maf, [0.0 .25 .5 .75 1.0]), mean(maf)" + "using Statistics\n", + "Statistics.quantile(maf_cg10k, [0.0 .25 .5 .75 1.0]), mean(maf_cg10k)" ] }, { "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true - }, - "outputs": [], + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "IOPub data rate exceeded.\n", + "The notebook server will temporarily stop sending output\n", + "to the client in order to avoid crashing it.\n", + "To change this limit, set the config variable\n", + "`--NotebookApp.iopub_data_rate_limit`.\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlgAAAH0CAIAAABuMsSDAAAABmJLR0QA/wD/AP+gvaeTAAAgAElEQVR4nO3deXxU1f3/8TOZLJCFQCJrILGAylbKWhfiQlG/CoQQl1YEEYstliJI+aloAatQVr8qYi11iaI2AomEqGAQgYCAFhQqioSoZY0hgewLmeXO/f0x7XzTmTtDlsm9Mzmv54M/Jvdzl8+cuTNv7tyZOyZVVQUAALIKMboBAACMRBACAKRGEAIApEYQAgCkRhACAKRGEAIApEYQAgCkRhACAKRGEAIApEYQAgCk1paD8PTp0+Xl5f5dp91u56J0QgiHw6EoitFdBASbzWZ0CwFBURSHw2F0F8ZTVdVutxvdRUAIolfLthyEf/zjHz/88EP/rrOiooK9XAhRX19fU1NjdBcB4cKFC0a3EBCqq6stFovRXRjPZrNVVlYa3UVAKCsrC5b/G7XlIAQA4JIIQgCA1AhCAIDUCEIAgNQIQgCA1AhCAIDUCEIAgNQIQgCA1AhCAIDUCEIAgNQIQgCA1EKNbgA6uXDhwtmzZz2n19fX3/PrGaeOHdFcasS1yQf3f9rKrQGAkQhCWdw0Nu3M+XJTiNltusNmrS45I161aizz3d76vEV6NAcAxiEIZXHq+/yahV+JmM7uheICsfhqIzoCgIDAOUIAgNQIQjRHfn7+FT8d2iG+q+e/2Mu6vvrqq0Y3CACNxVujaI59+/ZVJ15Xf8dyz1L4B3+qra3VvyUAaB6CEM1kCm8vOnTVmh6pfzMA0Gy8NQoAkBpHhAAQQJYtW7Zx40aju/ADRVHMZvfvazXGVVddtX79er/340PQB+Hhw4eTk5M5KdUqLladOF14eb/BnpWqilLbFbfo3xHQ5p05c+b222+/++67jW7EGAUFBUuXLtV5o8EdhOfOnXv66afr6uqMbqSNKj110RR+6p43NUqbFpiFqnc/gBx69uw5dOhQo7swRkiIASfsgjgI6+vrZ8yYsWbNmpycHKN7abNMYe1Ez59qFCI76t4LALSKIP6wzMMPPzx37tzExESjGwEABLEgPiLMzMx87bXXnLdNJtPFixfbtWtnbEsQQgiHsn//fm+Pxfjx43v27KlzRwDgQxAHYUVFhfOGyWRSVc5XBQr7yUObTqgflh7yLKlHPurduzdBCCCg+D8IFUUZOHBgfn6+Z2nUqFH79+933p4xY8batWubtJ7y8vKpU6fu27cvOTl53bp1nTp18m/n8A/Fpvx80sWbZ3tWYsvG6t8OAPjm5yBcvXp1RkbG8ePHPUuqqubn5589ezY2NlYIERrqa9Oa61mxYkVSUlJWVta8efNWrly5bNky15o1V6IoypEjRz766CO36X379m32mUWr1WqxWBwOR/MWb20nTpx46623NUsWq9YPLbVEMw7CVdVms1ksFj93YijnLmF0F8ZzDoIhH/kLKFartYW7RMC+vOhGVdVmDGBYWFizdz8/B+HgwYP79OmTkpLiWSouLrZarampqcePHx8zZswrr7ziPI1kt9tdoei6rbme7OzsnJyciIiIWbNmpaamuoLQG4vF8uGHHx44cMBt+rRp01JTU5t3B2tra81mc1hYWPMWb213TX3waMQVIk4r5q023dtxpwpRX1/fxr70WVdX18buUfPU1dUpisKLuNVqra2tjYiIaPYabDb3p+rzq198J0unz8aHCLHw0TkTJkzQZ3OaVFVtxnMqOjo6PDy8eVv0cxCOHj3aW+ncuXMjR4587rnnEhMT586dO2fOnHfffVcIMWPGjLFjx955550rVqywWCyLFi3ytp7CwsKkpCQhRFJSUlFR0SWbiYyMfPLJJ++7777m3x8PDocjNjY2YIPQbDaLa6eIK2/QqOWu8vPGTE1fwmSKiYmJi4vzcyeGslgsbeweNU9ISEhERET79u2NbsRgVqs1LCysJbuEZ4geOZp/KGKA+Nn4lrXWKOF5L509e1aHDfkQEhKi83NKvw/LDBkyZOfOnc7by5cvHzhwoPP2qlWrbrvttpycnNLS0k2bNvlYg6qqJpPJeUNRlNZuGAACRbcrRf9f6LAd81fZmtNNJtOkSZMyMjJcUyZPnpyRkeE6M2W32xMSEuLi4r799lvnC7VzKberrNnt9tZpvEX0e0P/0KFDrk/KhIeHu/7XExcXl5aWtmHDhmnTpvl+P6FHjx5nzpwRQhQWFiYkJLR2wwAAl7y8PNepO6vVmpeX17C6e/fuyy677MKFC0ePHm043f7fdOu2SfQ4IszLy7vppptqa2vvuuuuvLy8Pn36LF68eOLEic7q2rVrd+3aVVBQMHHixKioqLFjvX6wMCUlJT09fenSpenp6c0+yQcDqXbrjh07iouLNat33nlnZCQ/4QQEqOTk5O3bt48fP14IsXPnzlGjRmVmZrqqGzduvP/++0+cOJGZmTlo0CDj2mwOPYJw9OjRqqomJyc/9dRTKSkplZWVY8eOXbNmjbPaq1evzZs3R0ZGbt26VfPjpi6LFi2aPHlyr169hg0b9vbb2p+NRCCru1C4+sMDYZ8VepYsB7NvuukmghAIWHfcccd7773nDMLs7Oy0tDRXENpsts2bNx86dOi777773e9+96c//cn17mhQaJUgdPs+g/NPk8k0c+bMmTNnus08btw4543u3bt3797dx3o6duy4ZcsW/7cLvag2m+XWRy0DNX62Iip/t/79AGi8cePGzZs3z2azmc3m3NzclStXuko7d+4cPHhwQkJCt27dKisrjx496joobJiIr7766oMPPqh3340QxFeWQVtic4i0e7VPEneKifwwO0v/lgA0FBMTM3z48Ly8vMjIyEGDBjm/Ee60cePGTz75xJV5Dd8dDYrLfhGECAi22sovr5wiouM9CvXhbz6we7f28WKHDh2k/bUaQH/Od0ejo6PT0tJcE61Wa3Z29qlTp5wXKtm2bdsjjzwSXO+OEoQIGFfdJOI8LkNafd4qQifOWug5u/1idWJM2NHD7hdMANBKJkyYsGDBgsjIyMcee8w1cfv27YmJia7Ldd14442nT59u+O5o4CMIEdgcdmGzVMzZoVE6cdC+5RHdGwLkFRcXN2DAAIvF0qVLF9fEjRs3NrwKWLt27caMGRNcnx0lCAEAl+A61ffxxx+7TVy3bp3bzO+//77bUgGOIETQUmwXyirunvKAZvHWm5J/8+B0nTsCWkNcfFzEzrfCTn2uw7bs/zooxGAdNhRQCEIEreLvyqtrssJGaZS+36/u+pQgRNuw9E8Lhw7sp9fWbv/5z3+u17YCBUGIIGaKjFVH3a9VMAnLp7q3A7SKiIiIKVOmGN1FWyb7j4cBACRHEAIApEYQAgCkxjlCtE2VFeVffPGFZqlnz57dunXTuR8AAYsgRFtUdCxv/2c33zvDs2Ip/fFPjz3y+OOP698UgMBEEKItqikT/UdXPviOZyV005P6twMgkBGEABBA2rdvn5mZmZ+fb3Qjxrhw4YL+GyUIASCALFmy5NVXXzW6Cz+orq6OiooKCWnaRzL79u3b8Mql+iAIASCAtG/ffvbs2UZ34QclJSXx8fFms9noRi6Nr08AAKTGESHkotaU/u/Lm99Yv8mzZDaJtaufvf766/XvCoCBCMLg88HWbVN/rX056eqLFsHLuE/qj8fO97nl/LWTPUvRm+eXl5fr3xIAYxGEwSdzU3bFqN+JazReykOWaf0UA9zE9RQ/0bi+vjmqo/69ADAcQRicIjuKTgmek00m/VsBgODGh2UAAFLjiBD4N1Wx7d279+LFi5rVcePGRUdH69wSAB0QhMC/XSw+9eLmyvB9Jz1LlsMfHT1yuG/fvro3BaDVEYTAv6k2i+V/HrP8bLxnKfpkf/37AaAPzhECAKRGEAIApEYQAgCkRhACAKTGh2WAS6u3Wq/qP0DzcgWhoaH1F+v0bgiA/xCEwKUpdVXqoi9F597uBYddnX2ZER0B8BuCEGgcc5gIDXefqHBROyDoEYRAC6iq2q5Dv2HXaBaTenRZ8PijmqXOnTv369evNTsD0FgEIdACDrtaV3H8tmc1Sj98/t3W5QdOPuFZsVZeGHP1kPez1rd6ewAagSAEWqz31RoTy86I9rEVj+zSKB3caL/wQWs3BaCR+PoEAEBqBCEAQGoEIQBAagQhAEBqBCEAQGoEIQBAagQhAEBqBCEAQGp8oR7QnbXun18fHTP+Ds+KSYgZ90+6++679W8KkBZBCOjuzFcl9nZFiZM0Sgc39N+zZ9CgQZ6VdzI3P7tiqeZFvk0mcfiLg1y8FGgeghAwgKljdzFsokbhwPq/vvnOuvc/8azUlhY77lkjhk7wLMWsHKUoit+bBCRBEAKBpOq88ovZ1SkLPCuhCwc6wtqJdjGeJVMIJ/uB5uP5AwCQGkEIAJAab40CQU9V7Dt27CgoKNCspqWl6dwPEFwIQiDo1dXWPPFKdmhUrGep6kBOVVVViNZJxJCQkPbt27d+d0CgIwiBoOew1df98gXRY4BG7Zsu8V26eU5WVbVnr8QT3+W3enNAwCMIgTatrsL2Sr0weRwRnjtufeMuIxoCAg5BGLgOHjxos9k8p58/f1500L8dAGibCMIAlZOTM+k3D4dfluBZqjlTICb+Qv+WAKBNIggD1KlTpxxDJlT+8nnPknnxCP37AYC2iu8RAgCkxhEhIKX66vPlldGdLtMsjh83bv0763TuCDAKQQhIqfqCXTXZnvpao/R1bsm593VvCDAMQQjIyhQiorWOCLWu6w20YZwjBABIjSNCAH5QUlLyl7/+LTRU4yXFZBJz58xOSND4LhAQCAhCAH7wyPyFH51VReIQz1K73S9P+tUvCUIELIIQgJ8MHieuuddzcsRXWfr3AjQe5wgBAFLjiBBAYymKsmDRU4qieJaOHz8urh2ruZSl/uL4u+4NDw/3LJkc1hf/d1VYWJhnKTIy8sYbb2xhw0BjEIQA3FWWl+Xm5npOX79+/Vufn1AH3OJZCim64G1t1qrSc/e8IHr016ituPG+p9aYQsxukx2Wi+0rThWfPdmktoHmIQgB/Lcfv/3qWME9T2pc57b2hy/FqF+LsY97lkL+8XeHj3V26SMSBmlMt1ys+u17Iszj94FLT4W/eHPjWwZagiAE8N/qa0w9f1o580ON0vNjTbq3A7Q2ghBAILJZLenp6ZqlXr163XKLxtuzQPMQhAACT+W56tq62W/t8awoZYUDY6yXXaZ9ufCuXbv26NGjlZtDW0MQAgg8DkWERtROeVWjtOOlf25ZOvruBzwrlorzv516z+rn/rfV20PbQhACCCp1FaJH/8r/t0Oj9PHzDrVY94YQ9PhCPQBAagQhAEBqBCEAQGoEIQBAanxYBgD8pqSkZOzdU0qKCjWrv5123zN/WqRzS7gkghCA1KxWa1lZmWbpyy+/PHDggMmkcTmdmJiYefPmeU7/9ttvT5RdrPnt+xqr+zzjx+KSljWLVkEQAmj7qqurX3rpJc3Sxi3bvz3yT3OYxo9jXKyqEDfPFmHt3AuWmg6H/qYZhEKIkPD2oktfjYIpZNOHubv3jdComMSfFzx+9913+7wTTfDOuxs+/8c/NEtX9P7JnNkP+2tDbQNBCKDt+90j/+/dL86Yev7Us+Q48rX6+82iz7Vai8WI2x8VkR3dp1cWiUPvNrmJwqOVXQeX3/aYZyVi+7MlJX47WCwrK3vggQfsqU9r1C6cGrBvPUHohiAE0HacOnHi/fc13pY8deqUY+gdIlnjejQh+99WW78xJ1N0vLh8uMb0Dp39uyFzeDv7LY9oFL7fL3Yc9u+22gCCEEBbcerwRycO7/nR5lmpOX5I9LxD/46Cy8svv1xdXa1ZGjNmzIgRGm/qtg0EIYC2wlLt+FlK5a80rjUa+sxQ/dsJLm+88cbclX9zDLhVo5a/KywsjCAEAPyHw25xmMaM1zjELLtQYrEo+nfUcna7PbT3iLo7lnqWwrMe1b8fPRGEANBENWWWmsqdiZM0SuUfmO3H9OniySUrNr23yXO6othtNqs+PbQNBCEANJkpNEwdNlGjcOGEKNIpCP/ywvNVU14VUXHuhfJCcVzjY0Gt4c2MzKeX/FmzZFbtedtze/bsqU8nLUEQAoDBlPOnZs1aO2vWLM3qzp07R48erb1kryEitpv7xOh/+dhWRen5BQsXek4/fPiwomj/3LEPa197/eSw6ZpfPmn/UorVGhwHpgQhABjMUVEk7l0tRv/Os9TxxVv8uaWz3xSVVf35a7NG6ciZ0F6dtJdSHYcPH373XY2vTpaWlorBPxG9fuZZMoWGtahVHRGEACARU4cu6vg/ahRKvve2iHLqn+uPVeSc0PheSu3JU37szSgEIQAELmtdzW/mPBodHe1Zqq2r06kJW7392vuqtb6hb37iCp16aE0EIQAELkv5uR9GzNV871F8map7O20TQQgAga3nYHHVjRrTNX4VA81BEBqsrKxM84NVVVVVQuh2BUQAkBdBaKT8/Pxrf/E/FovFs2StqRCjpuneEQBIhyA00pkzZ9QuV1yc9ZFG7a+/0vqAMwDAz0KMbgAAACMRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqfGFer/57LPPcnNzNUsdOnSYN2+ezv0AABqDIPSb0bf8j+XGmSI03L1QX9Pxn68RhAAQmAhCv7rtUdG+g/vEih/FPzcY0Q0A4NI4RwgAkBpBCACQGkEIAJAaQQgAkBoflml9DsViV++aPM2zUnLuR81f5QUA6IYgbH3V5y/WVL4Xcb1Gqeojs71K94YAAP+HINSDKTRCvW6qRqGyWFSc1b0dAMD/4RwhAEBqBCEAQGoEIQBAagQhAEBqBCEAQGoEIQBAakEfhIcPH46KijK6CwBAsAruIDx37tzTTz9dV1dndCMAgGAVxEFYX18/Y8aMF1980ehGAABBLIiD8OGHH547d25iYqLRjQAAglgQB2FmZubo0aNNJpMQwmQy1dfXG90RACD4BPG1RisqKpw3TCaTqqrGNgMACFL+PyJUFKVfv36apfLy8pSUlLi4uAkTJpSXlzd1PU1aHACAxvDzEeHq1aszMjKOHz+uWV2xYkVSUlJWVta8efNWrly5bNmyJq3H2+LeDgdramrmzp27YMECt+lz5syZMmVKE+5VA2VlZRaLJSwsTKPm66jUe8mvh7J+PzD2tb7mbEu/A3dvQ+HrQfJ/d17X2Kz3MHQcvaZvqxWa8+cqfY638btygLyj5a0Nn7urr528tLQ0Ojq6hV01UmxsbERERPOW9XMQDh48uE+fPikpKZrV7OzsnJyciIiIWbNmpaamOpPMbreHhv67DddtzfVoLu5DVFTU/PnzU1NT3abHx8fHxsY27w6qqhobG6sdhCaT9+W8l3ws1HS+WmjeCptZ8+cyzeNtKEw+nrj+787rGk3Neah0HD0dt+WdP3vwOd7N3JX9mF6BMNzCexs+d1dfO3mnTp3i4+Nb2FUjhYQ0/w1OPwfh6NGjfVQLCwuTkpKEEElJSUVFRc6JM2bMGDt27J133rlixQqLxbJo0SJv69Fc3AeTydS1a9crrriiGXfEG/N/+HGdCBAB8mIUpBg9eAoJCQmKV0tdPyyjqqrzfxaqqiqK4py4atWq2267LScnp7S0dNOmTU1dHACAltD16xM9evQ4c+aMEKKwsDAhIcE5MS4uLi0tbcOGDdOmTfP9Dq/m4gAAtIQeQZiXl+e8kZKSkp6erqpqenq669Td2rVrd+3aVVBQsHTp0q1bt/pYj+biAAC0hB5B6Drht2jRoiNHjvTq1evo0aOuD3P26tVr8+bNSUlJW7dujYyM9LEezcUBAGiJVjlH6PZZW9efHTt23LJli9vM48aNc97o3r179+7dfaxHc3EAAFoiiC+xBgBAyxGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkFcRA6HI5p06aNGDGiX79+mZmZRrcDAAhKQRyE27dvt9lsX3zxRWZm5syZM41uBwAQlII4CH/2s58tX768vr7+zJkz3bt3N7odAEBQCjW6gebr1q2bEGL8+PFbtmz5+OOPjW4HABCUgviIUFEUh8ORk5OTlZX10EMPGd0OACAoBXEQvvzyy48//rjZbO7Tp09dXZ3R7QAAgpKfg3DUqFGm//A8SisvL09JSYmLi5swYUJ5ebnvVSmK0q9fPx+LT5069Ycffhg8ePCkSZNeeeUV/94RAIAk/HmOUFXV/Pz8s2fPxsbGCiFCQ91XvmLFiqSkpKysrHnz5q1cuXLZsmXeVrV69eqMjIzjx4/7XnzTpk0++rFYLJs3bz558qTb9BtuuGH48OFNumsutbW1ZrM5LCxMo6Y2a43NWyoQNKdzHe9t0zcV8A+Ffg02Y0ut0Jw/V6n6WlnAP/JGU30Nn/eSqtbV1dXU1LRCRxratWvnGTqN5M8jwuLiYqvVmpqa2r179ylTplRVVbnNkJ2dPWvWrIiIiFmzZrkyzG63u2Zw3R48ePDChQsbs7hvtbW1ZR4uXryotoJmjhoAtFGt8UrbGq/A/jwiPHfu3MiRI5977rnExMS5c+fOmTPn3XffbThDYWFhUlKSECIpKamoqMg5ccaMGWPHjr3zzjtXrFhhsVgWLVokhBg9erTn+jUX9yEiImLy5Mn33Xdfy++ai8ViiYmJ0T4iNDVrjc1bKhA0p3Md723TNxXwD4V+DTZjS63QnD9XafK1soB/5I1m8jV83ksmU1RUVExMTCt05Gf+DMIhQ4bs3LnTeXv58uUDBw50m0FVVeeAqqqqKIpz4qpVq2677bacnJzS0lLfx3maiwMA0BL+fGv00KFD+/fvd94ODw+PiIhwm6FHjx5nzpwRQhQWFiYkJDgnxsXFpaWlbdiwYdq0aZ6LXHJxAABawp9BWFtbm5aWduzYMavVunjx4okTJzqn5+XlOW+kpKSkp6erqpqenp6amuqcuHbt2l27dhUUFCxdunTr1q0+1q+5OAAALeHPIExOTn7qqadSUlISEhLKy8tXrFjhnO464bdo0aIjR4706tXr6NGjCxYscE7s1avX5s2bk5KStm7dGhkZ6WP9mosDANAS/jxHaDKZZs6c6Xn9a9fneTp27Lhlyxa36rhx45w3unfv7nbJULcPAmkuDgBASwTxlWUAAGg5ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACC1UKMbCDI5H21/4613TCaTZ8lms+nfDwCghQjCpln+7HPfX3W36HalRm3Xdt3bAQC0FEHYdH2vE72vNroJAIB/cI4QACC14A7C8vLylJSUuLi4CRMmlJeXG90OACD4BHcQrlixIikpqaioKDExceXKlUa3AwAIPsF9jjA7OzsnJyciImLWrFmpqanLli1rWFVVtaamprS01G2pyMjIiIiI5m5TFfXVos7L0efFSqEqHhOrhVC1F7HVC4eiXVJswm7RLqkOYa3TLKmqEPU13turEmaPR/xitcUOxI4AABDGSURBVBBCexFrneqtPbtVmLzcKYdDWC966UEVllpfo1cX5T6xrspHe8Lh8DJ6VtVu0x4ihyKs9V5Gz2d79VUaJWu9r/ZUL6NnswhFuz3hsPt83H2MnlZ7TnUVwvNzzpYaoXp5BO31vtqzeRs9r7ulEMLXs6auQoTVa8yvenlwm/WsUR2q9/Z8P6mrhFC12mvek9rq/cH19qwRPp81VSLM49XMx2uOtc7rwNqtakiI92dNk5/UVkUdft0NISFmz1KHqPY/HP9Wa23NFxLS/OM6k6p6PMbBIzo6+vz58+3bt7948WLXrl2rqqoaVtPS0rZt2xYeHu621GOPPTZ9+vTmbXHcxDuPfvONZsliqY+IaOc5XVVVm6U+vF17z5JDsSuKEhaukcqKzaqaTKGhYZ4lu91uMpnMZo3dy6Yo5hBTiEljh7BYLRHh4UK4vyCqqmq1XIxoF6nVnqIodq/tCVNomGZ7NpMpxEt7DrNJe3+1Wq1h4WEmj/aEqlq8tedQ7DZbuNaYK3abqgov7VlNphCz538InKNnMmm3Z7OFhYZqfm3GcrE2or1HfguhOhSb9/YcqhoW5r5nCiHsVoswm0O9tic0X1asNntYqNlLe3UR7TVGT3U4bDarZnsOu11RHdrt2SzCZA4N1WrPbg/xslta7fYws5f26i9GaD01VNVhs1i8P2scYR7Pa/HvZ02IZnu+njV2xWz28qzx/qS2Wuo1O2/uk9r7s8aumEO0d0uL1RoRHua3J7Xdqoqmt+f9SW2xWsPDwjQf96jI9keP/NNzekt07Nix2Uc4wX1EqKqqc5RVVVUU90Ox6Ojov/3tb/fdd58ft5j7webY2NgwrVdYqdTV1dlsttjYWKMbMV5RUVH37t2N7sJ4FRUVERER7dtrZINUrFZrdXV1fHy80Y0Yr6SkJD4+XjM+A01wnyPs0aPHmTNnhBCFhYUJCQlGtwMACD7BHYQpKSnp6emqqqanp6emphrdDgAg+AR3EC5atOjIkSO9evU6evToggULjG4HABB8gvscYceOHbds2WJ0FwCAIBbcR4T6e+GFF06ePGl0F8bbu3fvhg0bjO7CeIqizJ8/3+guAsLbb7998OBBo7swXkFBwcsvv2x0FwFhyZIlZWVlRnfRKARh0+Tm5hYXFxvdhfGOHz/+2WefGd2F8RwOR0ZGhtFdBIRPP/30+++/N7oL4/34448ff/yx0V0EhE2bNlVXVxvdRaMQhAAAqRGEAACpEYQAAKkRhAAAqQX31yd8s9lse/bssdvtflznhQsXtm7d+t133/lxncHoH//4x6lTp9544w2jGzGYoigOh4NxEEKcPHly7969mheWlMo333xTXFzMLiGEqKury8rK6ty5sz6bGzNmTGJiYvOWDe6LbvtWUFCwePFizSvwNltdXV27du1acpnztsFmsymK0q6dxvWIZVNTUxMdHW10F8arr68PDQ3179MtGCmKYrFYIiM1Lngtm9ra2sjISN3+b/Twww8PGzasecu25SAEAOCSZD+yAQBIjiAEAEiNIAQASI0gBABIjSD0pby8PCUlJS4ubsKECeXl5U2qtjGXvLOKovTr10//xvTkexBycnIGDRrUsWPHG264oaCgwJAOdeN7KHJzcwcMGNCxY8cBAwa07QtvNuZF4JtvvomKitK5Mf35HopRo0aZ/uOhhx4ypEMfCEJfVqxYkZSUVFRUlJiYuHLlyiZV2xjfd3b16tXXXXfd8ePHDelNNz4G4fTp01OmTHn11VeLioomTJjwwAMPGNWkPnwMhcPhmDx58po1a8rKyp555pm2PRSXfBGorKycNm1aXV2d/r3pzMdQqKqan59/9uzZ6urq6urqF154wagmvVLh3ZVXXnns2DFVVY8dO3bllVc2qdrG+L6zO3fu/OCDD9r87uRjEHbt2vXggw86b5eUlMTHxxvQn458DIXFYtmyZYvD4aiqqnr//fcHDBhgUI968P28cDgcEydOzMzMbPNPDdXnUBQVFUVHRw8fPjw6Ojo1NbW4uNigHr1q+w9PS0RFRdXV1amqWldXFxMT06RqG9OYO9vmn+2NGQS73f7QQw/NnDlT39b0dsmhcP7+jslk2rdvn+7d6cf3OCxbtmzevHmqBE8N1edQHD58ePTo0YcPHy4tLZ06deo999xjUI9e8daoL6qqOi+LoKqqoihNqrYxUt1Zby45CJ988snIkSNjY2NXr16te3e6uuRQREdH19TULFmyZM6cObp3px8f47Br167c3Nxly5YZ1JrefAzFkCFDdu7cOWTIkLi4uOXLl2/bts2gHr0iCH3p0aPHmTNnhBCFhYUJCQlNqrYxUt1Zb3wMgqqqTzzxxDPPPLN+/frly5e3+SuN+RiKkydPPvroo0KIqKio6dOnHzt2zJgWdeFjHHbs2LF79+7w8HBnPJhMpr179xrTpS58DMWhQ4f279/vvB0eHh4REWFAfz4RhL6kpKSkp6erqpqenp6amuqcmJeX56PaVvkeCkn4GIT9+/dnZ2e///77PXr0qKmpqampMbLR1udjKHr06PH666/v3r1bVdUNGzYMHTrUyEZbmY9xWLJkieudNyGEqqrJyckGttrafAxFbW1tWlrasWPHrFbr4sWLJ06caGSjmnR4+zV4lZeXjx07NiEhISUlpaKiwjnRNWia1bbK91Bo/tn2+BiEJUuWSPXM8r0/5OXlDRs2rFOnTtdee63zAxRtVWOeF5pT2h4fQ+FwOP7yl7/06dPnsssumzp1amVlpaGdauCi2wAAqfHWKABAagQhAEBqBCEAQGoEIQBAagQhAEBqBCEAQGoEIQBAagQhAEBqBCEAQGoEIQBAagQhAEBqBCEClMlkioiIqKiocJteUVERERHh/Gkb52z5+fmt3cygQYNMJtPrr7/u2aRr643spBkNm7xo0kqCi9VqHTlyZElJiWj0nuDi48HyNozFxcUjRoywWq2tdocQ0Nr4r6YhqFmt1k2bNv36179uODE7O1vnF6yjR48ePXo0KioqMzNz+vTpem7aZc2aNT179jRk04ZYs2bNzTff3KVLF+efjd8TfD9Y3oaxa9euv/jFL1566aU//OEP/rsTCB5G//wFoE0IMXTo0DFjxrhNv+WWW5w/cef8s7q6WlGUVu1k4cKFsbGxf/7zn81m84ULF9yadP3MUMPbPjRythYuEtRsNluvXr1OnDjh/LORe4JTIx8sTz/88ENiYqLdbvfPfUBQ4a1RBK577713586dRUVFriklJSU7duyYNGmSa0p0dHRISCvuxqqqbtiw4a677rr33nsVRcnOzm69bcFp27ZtPXv2vPzyy11TGrMniJY9WL179+7evfu2bdta3D6CD0GIwHXrrbd26tRpw4YNrilZWVkdOnS49dZbXVPcztIdPHjwV7/6VVxcXO/evRsu6HA4XnjhhYEDB0ZHRw8fPjwrK6vhGg4ePJiSknL11Vd79vDVV18VFBRMmjTp8ssvHzFiRGZmZmM6V1V17dq1/fv3j4yMHDp06Lp161StH/5s5Gy+ufXvY50HDhwYM2ZMp06devfu/Zvf/Obzzz/XPNWan5/f8MSbjxX6GHBVVV988cUBAwZ06NDh+uuv37t3rxBiwYIFP/nJT1yLnz59OiQkJCcnx+0ebdu27cYbb2w4pTF7gmjug+Vy4403fvzxx01aBG2EMQeiwKUIIY4dO/bb3/525MiRronJycnTp08/duyYa9cV//3m5NVXX7127dr9+/dPnTo1PDy8pqbGWVq1alV0dPTKlSu3bt06f/58s9n8wQcfuJYaMWLE9OnT//73v3u2MX/+/G7dujnfMVu+fLnZbD5//rxbk56333rrrQEDBqxbt+6jjz5ybu6vf/1rk2ZzG4otW7Yc+2/19fWa/Xtb5xdffGE2mydOnJiRkZGZmTlp0qT4+HjNYWw4vJe8L94GfPXq1bGxsS+++GJWVlZaWprZbP7qq6+++eYbIcTnn3/unGfp0qVdu3a1Wq1u93fYsGFZWVlug3zJPaExD5aPYVRVdcOGDcOHD/ccf7R5BCEClPPlLy8vTwhRUFCgqurp06eFEJ988omPIHz++eedt6uqqlwlh8MRHx/vPJRxmj9//vXXX+9a6tFHH9XsweFw9O7de/bs2c4/v//+eyHEK6+84tak5+1hw4Z9//33rtn+8Ic/JCcnN2k2t6Hw9PXXX2v2722dt99++9SpUxuu1vlZEs+u3NLF933xNuDdunXLyMhwlhRFuf32299++21VVQcPHjx37lznPP369dMc+c6dO+/bt6/h3W/MntCYB8vHMKqqumfPni5dunj2gzaPIESAcr78KYqSkJDw9NNPq6q6atWqrl272mw2H0H4xRdfuK1BVdXi4mIhRGlpqau0Z8+e+Ph412x79uzR7OHAgQOeL5233HKL5ybcbkdFRbkt1bVr1ybN5jkUPgaqYf/e1tmlS5ddu3Y1XHDnzp2NCULf90VzwM+fPy+EcPusitOyZcsSEhIURXGO7bfffus5T2ho6JEjR9xWe8k9oUkPlqavvvoqNDTUxwxoq/j6BAJaSEjIpEmTMjIyFi5cuH79+l/+8pehob52Ws9XbW+rVRTF9Wfnzp01Z9uwYUPnzp13797tOme2bt26VatWnT9/3tsiTpGRkZ9++mn79u19t9HI2S6pYTPe1mm1Wt2+cmc2mzXXVldX1/gmNQfcbrd7W/8999zzxBNP7Nu3LzMz89prr+3fv7/nPB07dqypqXGbeMk9odkPlkt1dXWnTp0aMyfaGD4sg0B37733Hj9+fOPGjV9++aXbpwQbqXPnzvHx8Vu3bnVN2bJly6BBg3wv5XA4Nm7cmJqa2r9//37/cf/99zfm44gDBw4sLCx0LnLVVVetWbNm3bp1zZ6tSbyt86c//embb77ZcM6MjIyGf7ry77PPPmthk127du3UqdOOHTucfzocjhEjRixdulQIcfnll1933XXvvPPOu+++6+1LmT179mz4AVEXH3tCSx4sl6KiIqm+rIn/Y/QhKaBNNDjhdNVVV3Xq1CkpKcnhcKj//d6d8PLmpNufK1eujImJefbZZzU/LKP5jtm+ffuEEFu2bGk40eFw9O3b9+abb/a99ZycnJiYmOeffz47O3vq1KlCCM3N+ZhNcyh8D5TvdW7fvl0Icccdd6xfvz4rK2vy5MmxsbGuYUxKShozZkxubu6bb77p9uW8Rt4Xtz+XLl0aFxf30ksv5ebm3n///eHh4a6zcWvWrAkJCYmMjKyqqtK8RzNnznzyySc9V+tjT2jqg6XpiSee+P3vf+9jBrRVBCECVMOXrWeeeUYI8fjjjzv/bEYQKory3HPPOb8DMGTIEM8PJXo2MHv27JiYmIafKnSaN29eSEhISUmJ762//vrr/fr1i4yMHD58+Hvvvedtc95m8zYUjal6W2dubu4111wTHR19+eWXT5ky5dNPP3UN444dO/r16xcdHT1mzJh//etfbv9FbuR9cRvw5cuX9+3bNzIycuTIkdu2bXPNdu7cuZCQkGnTpnm7R9nZ2Q0/NNSYPaGpD5am5OTkzZs3+5gBbZVJbfr3lgC0Afn5+f3799f/FeDEiRO9e/fes2fP9ddfrzmDzWZLSko6cOCAnm9Unj59+pprrjl16lRYWJhuG0WA4BwhAJ3Y7fbq6urFixf369cvOTnZ22xhYWEzZ8587bXX9Ozt9ddf//3vf08Kyun/A4weD+bO6sD4AAAAAElFTkSuQmCC" + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Pkg.add(\"Plots\")\n", "# Pkg.add(\"PyPlot\")\n", "using Plots\n", - "pyplot()\n", - "\n", - "histogram(maf, xlab = \"Minor Allele Frequency (MAF)\", label = \"MAF\")" + "gr(size=(600,500), html_output_format=:png)\n", + "histogram(maf_cg10k, xlab = \"Minor Allele Frequency (MAF)\", label = \"MAF\")" ] }, { "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true - }, - "outputs": [], + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0.0013128198764010824" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# proportion of missing genotypes\n", "sum(missings_by_snp) / length(cg10k)" @@ -216,14 +398,23 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true - }, - "outputs": [], + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0.07228069619249913" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# proportion of rare SNPs with maf < 0.05\n", - "countnz(maf .< 0.05) / length(maf)" + "count(!iszero, maf_cg10k .< 0.05) / length(maf_cg10k)" ] }, { @@ -237,14 +428,57 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true - }, - "outputs": [], + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "629.627208 seconds (2.89 M allocations: 493.650 MiB, 0.28% gc time)\n" + ] + }, + { + "data": { + "text/plain": [ + "6670×6670 Array{Float64,2}:\n", + " 0.502735 0.00328112 -6.79435e-5 … -6.09591e-5 -0.00277687 \n", + " 0.00328112 0.49807 -0.00195548 0.000884994 0.00341591 \n", + " -6.79435e-5 -0.00195548 0.492348 0.000198191 -0.000337529\n", + " 0.00087876 -0.00322605 -0.00192088 -0.00235314 -0.00124267 \n", + " -5.03227e-5 -0.00352498 0.00184588 0.00219109 -0.00163189 \n", + " 0.00203199 0.000597697 0.00251237 … 0.00088428 2.21226e-5 \n", + " 0.000560862 0.00244517 -0.00183233 0.00120214 -0.00120686 \n", + " -0.000656949 0.00322249 -0.00101472 0.00355832 -0.000240444\n", + " -0.00103874 -0.00125164 -0.000599731 0.00176492 0.00176928 \n", + " -0.00137058 0.00209596 0.000146711 -0.0014453 -0.00103066 \n", + " -0.00209312 0.000140721 -0.000442031 … -0.000214963 -0.00107391 \n", + " 0.000933587 0.00168842 0.00185731 -0.000787908 -0.00311063 \n", + " 0.000334572 -0.000886623 0.00304182 0.000752285 -0.00123334 \n", + " ⋮ ⋱ \n", + " 0.00298109 0.00121739 0.00102822 9.66935e-6 0.00306953 \n", + " -0.00209728 0.00271452 -0.00182325 -0.00108139 0.00366901 \n", + " 0.000549425 -0.00244419 -0.00301368 … -0.000631961 0.00215641 \n", + " -0.00423362 -0.00208073 -0.00107904 -0.000619315 -0.000593852\n", + " -0.00326697 -0.000769552 0.00310511 0.000520658 -0.000113441\n", + " 0.000430563 -0.0020236 0.00265425 -0.00635493 -0.00520252 \n", + " 0.00218746 0.000798767 -0.00105684 -0.000918245 -0.00061484 \n", + " -0.00230525 -0.000101149 0.000117936 … 0.000879829 -0.00233479 \n", + " -0.00201305 0.00233864 -0.00134496 0.00197044 -0.000486275\n", + " -0.000990534 -0.000924159 -9.12302e-5 0.00122311 -0.00298296 \n", + " -6.09591e-5 0.000884994 0.000198191 0.499289 0.000481492\n", + " -0.00277687 0.00341591 -0.000337529 0.000481492 0.499799 " + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "# GRM using SNPs with maf > 0.01 (default) (~10 mins on my laptop)\n", - "srand(123)\n", + "## GRM using SNPs with maf > 0.01 (default) (~10 mins on my laptop)\n", + "using Random \n", + "Random.seed!(123)\n", "@time Φgrm = grm(cg10k; method = :GRM)" ] }, @@ -264,11 +498,30 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true - }, - "outputs": [], + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "ename": "SystemError", + "evalue": "SystemError: opening file cg10k_traits.txt: No such file or directory", + "output_type": "error", + "traceback": [ + "SystemError: opening file cg10k_traits.txt: No such file or directory", + "", + "Stacktrace:", + " [1] #systemerror#39(::Nothing, ::Function, ::String, ::Bool) at ./error.jl:106", + " [2] systemerror at ./error.jl:106 [inlined]", + " [3] #open#297(::Bool, ::Nothing, ::Nothing, ::Nothing, ::Nothing, ::Function, ::String) at ./iostream.jl:283", + " [4] #open at ./none:0 [inlined]", + " [5] open(::String, ::String) at ./iostream.jl:339", + " [6] #open#298(::Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}}, ::Function, ::getfield(DataFrames, Symbol(\"#_r#302\")){Bool,Char,Array{Char,1},Char,Array{String,1},Array{String,1},Array{String,1},Bool,Int64,Array{Symbol,1},Array{DataType,1},Bool,Char,Bool,Int64,Array{Int64,1},Bool,Symbol,Bool,Bool}, ::String, ::Vararg{String,N} where N) at ./iostream.jl:367", + " [7] open at ./iostream.jl:367 [inlined]", + " [8] #readtable#301(::Bool, ::Char, ::Array{Char,1}, ::Char, ::Array{String,1}, ::Array{String,1}, ::Array{String,1}, ::Bool, ::Int64, ::Array{Symbol,1}, ::Array{DataType,1}, ::Bool, ::Char, ::Bool, ::Int64, ::Array{Int64,1}, ::Bool, ::Symbol, ::Bool, ::Bool, ::typeof(readtable), ::String) at /Users/juhyun-kim/.julia/packages/DataFrames/lyCjP/src/deprecated.jl:1129", + " [9] (::getfield(DataFrames, Symbol(\"#kw##readtable\")))(::NamedTuple{(:separator, :names, :eltypes),Tuple{Char,Array{Symbol,1},Array{DataType,1}}}, ::typeof(readtable), ::String) at ./none:0", + " [10] top-level scope at In[19]:3" + ] + } + ], "source": [ "# Pkg.add(\"DataFrames\")\n", "using DataFrames\n", @@ -324,26 +577,25 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 20, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "┌ Info: Recompiling stale cache file /Users/juhyun-kim/.julia/compiled/v0.7/VarianceComponentModels/kuJce.ji for VarianceComponentModels [813005db-34b4-5f71-be9e-1bbf0a1d8f1c]\n", - "└ @ Base loading.jl:1185\n" + "WARNING: using VarianceComponentModels.mean in module Main conflicts with an existing identifier.\n" ] }, { "ename": "UndefVarError", - "evalue": "UndefVarError: Φgrm not defined", + "evalue": "UndefVarError: Y not defined", "output_type": "error", "traceback": [ - "UndefVarError: Φgrm not defined", + "UndefVarError: Y not defined", "", "Stacktrace:", - " [1] top-level scope at In[7]:2" + " [1] top-level scope at In[20]:2" ] } ], From d44d13d55b8df0a0e87cac017edc8bce83f909fc Mon Sep 17 00:00:00 2001 From: Juhyun Kim Date: Fri, 25 Jan 2019 23:05:11 -0800 Subject: [PATCH 45/56] update chunks for cg10k.txt --- docs/heritability_v0.7.ipynb | 1212 +++++++++++++++++++++++++++++----- 1 file changed, 1050 insertions(+), 162 deletions(-) diff --git a/docs/heritability_v0.7.ipynb b/docs/heritability_v0.7.ipynb index 02518ab..b326236 100644 --- a/docs/heritability_v0.7.ipynb +++ b/docs/heritability_v0.7.ipynb @@ -16,18 +16,7 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - ";ls cg10k.bed cg10k.bim cg10k.fam #cg10k_traits.txt" - ] - }, - { - "cell_type": "code", - "execution_count": 2, + "execution_count": 1, "metadata": {}, "outputs": [ { @@ -36,12 +25,13 @@ "text": [ "cg10k.bed\n", "cg10k.bim\n", - "cg10k.fam\n" + "cg10k.fam\n", + "cg10k_traits.txt\n" ] } ], "source": [ - ";ls cg10k.bed cg10k.bim cg10k.fam " + ";ls cg10k.bed cg10k.bim cg10k.fam cg10k_traits.txt" ] }, { @@ -53,7 +43,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "metadata": {}, "outputs": [ { @@ -95,8 +85,10 @@ }, { "cell_type": "code", - "execution_count": 5, - "metadata": {}, + "execution_count": 3, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "using SnpArrays" @@ -104,7 +96,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 4, "metadata": { "scrolled": true }, @@ -113,7 +105,7 @@ "name": "stdout", "output_type": "stream", "text": [ - " 0.178436 seconds (466.95 k allocations: 43.173 MiB, 5.68% gc time)\n" + " 0.196193 seconds (466.92 k allocations: 43.171 MiB, 11.74% gc time)\n" ] }, { @@ -148,7 +140,7 @@ " 0x03 0x03 0x03 0x03 0x03 0x02 0x03 0x00 0x03 0x03 0x03 0x03" ] }, - "execution_count": 6, + "execution_count": 4, "metadata": {}, "output_type": "execute_result" } @@ -167,7 +159,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 5, "metadata": { "scrolled": true }, @@ -178,7 +170,7 @@ "(6670, 630860)" ] }, - "execution_count": 9, + "execution_count": 5, "metadata": {}, "output_type": "execute_result" } @@ -196,7 +188,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 6, "metadata": {}, "outputs": [ { @@ -230,7 +222,7 @@ " [6511 , 630860] = true" ] }, - "execution_count": 10, + "execution_count": 6, "metadata": {}, "output_type": "execute_result" } @@ -248,7 +240,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 7, "metadata": {}, "outputs": [ { @@ -258,7 +250,7 @@ " 2 0 132 77 0 27 2 2 6 27 2 … 6 4 5 11 0 0 4 29 0 5 43" ] }, - "execution_count": 11, + "execution_count": 7, "metadata": {}, "output_type": "execute_result" } @@ -276,7 +268,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 8, "metadata": {}, "outputs": [ { @@ -311,7 +303,7 @@ " 0.1324128564961521 " ] }, - "execution_count": 12, + "execution_count": 8, "metadata": {}, "output_type": "execute_result" } @@ -322,7 +314,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 9, "metadata": {}, "outputs": [ { @@ -331,7 +323,7 @@ "([0.00841726 0.124063 … 0.364253 0.5], 0.24536516625042462)" ] }, - "execution_count": 13, + "execution_count": 9, "metadata": {}, "output_type": "execute_result" } @@ -344,7 +336,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 10, "metadata": {}, "outputs": [ { @@ -362,7 +354,7 @@ "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlgAAAH0CAIAAABuMsSDAAAABmJLR0QA/wD/AP+gvaeTAAAgAElEQVR4nO3deXxU1f3/8TOZLJCFQCJrILGAylbKWhfiQlG/CoQQl1YEEYstliJI+aloAatQVr8qYi11iaI2AomEqGAQgYCAFhQqioSoZY0hgewLmeXO/f0x7XzTmTtDlsm9Mzmv54M/Jvdzl8+cuTNv7tyZOyZVVQUAALIKMboBAACMRBACAKRGEAIApEYQAgCkRhACAKRGEAIApEYQAgCkRhACAKRGEAIApEYQAgCk1paD8PTp0+Xl5f5dp91u56J0QgiHw6EoitFdBASbzWZ0CwFBURSHw2F0F8ZTVdVutxvdRUAIolfLthyEf/zjHz/88EP/rrOiooK9XAhRX19fU1NjdBcB4cKFC0a3EBCqq6stFovRXRjPZrNVVlYa3UVAKCsrC5b/G7XlIAQA4JIIQgCA1AhCAIDUCEIAgNQIQgCA1AhCAIDUCEIAgNQIQgCA1AhCAIDUCEIAgNQIQgCA1EKNbgA6uXDhwtmzZz2n19fX3/PrGaeOHdFcasS1yQf3f9rKrQGAkQhCWdw0Nu3M+XJTiNltusNmrS45I161aizz3d76vEV6NAcAxiEIZXHq+/yahV+JmM7uheICsfhqIzoCgIDAOUIAgNQIQjRHfn7+FT8d2iG+q+e/2Mu6vvrqq0Y3CACNxVujaI59+/ZVJ15Xf8dyz1L4B3+qra3VvyUAaB6CEM1kCm8vOnTVmh6pfzMA0Gy8NQoAkBpHhAAQQJYtW7Zx40aju/ADRVHMZvfvazXGVVddtX79er/340PQB+Hhw4eTk5M5KdUqLladOF14eb/BnpWqilLbFbfo3xHQ5p05c+b222+/++67jW7EGAUFBUuXLtV5o8EdhOfOnXv66afr6uqMbqSNKj110RR+6p43NUqbFpiFqnc/gBx69uw5dOhQo7swRkiIASfsgjgI6+vrZ8yYsWbNmpycHKN7abNMYe1Ez59qFCI76t4LALSKIP6wzMMPPzx37tzExESjGwEABLEgPiLMzMx87bXXnLdNJtPFixfbtWtnbEsQQgiHsn//fm+Pxfjx43v27KlzRwDgQxAHYUVFhfOGyWRSVc5XBQr7yUObTqgflh7yLKlHPurduzdBCCCg+D8IFUUZOHBgfn6+Z2nUqFH79+933p4xY8batWubtJ7y8vKpU6fu27cvOTl53bp1nTp18m/n8A/Fpvx80sWbZ3tWYsvG6t8OAPjm5yBcvXp1RkbG8ePHPUuqqubn5589ezY2NlYIERrqa9Oa61mxYkVSUlJWVta8efNWrly5bNky15o1V6IoypEjRz766CO36X379m32mUWr1WqxWBwOR/MWb20nTpx46623NUsWq9YPLbVEMw7CVdVms1ksFj93YijnLmF0F8ZzDoIhH/kLKFartYW7RMC+vOhGVdVmDGBYWFizdz8/B+HgwYP79OmTkpLiWSouLrZarampqcePHx8zZswrr7ziPI1kt9tdoei6rbme7OzsnJyciIiIWbNmpaamuoLQG4vF8uGHHx44cMBt+rRp01JTU5t3B2tra81mc1hYWPMWb213TX3waMQVIk4r5q023dtxpwpRX1/fxr70WVdX18buUfPU1dUpisKLuNVqra2tjYiIaPYabDb3p+rzq198J0unz8aHCLHw0TkTJkzQZ3OaVFVtxnMqOjo6PDy8eVv0cxCOHj3aW+ncuXMjR4587rnnEhMT586dO2fOnHfffVcIMWPGjLFjx955550rVqywWCyLFi3ytp7CwsKkpCQhRFJSUlFR0SWbiYyMfPLJJ++7777m3x8PDocjNjY2YIPQbDaLa6eIK2/QqOWu8vPGTE1fwmSKiYmJi4vzcyeGslgsbeweNU9ISEhERET79u2NbsRgVqs1LCysJbuEZ4geOZp/KGKA+Nn4lrXWKOF5L509e1aHDfkQEhKi83NKvw/LDBkyZOfOnc7by5cvHzhwoPP2qlWrbrvttpycnNLS0k2bNvlYg6qqJpPJeUNRlNZuGAACRbcrRf9f6LAd81fZmtNNJtOkSZMyMjJcUyZPnpyRkeE6M2W32xMSEuLi4r799lvnC7VzKberrNnt9tZpvEX0e0P/0KFDrk/KhIeHu/7XExcXl5aWtmHDhmnTpvl+P6FHjx5nzpwRQhQWFiYkJLR2wwAAl7y8PNepO6vVmpeX17C6e/fuyy677MKFC0ePHm043f7fdOu2SfQ4IszLy7vppptqa2vvuuuuvLy8Pn36LF68eOLEic7q2rVrd+3aVVBQMHHixKioqLFjvX6wMCUlJT09fenSpenp6c0+yQcDqXbrjh07iouLNat33nlnZCQ/4QQEqOTk5O3bt48fP14IsXPnzlGjRmVmZrqqGzduvP/++0+cOJGZmTlo0CDj2mwOPYJw9OjRqqomJyc/9dRTKSkplZWVY8eOXbNmjbPaq1evzZs3R0ZGbt26VfPjpi6LFi2aPHlyr169hg0b9vbb2p+NRCCru1C4+sMDYZ8VepYsB7NvuukmghAIWHfcccd7773nDMLs7Oy0tDRXENpsts2bNx86dOi777773e9+96c//cn17mhQaJUgdPs+g/NPk8k0c+bMmTNnus08btw4543u3bt3797dx3o6duy4ZcsW/7cLvag2m+XWRy0DNX62Iip/t/79AGi8cePGzZs3z2azmc3m3NzclStXuko7d+4cPHhwQkJCt27dKisrjx496joobJiIr7766oMPPqh3340QxFeWQVtic4i0e7VPEneKifwwO0v/lgA0FBMTM3z48Ly8vMjIyEGDBjm/Ee60cePGTz75xJV5Dd8dDYrLfhGECAi22sovr5wiouM9CvXhbz6we7f28WKHDh2k/bUaQH/Od0ejo6PT0tJcE61Wa3Z29qlTp5wXKtm2bdsjjzwSXO+OEoQIGFfdJOI8LkNafd4qQifOWug5u/1idWJM2NHD7hdMANBKJkyYsGDBgsjIyMcee8w1cfv27YmJia7Ldd14442nT59u+O5o4CMIEdgcdmGzVMzZoVE6cdC+5RHdGwLkFRcXN2DAAIvF0qVLF9fEjRs3NrwKWLt27caMGRNcnx0lCAEAl+A61ffxxx+7TVy3bp3bzO+//77bUgGOIETQUmwXyirunvKAZvHWm5J/8+B0nTsCWkNcfFzEzrfCTn2uw7bs/zooxGAdNhRQCEIEreLvyqtrssJGaZS+36/u+pQgRNuw9E8Lhw7sp9fWbv/5z3+u17YCBUGIIGaKjFVH3a9VMAnLp7q3A7SKiIiIKVOmGN1FWyb7j4cBACRHEAIApEYQAgCkxjlCtE2VFeVffPGFZqlnz57dunXTuR8AAYsgRFtUdCxv/2c33zvDs2Ip/fFPjz3y+OOP698UgMBEEKItqikT/UdXPviOZyV005P6twMgkBGEABBA2rdvn5mZmZ+fb3Qjxrhw4YL+GyUIASCALFmy5NVXXzW6Cz+orq6OiooKCWnaRzL79u3b8Mql+iAIASCAtG/ffvbs2UZ34QclJSXx8fFms9noRi6Nr08AAKTGESHkotaU/u/Lm99Yv8mzZDaJtaufvf766/XvCoCBCMLg88HWbVN/rX056eqLFsHLuE/qj8fO97nl/LWTPUvRm+eXl5fr3xIAYxGEwSdzU3bFqN+JazReykOWaf0UA9zE9RQ/0bi+vjmqo/69ADAcQRicIjuKTgmek00m/VsBgODGh2UAAFLjiBD4N1Wx7d279+LFi5rVcePGRUdH69wSAB0QhMC/XSw+9eLmyvB9Jz1LlsMfHT1yuG/fvro3BaDVEYTAv6k2i+V/HrP8bLxnKfpkf/37AaAPzhECAKRGEAIApEYQAgCkRhACAKTGh2WAS6u3Wq/qP0DzcgWhoaH1F+v0bgiA/xCEwKUpdVXqoi9F597uBYddnX2ZER0B8BuCEGgcc5gIDXefqHBROyDoEYRAC6iq2q5Dv2HXaBaTenRZ8PijmqXOnTv369evNTsD0FgEIdACDrtaV3H8tmc1Sj98/t3W5QdOPuFZsVZeGHP1kPez1rd6ewAagSAEWqz31RoTy86I9rEVj+zSKB3caL/wQWs3BaCR+PoEAEBqBCEAQGoEIQBAagQhAEBqBCEAQGoEIQBAagQhAEBqBCEAQGp8oR7QnbXun18fHTP+Ds+KSYgZ90+6++679W8KkBZBCOjuzFcl9nZFiZM0Sgc39N+zZ9CgQZ6VdzI3P7tiqeZFvk0mcfiLg1y8FGgeghAwgKljdzFsokbhwPq/vvnOuvc/8azUlhY77lkjhk7wLMWsHKUoit+bBCRBEAKBpOq88ovZ1SkLPCuhCwc6wtqJdjGeJVMIJ/uB5uP5AwCQGkEIAJAab40CQU9V7Dt27CgoKNCspqWl6dwPEFwIQiDo1dXWPPFKdmhUrGep6kBOVVVViNZJxJCQkPbt27d+d0CgIwiBoOew1df98gXRY4BG7Zsu8V26eU5WVbVnr8QT3+W3enNAwCMIgTatrsL2Sr0weRwRnjtufeMuIxoCAg5BGLgOHjxos9k8p58/f1500L8dAGibCMIAlZOTM+k3D4dfluBZqjlTICb+Qv+WAKBNIggD1KlTpxxDJlT+8nnPknnxCP37AYC2iu8RAgCkxhEhIKX66vPlldGdLtMsjh83bv0763TuCDAKQQhIqfqCXTXZnvpao/R1bsm593VvCDAMQQjIyhQiorWOCLWu6w20YZwjBABIjSNCAH5QUlLyl7/+LTRU4yXFZBJz58xOSND4LhAQCAhCAH7wyPyFH51VReIQz1K73S9P+tUvCUIELIIQgJ8MHieuuddzcsRXWfr3AjQe5wgBAFLjiBBAYymKsmDRU4qieJaOHz8urh2ruZSl/uL4u+4NDw/3LJkc1hf/d1VYWJhnKTIy8sYbb2xhw0BjEIQA3FWWl+Xm5npOX79+/Vufn1AH3OJZCim64G1t1qrSc/e8IHr016ituPG+p9aYQsxukx2Wi+0rThWfPdmktoHmIQgB/Lcfv/3qWME9T2pc57b2hy/FqF+LsY97lkL+8XeHj3V26SMSBmlMt1ys+u17Iszj94FLT4W/eHPjWwZagiAE8N/qa0w9f1o580ON0vNjTbq3A7Q2ghBAILJZLenp6ZqlXr163XKLxtuzQPMQhAACT+W56tq62W/t8awoZYUDY6yXXaZ9ufCuXbv26NGjlZtDW0MQAgg8DkWERtROeVWjtOOlf25ZOvruBzwrlorzv516z+rn/rfV20PbQhACCCp1FaJH/8r/t0Oj9PHzDrVY94YQ9PhCPQBAagQhAEBqBCEAQGoEIQBAanxYBgD8pqSkZOzdU0qKCjWrv5123zN/WqRzS7gkghCA1KxWa1lZmWbpyy+/PHDggMmkcTmdmJiYefPmeU7/9ttvT5RdrPnt+xqr+zzjx+KSljWLVkEQAmj7qqurX3rpJc3Sxi3bvz3yT3OYxo9jXKyqEDfPFmHt3AuWmg6H/qYZhEKIkPD2oktfjYIpZNOHubv3jdComMSfFzx+9913+7wTTfDOuxs+/8c/NEtX9P7JnNkP+2tDbQNBCKDt+90j/+/dL86Yev7Us+Q48rX6+82iz7Vai8WI2x8VkR3dp1cWiUPvNrmJwqOVXQeX3/aYZyVi+7MlJX47WCwrK3vggQfsqU9r1C6cGrBvPUHohiAE0HacOnHi/fc13pY8deqUY+gdIlnjejQh+99WW78xJ1N0vLh8uMb0Dp39uyFzeDv7LY9oFL7fL3Yc9u+22gCCEEBbcerwRycO7/nR5lmpOX5I9LxD/46Cy8svv1xdXa1ZGjNmzIgRGm/qtg0EIYC2wlLt+FlK5a80rjUa+sxQ/dsJLm+88cbclX9zDLhVo5a/KywsjCAEAPyHw25xmMaM1zjELLtQYrEo+nfUcna7PbT3iLo7lnqWwrMe1b8fPRGEANBENWWWmsqdiZM0SuUfmO3H9OniySUrNr23yXO6othtNqs+PbQNBCEANJkpNEwdNlGjcOGEKNIpCP/ywvNVU14VUXHuhfJCcVzjY0Gt4c2MzKeX/FmzZFbtedtze/bsqU8nLUEQAoDBlPOnZs1aO2vWLM3qzp07R48erb1kryEitpv7xOh/+dhWRen5BQsXek4/fPiwomj/3LEPa197/eSw6ZpfPmn/UorVGhwHpgQhABjMUVEk7l0tRv/Os9TxxVv8uaWz3xSVVf35a7NG6ciZ0F6dtJdSHYcPH373XY2vTpaWlorBPxG9fuZZMoWGtahVHRGEACARU4cu6vg/ahRKvve2iHLqn+uPVeSc0PheSu3JU37szSgEIQAELmtdzW/mPBodHe1Zqq2r06kJW7392vuqtb6hb37iCp16aE0EIQAELkv5uR9GzNV871F8map7O20TQQgAga3nYHHVjRrTNX4VA81BEBqsrKxM84NVVVVVQuh2BUQAkBdBaKT8/Pxrf/E/FovFs2StqRCjpuneEQBIhyA00pkzZ9QuV1yc9ZFG7a+/0vqAMwDAz0KMbgAAACMRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqfGFer/57LPPcnNzNUsdOnSYN2+ezv0AABqDIPSb0bf8j+XGmSI03L1QX9Pxn68RhAAQmAhCv7rtUdG+g/vEih/FPzcY0Q0A4NI4RwgAkBpBCACQGkEIAJAaQQgAkBoflml9DsViV++aPM2zUnLuR81f5QUA6IYgbH3V5y/WVL4Xcb1Gqeojs71K94YAAP+HINSDKTRCvW6qRqGyWFSc1b0dAMD/4RwhAEBqBCEAQGoEIQBAagQhAEBqBCEAQGoEIQBAakEfhIcPH46KijK6CwBAsAruIDx37tzTTz9dV1dndCMAgGAVxEFYX18/Y8aMF1980ehGAABBLIiD8OGHH547d25iYqLRjQAAglgQB2FmZubo0aNNJpMQwmQy1dfXG90RACD4BPG1RisqKpw3TCaTqqrGNgMACFL+PyJUFKVfv36apfLy8pSUlLi4uAkTJpSXlzd1PU1aHACAxvDzEeHq1aszMjKOHz+uWV2xYkVSUlJWVta8efNWrly5bNmyJq3H2+LeDgdramrmzp27YMECt+lz5syZMmVKE+5VA2VlZRaLJSwsTKPm66jUe8mvh7J+PzD2tb7mbEu/A3dvQ+HrQfJ/d17X2Kz3MHQcvaZvqxWa8+cqfY638btygLyj5a0Nn7urr528tLQ0Ojq6hV01UmxsbERERPOW9XMQDh48uE+fPikpKZrV7OzsnJyciIiIWbNmpaamOpPMbreHhv67DddtzfVoLu5DVFTU/PnzU1NT3abHx8fHxsY27w6qqhobG6sdhCaT9+W8l3ws1HS+WmjeCptZ8+cyzeNtKEw+nrj+787rGk3Neah0HD0dt+WdP3vwOd7N3JX9mF6BMNzCexs+d1dfO3mnTp3i4+Nb2FUjhYQ0/w1OPwfh6NGjfVQLCwuTkpKEEElJSUVFRc6JM2bMGDt27J133rlixQqLxbJo0SJv69Fc3AeTydS1a9crrriiGXfEG/N/+HGdCBAB8mIUpBg9eAoJCQmKV0tdPyyjqqrzfxaqqiqK4py4atWq2267LScnp7S0dNOmTU1dHACAltD16xM9evQ4c+aMEKKwsDAhIcE5MS4uLi0tbcOGDdOmTfP9Dq/m4gAAtIQeQZiXl+e8kZKSkp6erqpqenq669Td2rVrd+3aVVBQsHTp0q1bt/pYj+biAAC0hB5B6Drht2jRoiNHjvTq1evo0aOuD3P26tVr8+bNSUlJW7dujYyM9LEezcUBAGiJVjlH6PZZW9efHTt23LJli9vM48aNc97o3r179+7dfaxHc3EAAFoiiC+xBgBAyxGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkFcRA6HI5p06aNGDGiX79+mZmZRrcDAAhKQRyE27dvt9lsX3zxRWZm5syZM41uBwAQlII4CH/2s58tX768vr7+zJkz3bt3N7odAEBQCjW6gebr1q2bEGL8+PFbtmz5+OOPjW4HABCUgviIUFEUh8ORk5OTlZX10EMPGd0OACAoBXEQvvzyy48//rjZbO7Tp09dXZ3R7QAAgpKfg3DUqFGm//A8SisvL09JSYmLi5swYUJ5ebnvVSmK0q9fPx+LT5069Ycffhg8ePCkSZNeeeUV/94RAIAk/HmOUFXV/Pz8s2fPxsbGCiFCQ91XvmLFiqSkpKysrHnz5q1cuXLZsmXeVrV69eqMjIzjx4/7XnzTpk0++rFYLJs3bz558qTb9BtuuGH48OFNumsutbW1ZrM5LCxMo6Y2a43NWyoQNKdzHe9t0zcV8A+Ffg02Y0ut0Jw/V6n6WlnAP/JGU30Nn/eSqtbV1dXU1LRCRxratWvnGTqN5M8jwuLiYqvVmpqa2r179ylTplRVVbnNkJ2dPWvWrIiIiFmzZrkyzG63u2Zw3R48ePDChQsbs7hvtbW1ZR4uXryotoJmjhoAtFGt8UrbGq/A/jwiPHfu3MiRI5977rnExMS5c+fOmTPn3XffbThDYWFhUlKSECIpKamoqMg5ccaMGWPHjr3zzjtXrFhhsVgWLVokhBg9erTn+jUX9yEiImLy5Mn33Xdfy++ai8ViiYmJ0T4iNDVrjc1bKhA0p3Md723TNxXwD4V+DTZjS63QnD9XafK1soB/5I1m8jV83ksmU1RUVExMTCt05Gf+DMIhQ4bs3LnTeXv58uUDBw50m0FVVeeAqqqqKIpz4qpVq2677bacnJzS0lLfx3maiwMA0BL+fGv00KFD+/fvd94ODw+PiIhwm6FHjx5nzpwRQhQWFiYkJDgnxsXFpaWlbdiwYdq0aZ6LXHJxAABawp9BWFtbm5aWduzYMavVunjx4okTJzqn5+XlOW+kpKSkp6erqpqenp6amuqcuHbt2l27dhUUFCxdunTr1q0+1q+5OAAALeHPIExOTn7qqadSUlISEhLKy8tXrFjhnO464bdo0aIjR4706tXr6NGjCxYscE7s1avX5s2bk5KStm7dGhkZ6WP9mosDANAS/jxHaDKZZs6c6Xn9a9fneTp27Lhlyxa36rhx45w3unfv7nbJULcPAmkuDgBASwTxlWUAAGg5ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACC1UKMbCDI5H21/4613TCaTZ8lms+nfDwCghQjCpln+7HPfX3W36HalRm3Xdt3bAQC0FEHYdH2vE72vNroJAIB/cI4QACC14A7C8vLylJSUuLi4CRMmlJeXG90OACD4BHcQrlixIikpqaioKDExceXKlUa3AwAIPsF9jjA7OzsnJyciImLWrFmpqanLli1rWFVVtaamprS01G2pyMjIiIiI5m5TFfXVos7L0efFSqEqHhOrhVC1F7HVC4eiXVJswm7RLqkOYa3TLKmqEPU13turEmaPR/xitcUOxI4AABDGSURBVBBCexFrneqtPbtVmLzcKYdDWC966UEVllpfo1cX5T6xrspHe8Lh8DJ6VtVu0x4ihyKs9V5Gz2d79VUaJWu9r/ZUL6NnswhFuz3hsPt83H2MnlZ7TnUVwvNzzpYaoXp5BO31vtqzeRs9r7ulEMLXs6auQoTVa8yvenlwm/WsUR2q9/Z8P6mrhFC12mvek9rq/cH19qwRPp81VSLM49XMx2uOtc7rwNqtakiI92dNk5/UVkUdft0NISFmz1KHqPY/HP9Wa23NFxLS/OM6k6p6PMbBIzo6+vz58+3bt7948WLXrl2rqqoaVtPS0rZt2xYeHu621GOPPTZ9+vTmbXHcxDuPfvONZsliqY+IaOc5XVVVm6U+vF17z5JDsSuKEhaukcqKzaqaTKGhYZ4lu91uMpnMZo3dy6Yo5hBTiEljh7BYLRHh4UK4vyCqqmq1XIxoF6nVnqIodq/tCVNomGZ7NpMpxEt7DrNJe3+1Wq1h4WEmj/aEqlq8tedQ7DZbuNaYK3abqgov7VlNphCz538InKNnMmm3Z7OFhYZqfm3GcrE2or1HfguhOhSb9/YcqhoW5r5nCiHsVoswm0O9tic0X1asNntYqNlLe3UR7TVGT3U4bDarZnsOu11RHdrt2SzCZA4N1WrPbg/xslta7fYws5f26i9GaD01VNVhs1i8P2scYR7Pa/HvZ02IZnu+njV2xWz28qzx/qS2Wuo1O2/uk9r7s8aumEO0d0uL1RoRHua3J7Xdqoqmt+f9SW2xWsPDwjQf96jI9keP/NNzekt07Nix2Uc4wX1EqKqqc5RVVVUU90Ox6Ojov/3tb/fdd58ft5j7webY2NgwrVdYqdTV1dlsttjYWKMbMV5RUVH37t2N7sJ4FRUVERER7dtrZINUrFZrdXV1fHy80Y0Yr6SkJD4+XjM+A01wnyPs0aPHmTNnhBCFhYUJCQlGtwMACD7BHYQpKSnp6emqqqanp6emphrdDgAg+AR3EC5atOjIkSO9evU6evToggULjG4HABB8gvscYceOHbds2WJ0FwCAIBbcR4T6e+GFF06ePGl0F8bbu3fvhg0bjO7CeIqizJ8/3+guAsLbb7998OBBo7swXkFBwcsvv2x0FwFhyZIlZWVlRnfRKARh0+Tm5hYXFxvdhfGOHz/+2WefGd2F8RwOR0ZGhtFdBIRPP/30+++/N7oL4/34448ff/yx0V0EhE2bNlVXVxvdRaMQhAAAqRGEAACpEYQAAKkRhAAAqQX31yd8s9lse/bssdvtflznhQsXtm7d+t133/lxncHoH//4x6lTp9544w2jGzGYoigOh4NxEEKcPHly7969mheWlMo333xTXFzMLiGEqKury8rK6ty5sz6bGzNmTGJiYvOWDe6LbvtWUFCwePFizSvwNltdXV27du1acpnztsFmsymK0q6dxvWIZVNTUxMdHW10F8arr68PDQ3179MtGCmKYrFYIiM1Lngtm9ra2sjISN3+b/Twww8PGzasecu25SAEAOCSZD+yAQBIjiAEAEiNIAQASI0gBABIjSD0pby8PCUlJS4ubsKECeXl5U2qtjGXvLOKovTr10//xvTkexBycnIGDRrUsWPHG264oaCgwJAOdeN7KHJzcwcMGNCxY8cBAwa07QtvNuZF4JtvvomKitK5Mf35HopRo0aZ/uOhhx4ypEMfCEJfVqxYkZSUVFRUlJiYuHLlyiZV2xjfd3b16tXXXXfd8ePHDelNNz4G4fTp01OmTHn11VeLioomTJjwwAMPGNWkPnwMhcPhmDx58po1a8rKyp555pm2PRSXfBGorKycNm1aXV2d/r3pzMdQqKqan59/9uzZ6urq6urqF154wagmvVLh3ZVXXnns2DFVVY8dO3bllVc2qdrG+L6zO3fu/OCDD9r87uRjEHbt2vXggw86b5eUlMTHxxvQn458DIXFYtmyZYvD4aiqqnr//fcHDBhgUI968P28cDgcEydOzMzMbPNPDdXnUBQVFUVHRw8fPjw6Ojo1NbW4uNigHr1q+w9PS0RFRdXV1amqWldXFxMT06RqG9OYO9vmn+2NGQS73f7QQw/NnDlT39b0dsmhcP7+jslk2rdvn+7d6cf3OCxbtmzevHmqBE8N1edQHD58ePTo0YcPHy4tLZ06deo999xjUI9e8daoL6qqOi+LoKqqoihNqrYxUt1Zby45CJ988snIkSNjY2NXr16te3e6uuRQREdH19TULFmyZM6cObp3px8f47Br167c3Nxly5YZ1JrefAzFkCFDdu7cOWTIkLi4uOXLl2/bts2gHr0iCH3p0aPHmTNnhBCFhYUJCQlNqrYxUt1Zb3wMgqqqTzzxxDPPPLN+/frly5e3+SuN+RiKkydPPvroo0KIqKio6dOnHzt2zJgWdeFjHHbs2LF79+7w8HBnPJhMpr179xrTpS58DMWhQ4f279/vvB0eHh4REWFAfz4RhL6kpKSkp6erqpqenp6amuqcmJeX56PaVvkeCkn4GIT9+/dnZ2e///77PXr0qKmpqampMbLR1udjKHr06PH666/v3r1bVdUNGzYMHTrUyEZbmY9xWLJkieudNyGEqqrJyckGttrafAxFbW1tWlrasWPHrFbr4sWLJ06caGSjmnR4+zV4lZeXjx07NiEhISUlpaKiwjnRNWia1bbK91Bo/tn2+BiEJUuWSPXM8r0/5OXlDRs2rFOnTtdee63zAxRtVWOeF5pT2h4fQ+FwOP7yl7/06dPnsssumzp1amVlpaGdauCi2wAAqfHWKABAagQhAEBqBCEAQGoEIQBAagQhAEBqBCEAQGoEIQBAagQhAEBqBCEAQGoEIQBAagQhAEBqBCEClMlkioiIqKiocJteUVERERHh/Gkb52z5+fmt3cygQYNMJtPrr7/u2aRr643spBkNm7xo0kqCi9VqHTlyZElJiWj0nuDi48HyNozFxcUjRoywWq2tdocQ0Nr4r6YhqFmt1k2bNv36179uODE7O1vnF6yjR48ePXo0KioqMzNz+vTpem7aZc2aNT179jRk04ZYs2bNzTff3KVLF+efjd8TfD9Y3oaxa9euv/jFL1566aU//OEP/rsTCB5G//wFoE0IMXTo0DFjxrhNv+WWW5w/cef8s7q6WlGUVu1k4cKFsbGxf/7zn81m84ULF9yadP3MUMPbPjRythYuEtRsNluvXr1OnDjh/LORe4JTIx8sTz/88ENiYqLdbvfPfUBQ4a1RBK577713586dRUVFriklJSU7duyYNGmSa0p0dHRISCvuxqqqbtiw4a677rr33nsVRcnOzm69bcFp27ZtPXv2vPzyy11TGrMniJY9WL179+7evfu2bdta3D6CD0GIwHXrrbd26tRpw4YNrilZWVkdOnS49dZbXVPcztIdPHjwV7/6VVxcXO/evRsu6HA4XnjhhYEDB0ZHRw8fPjwrK6vhGg4ePJiSknL11Vd79vDVV18VFBRMmjTp8ssvHzFiRGZmZmM6V1V17dq1/fv3j4yMHDp06Lp161StH/5s5Gy+ufXvY50HDhwYM2ZMp06devfu/Zvf/Obzzz/XPNWan5/f8MSbjxX6GHBVVV988cUBAwZ06NDh+uuv37t3rxBiwYIFP/nJT1yLnz59OiQkJCcnx+0ebdu27cYbb2w4pTF7gmjug+Vy4403fvzxx01aBG2EMQeiwKUIIY4dO/bb3/525MiRronJycnTp08/duyYa9cV//3m5NVXX7127dr9+/dPnTo1PDy8pqbGWVq1alV0dPTKlSu3bt06f/58s9n8wQcfuJYaMWLE9OnT//73v3u2MX/+/G7dujnfMVu+fLnZbD5//rxbk56333rrrQEDBqxbt+6jjz5ybu6vf/1rk2ZzG4otW7Yc+2/19fWa/Xtb5xdffGE2mydOnJiRkZGZmTlp0qT4+HjNYWw4vJe8L94GfPXq1bGxsS+++GJWVlZaWprZbP7qq6+++eYbIcTnn3/unGfp0qVdu3a1Wq1u93fYsGFZWVlug3zJPaExD5aPYVRVdcOGDcOHD/ccf7R5BCEClPPlLy8vTwhRUFCgqurp06eFEJ988omPIHz++eedt6uqqlwlh8MRHx/vPJRxmj9//vXXX+9a6tFHH9XsweFw9O7de/bs2c4/v//+eyHEK6+84tak5+1hw4Z9//33rtn+8Ic/JCcnN2k2t6Hw9PXXX2v2722dt99++9SpUxuu1vlZEs+u3NLF933xNuDdunXLyMhwlhRFuf32299++21VVQcPHjx37lznPP369dMc+c6dO+/bt6/h3W/MntCYB8vHMKqqumfPni5dunj2gzaPIESAcr78KYqSkJDw9NNPq6q6atWqrl272mw2H0H4xRdfuK1BVdXi4mIhRGlpqau0Z8+e+Ph412x79uzR7OHAgQOeL5233HKL5ybcbkdFRbkt1bVr1ybN5jkUPgaqYf/e1tmlS5ddu3Y1XHDnzp2NCULf90VzwM+fPy+EcPusitOyZcsSEhIURXGO7bfffus5T2ho6JEjR9xWe8k9oUkPlqavvvoqNDTUxwxoq/j6BAJaSEjIpEmTMjIyFi5cuH79+l/+8pehob52Ws9XbW+rVRTF9Wfnzp01Z9uwYUPnzp13797tOme2bt26VatWnT9/3tsiTpGRkZ9++mn79u19t9HI2S6pYTPe1mm1Wt2+cmc2mzXXVldX1/gmNQfcbrd7W/8999zzxBNP7Nu3LzMz89prr+3fv7/nPB07dqypqXGbeMk9odkPlkt1dXWnTp0aMyfaGD4sg0B37733Hj9+fOPGjV9++aXbpwQbqXPnzvHx8Vu3bnVN2bJly6BBg3wv5XA4Nm7cmJqa2r9//37/cf/99zfm44gDBw4sLCx0LnLVVVetWbNm3bp1zZ6tSbyt86c//embb77ZcM6MjIyGf7ry77PPPmthk127du3UqdOOHTucfzocjhEjRixdulQIcfnll1933XXvvPPOu+++6+1LmT179mz4AVEXH3tCSx4sl6KiIqm+rIn/Y/QhKaBNNDjhdNVVV3Xq1CkpKcnhcKj//d6d8PLmpNufK1eujImJefbZZzU/LKP5jtm+ffuEEFu2bGk40eFw9O3b9+abb/a99ZycnJiYmOeffz47O3vq1KlCCM3N+ZhNcyh8D5TvdW7fvl0Icccdd6xfvz4rK2vy5MmxsbGuYUxKShozZkxubu6bb77p9uW8Rt4Xtz+XLl0aFxf30ksv5ebm3n///eHh4a6zcWvWrAkJCYmMjKyqqtK8RzNnznzyySc9V+tjT2jqg6XpiSee+P3vf+9jBrRVBCECVMOXrWeeeUYI8fjjjzv/bEYQKory3HPPOb8DMGTIEM8PJXo2MHv27JiYmIafKnSaN29eSEhISUmJ762//vrr/fr1i4yMHD58+Hvvvedtc95m8zYUjal6W2dubu4111wTHR19+eWXT5ky5dNPP3UN444dO/r16xcdHT1mzJh//etfbv9FbuR9cRvw5cuX9+3bNzIycuTIkdu2bXPNdu7cuZCQkGnTpnm7R9nZ2Q0/NNSYPaGpD5am5OTkzZs3+5gBbZVJbfr3lgC0Afn5+f3799f/FeDEiRO9e/fes2fP9ddfrzmDzWZLSko6cOCAnm9Unj59+pprrjl16lRYWJhuG0WA4BwhAJ3Y7fbq6urFixf369cvOTnZ22xhYWEzZ8587bXX9Ozt9ddf//3vf08Kyun/A4weD+bO6sD4AAAAAElFTkSuQmCC" }, - "execution_count": 14, + "execution_count": 10, "metadata": {}, "output_type": "execute_result" } @@ -377,7 +369,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 11, "metadata": {}, "outputs": [ { @@ -386,7 +378,7 @@ "0.0013128198764010824" ] }, - "execution_count": 15, + "execution_count": 11, "metadata": {}, "output_type": "execute_result" } @@ -398,7 +390,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 12, "metadata": {}, "outputs": [ { @@ -407,7 +399,7 @@ "0.07228069619249913" ] }, - "execution_count": 16, + "execution_count": 12, "metadata": {}, "output_type": "execute_result" } @@ -428,14 +420,14 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 13, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "629.627208 seconds (2.89 M allocations: 493.650 MiB, 0.28% gc time)\n" + "609.049840 seconds (2.89 M allocations: 493.832 MiB, 0.36% gc time)\n" ] }, { @@ -470,7 +462,7 @@ " -0.00277687 0.00341591 -0.000337529 0.000481492 0.499799 " ] }, - "execution_count": 17, + "execution_count": 13, "metadata": {}, "output_type": "execute_result" } @@ -498,33 +490,99 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 15, "metadata": {}, "outputs": [ { - "ename": "SystemError", - "evalue": "SystemError: opening file cg10k_traits.txt: No such file or directory", - "output_type": "error", - "traceback": [ - "SystemError: opening file cg10k_traits.txt: No such file or directory", - "", - "Stacktrace:", - " [1] #systemerror#39(::Nothing, ::Function, ::String, ::Bool) at ./error.jl:106", - " [2] systemerror at ./error.jl:106 [inlined]", - " [3] #open#297(::Bool, ::Nothing, ::Nothing, ::Nothing, ::Nothing, ::Function, ::String) at ./iostream.jl:283", - " [4] #open at ./none:0 [inlined]", - " [5] open(::String, ::String) at ./iostream.jl:339", - " [6] #open#298(::Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}}, ::Function, ::getfield(DataFrames, Symbol(\"#_r#302\")){Bool,Char,Array{Char,1},Char,Array{String,1},Array{String,1},Array{String,1},Bool,Int64,Array{Symbol,1},Array{DataType,1},Bool,Char,Bool,Int64,Array{Int64,1},Bool,Symbol,Bool,Bool}, ::String, ::Vararg{String,N} where N) at ./iostream.jl:367", - " [7] open at ./iostream.jl:367 [inlined]", - " [8] #readtable#301(::Bool, ::Char, ::Array{Char,1}, ::Char, ::Array{String,1}, ::Array{String,1}, ::Array{String,1}, ::Bool, ::Int64, ::Array{Symbol,1}, ::Array{DataType,1}, ::Bool, ::Char, ::Bool, ::Int64, ::Array{Int64,1}, ::Bool, ::Symbol, ::Bool, ::Bool, ::typeof(readtable), ::String) at /Users/juhyun-kim/.julia/packages/DataFrames/lyCjP/src/deprecated.jl:1129", - " [9] (::getfield(DataFrames, Symbol(\"#kw##readtable\")))(::NamedTuple{(:separator, :names, :eltypes),Tuple{Char,Array{Symbol,1},Array{DataType,1}}}, ::typeof(readtable), ::String) at ./none:0", - " [10] top-level scope at In[19]:3" + "name": "stderr", + "output_type": "stream", + "text": [ + "┌ Info: Recompiling stale cache file /Users/juhyun-kim/.julia/compiled/v0.7/CSV/HHBkp.ji for CSV [336ed68f-0bac-5ca0-87d4-7b16caf5d00b]\n", + "└ @ Base loading.jl:1185\n" ] + }, + { + "data": { + "text/html": [ + "

6,670 rows × 13 columns

Trait1Trait2Trait3Trait4Trait5Trait6Trait7Trait8Trait9Trait10Trait11Trait12Trait13
Float64⍰Float64⍰Float64⍰Float64⍰Float64⍰Float64⍰Float64⍰Float64⍰Float64⍰Float64⍰Float64⍰Float64⍰Float64⍰
1-1.81573-0.946151.11363-2.098670.7444170.001391720.934732-1.226771.11608-0.4436280.824466-1.02853-0.394049
2-1.24440.109660.467119-1.621311.056680.9789471.000150.3248741.162322.692273.082641.090650.0256616
31.455671.538671.094030.586655-0.327965-0.303377-0.0334355-0.464463-0.33194-0.486839-1.10649-1.42016-0.687463
4-0.7688090.5134910.244263-1.31741.193941.173441.087370.5360230.8027590.2341590.394175-0.7673660.0635386
5-0.264415-0.34824-0.02390650.004739161.256191.203891.298010.3101140.626160.8992890.5499680.5406880.179675
6-1.37617-1.471920.29118-0.803111-0.26424-0.260573-0.165372-0.2192571.04702-0.09858160.9473930.5940150.245407
70.100942-0.191616-0.5674210.378571-0.246656-0.6088110.189081-1.27078-0.4524760.7025630.3326360.002691650.317117
8-0.3198181.357740.81869-1.155660.6344840.2914620.933324-0.7410830.647478-0.9708780.2208610.852512-0.225905
9-0.2883340.5660830.254958-0.6525780.6689220.9783090.1228631.479090.06721320.07959040.1675320.2469160.539933
10-1.1576-0.781199-0.595808-1.005550.7898290.5710580.951304-0.2959630.990420.5613090.7331-1.73468-1.35278
110.7405691.408740.734690.0208323-0.337441-0.458304-0.142583-0.580392-0.684685-0.00785381-0.712244-0.313346-0.345419
12-0.6758920.2798930.267916-1.041040.9107420.8660281.074140.03817510.766355-0.340118-0.8090140.548522-0.0201829
13-0.79541-0.699990.39913-0.5104761.515521.287431.537720.1339891.020260.499019-0.369483-1.10153-0.598132
14-0.193483-0.286021-0.6914940.01315821.523371.401061.531150.3330661.043720.163207-0.422884-0.383528-0.489222
150.1512462.091852.038-1.124751.665571.625361.587510.6358520.8425780.450762-1.39479-0.5609840.28935
16-0.4646090.3612771.23277-0.8260341.434751.744520.2110972.648161.025110.1197570.0596832-0.631232-0.207879
17-0.732977-0.5262230.616579-0.554480.9474850.9368330.9725170.2902511.012850.516207-0.03006890.8787320.450255
18-0.1673260.1753270.287468-0.4026530.5511820.5222050.4368380.2995650.58311-0.704416-0.73081-1.95141-0.933505
191.411591.787220.8439760.481278-0.0887674-0.4995780.304196-1.23884-0.153476-0.8704860.0955473-0.983708-0.356345
20-1.42997-0.4901470.27273-1.61030.9907880.7116881.18858-0.3712291.24703-0.03891620.8834962.589883.3354
21-0.1472470.1232840.617549-0.1871310.2564380.177950.412612-0.2448090.09476250.723017-0.6839480.0873751-0.26221
22-0.187113-0.270777-1.015570.06028510.272420.869133-0.6575192.32389-0.9999361.446720.971158-0.358748-0.439658
23-1.82434-0.933481.29474-1.945450.3358470.3592020.513653-0.07319771.571391.533291.820772.22741.50063
24-2.29344-2.491620.40384-2.364881.410531.422441.170240.8447671.790270.648182-0.0857231-1.02790.491288
25-0.4341360.7408820.699576-1.024060.7595290.9566560.63330.7707340.8249891.842871.91046-0.5023170.13267
26-2.1921-2.494660.354855-1.931560.9419790.9789170.894860.4632391.125371.705280.7177930.6458880.783968
27-1.46602-1.249220.307978-1.550970.6189080.6625080.4759570.4847190.4015650.55988-0.376938-0.9339830.390013
28-1.83318-1.532692.55674-1.518280.789410.9087480.6499720.6683741.200580.2779631.25053.31372.22036
29-0.7845470.2765833.01105-1.119790.9208240.7502181.26154-0.4033640.400667-0.217598-0.72467-0.391945-0.650024
300.4644561.33264-1.2306-0.3579761.18251.54316-0.603393.383090.823741-0.129951-0.65798-0.499535-0.414477
⋮⋮⋮⋮⋮⋮⋮⋮⋮⋮⋮⋮⋮⋮
" + ], + "text/latex": [ + "\\begin{tabular}{r|ccccccccccccc}\n", + "\t& Trait1 & Trait2 & Trait3 & Trait4 & Trait5 & Trait6 & Trait7 & Trait8 & Trait9 & Trait10 & Trait11 & Trait12 & Trait13\\\\\n", + "\t\\hline\n", + "\t& Float64⍰ & Float64⍰ & Float64⍰ & Float64⍰ & Float64⍰ & Float64⍰ & Float64⍰ & Float64⍰ & Float64⍰ & Float64⍰ & Float64⍰ & Float64⍰ & Float64⍰\\\\\n", + "\t\\hline\n", + "\t1 & -1.81573 & -0.94615 & 1.11363 & -2.09867 & 0.744417 & 0.00139172 & 0.934732 & -1.22677 & 1.11608 & -0.443628 & 0.824466 & -1.02853 & -0.394049 \\\\\n", + "\t2 & -1.2444 & 0.10966 & 0.467119 & -1.62131 & 1.05668 & 0.978947 & 1.00015 & 0.324874 & 1.16232 & 2.69227 & 3.08264 & 1.09065 & 0.0256616 \\\\\n", + "\t3 & 1.45567 & 1.53867 & 1.09403 & 0.586655 & -0.327965 & -0.303377 & -0.0334355 & -0.464463 & -0.33194 & -0.486839 & -1.10649 & -1.42016 & -0.687463 \\\\\n", + "\t4 & -0.768809 & 0.513491 & 0.244263 & -1.3174 & 1.19394 & 1.17344 & 1.08737 & 0.536023 & 0.802759 & 0.234159 & 0.394175 & -0.767366 & 0.0635386 \\\\\n", + "\t5 & -0.264415 & -0.34824 & -0.0239065 & 0.00473916 & 1.25619 & 1.20389 & 1.29801 & 0.310114 & 0.62616 & 0.899289 & 0.549968 & 0.540688 & 0.179675 \\\\\n", + "\t6 & -1.37617 & -1.47192 & 0.29118 & -0.803111 & -0.26424 & -0.260573 & -0.165372 & -0.219257 & 1.04702 & -0.0985816 & 0.947393 & 0.594015 & 0.245407 \\\\\n", + "\t7 & 0.100942 & -0.191616 & -0.567421 & 0.378571 & -0.246656 & -0.608811 & 0.189081 & -1.27078 & -0.452476 & 0.702563 & 0.332636 & 0.00269165 & 0.317117 \\\\\n", + "\t8 & -0.319818 & 1.35774 & 0.81869 & -1.15566 & 0.634484 & 0.291462 & 0.933324 & -0.741083 & 0.647478 & -0.970878 & 0.220861 & 0.852512 & -0.225905 \\\\\n", + "\t9 & -0.288334 & 0.566083 & 0.254958 & -0.652578 & 0.668922 & 0.978309 & 0.122863 & 1.47909 & 0.0672132 & 0.0795904 & 0.167532 & 0.246916 & 0.539933 \\\\\n", + "\t10 & -1.1576 & -0.781199 & -0.595808 & -1.00555 & 0.789829 & 0.571058 & 0.951304 & -0.295963 & 0.99042 & 0.561309 & 0.7331 & -1.73468 & -1.35278 \\\\\n", + "\t11 & 0.740569 & 1.40874 & 0.73469 & 0.0208323 & -0.337441 & -0.458304 & -0.142583 & -0.580392 & -0.684685 & -0.00785381 & -0.712244 & -0.313346 & -0.345419 \\\\\n", + "\t12 & -0.675892 & 0.279893 & 0.267916 & -1.04104 & 0.910742 & 0.866028 & 1.07414 & 0.0381751 & 0.766355 & -0.340118 & -0.809014 & 0.548522 & -0.0201829 \\\\\n", + "\t13 & -0.79541 & -0.69999 & 0.39913 & -0.510476 & 1.51552 & 1.28743 & 1.53772 & 0.133989 & 1.02026 & 0.499019 & -0.369483 & -1.10153 & -0.598132 \\\\\n", + "\t14 & -0.193483 & -0.286021 & -0.691494 & 0.0131582 & 1.52337 & 1.40106 & 1.53115 & 0.333066 & 1.04372 & 0.163207 & -0.422884 & -0.383528 & -0.489222 \\\\\n", + "\t15 & 0.151246 & 2.09185 & 2.038 & -1.12475 & 1.66557 & 1.62536 & 1.58751 & 0.635852 & 0.842578 & 0.450762 & -1.39479 & -0.560984 & 0.28935 \\\\\n", + "\t16 & -0.464609 & 0.361277 & 1.23277 & -0.826034 & 1.43475 & 1.74452 & 0.211097 & 2.64816 & 1.02511 & 0.119757 & 0.0596832 & -0.631232 & -0.207879 \\\\\n", + "\t17 & -0.732977 & -0.526223 & 0.616579 & -0.55448 & 0.947485 & 0.936833 & 0.972517 & 0.290251 & 1.01285 & 0.516207 & -0.0300689 & 0.878732 & 0.450255 \\\\\n", + "\t18 & -0.167326 & 0.175327 & 0.287468 & -0.402653 & 0.551182 & 0.522205 & 0.436838 & 0.299565 & 0.58311 & -0.704416 & -0.73081 & -1.95141 & -0.933505 \\\\\n", + "\t19 & 1.41159 & 1.78722 & 0.843976 & 0.481278 & -0.0887674 & -0.499578 & 0.304196 & -1.23884 & -0.153476 & -0.870486 & 0.0955473 & -0.983708 & -0.356345 \\\\\n", + "\t20 & -1.42997 & -0.490147 & 0.27273 & -1.6103 & 0.990788 & 0.711688 & 1.18858 & -0.371229 & 1.24703 & -0.0389162 & 0.883496 & 2.58988 & 3.3354 \\\\\n", + "\t21 & -0.147247 & 0.123284 & 0.617549 & -0.187131 & 0.256438 & 0.17795 & 0.412612 & -0.244809 & 0.0947625 & 0.723017 & -0.683948 & 0.0873751 & -0.26221 \\\\\n", + "\t22 & -0.187113 & -0.270777 & -1.01557 & 0.0602851 & 0.27242 & 0.869133 & -0.657519 & 2.32389 & -0.999936 & 1.44672 & 0.971158 & -0.358748 & -0.439658 \\\\\n", + "\t23 & -1.82434 & -0.93348 & 1.29474 & -1.94545 & 0.335847 & 0.359202 & 0.513653 & -0.0731977 & 1.57139 & 1.53329 & 1.82077 & 2.2274 & 1.50063 \\\\\n", + "\t24 & -2.29344 & -2.49162 & 0.40384 & -2.36488 & 1.41053 & 1.42244 & 1.17024 & 0.844767 & 1.79027 & 0.648182 & -0.0857231 & -1.0279 & 0.491288 \\\\\n", + "\t25 & -0.434136 & 0.740882 & 0.699576 & -1.02406 & 0.759529 & 0.956656 & 0.6333 & 0.770734 & 0.824989 & 1.84287 & 1.91046 & -0.502317 & 0.13267 \\\\\n", + "\t26 & -2.1921 & -2.49466 & 0.354855 & -1.93156 & 0.941979 & 0.978917 & 0.89486 & 0.463239 & 1.12537 & 1.70528 & 0.717793 & 0.645888 & 0.783968 \\\\\n", + "\t27 & -1.46602 & -1.24922 & 0.307978 & -1.55097 & 0.618908 & 0.662508 & 0.475957 & 0.484719 & 0.401565 & 0.55988 & -0.376938 & -0.933983 & 0.390013 \\\\\n", + "\t28 & -1.83318 & -1.53269 & 2.55674 & -1.51828 & 0.78941 & 0.908748 & 0.649972 & 0.668374 & 1.20058 & 0.277963 & 1.2505 & 3.3137 & 2.22036 \\\\\n", + "\t29 & -0.784547 & 0.276583 & 3.01105 & -1.11979 & 0.920824 & 0.750218 & 1.26154 & -0.403364 & 0.400667 & -0.217598 & -0.72467 & -0.391945 & -0.650024 \\\\\n", + "\t30 & 0.464456 & 1.33264 & -1.2306 & -0.357976 & 1.1825 & 1.54316 & -0.60339 & 3.38309 & 0.823741 & -0.129951 & -0.65798 & -0.499535 & -0.414477 \\\\\n", + "\t$\\dots$ & $\\dots$ & $\\dots$ & $\\dots$ & $\\dots$ & $\\dots$ & $\\dots$ & $\\dots$ & $\\dots$ & $\\dots$ & $\\dots$ & $\\dots$ & $\\dots$ & $\\dots$ \\\\\n", + "\\end{tabular}\n" + ], + "text/plain": [ + "6670×13 DataFrame. Omitted printing of 8 columns\n", + "│ Row │ Trait1 │ Trait2 │ Trait3 │ Trait4 │ Trait5 │\n", + "│ │ \u001b[90mFloat64⍰\u001b[39m │ \u001b[90mFloat64⍰\u001b[39m │ \u001b[90mFloat64⍰\u001b[39m │ \u001b[90mFloat64⍰\u001b[39m │ \u001b[90mFloat64⍰\u001b[39m │\n", + "├──────┼───────────┼───────────┼────────────┼────────────┼────────────┤\n", + "│ 1 │ -1.81573 │ -0.94615 │ 1.11363 │ -2.09867 │ 0.744417 │\n", + "│ 2 │ -1.2444 │ 0.10966 │ 0.467119 │ -1.62131 │ 1.05668 │\n", + "│ 3 │ 1.45567 │ 1.53867 │ 1.09403 │ 0.586655 │ -0.327965 │\n", + "│ 4 │ -0.768809 │ 0.513491 │ 0.244263 │ -1.3174 │ 1.19394 │\n", + "│ 5 │ -0.264415 │ -0.34824 │ -0.0239065 │ 0.00473916 │ 1.25619 │\n", + "│ 6 │ -1.37617 │ -1.47192 │ 0.29118 │ -0.803111 │ -0.26424 │\n", + "│ 7 │ 0.100942 │ -0.191616 │ -0.567421 │ 0.378571 │ -0.246656 │\n", + "│ 8 │ -0.319818 │ 1.35774 │ 0.81869 │ -1.15566 │ 0.634484 │\n", + "│ 9 │ -0.288334 │ 0.566083 │ 0.254958 │ -0.652578 │ 0.668922 │\n", + "│ 10 │ -1.1576 │ -0.781199 │ -0.595808 │ -1.00555 │ 0.789829 │\n", + "⋮\n", + "│ 6660 │ -0.52427 │ 1.04173 │ 1.13749 │ -1.27262 │ -0.607382 │\n", + "│ 6661 │ 1.32516 │ 0.905899 │ 0.84261 │ 0.869143 │ -0.691461 │\n", + "│ 6662 │ -1.44368 │ -2.55708 │ -0.868193 │ 0.390457 │ -0.0331364 │\n", + "│ 6663 │ -1.8518 │ -1.25726 │ 1.81724 │ -2.17285 │ 0.948696 │\n", + "│ 6664 │ -0.810034 │ 0.0896703 │ 0.530939 │ -1.14212 │ 1.26049 │\n", + "│ 6665 │ -1.22395 │ -1.48953 │ -2.95847 │ -0.645307 │ -2.67022 │\n", + "│ 6666 │ -0.282847 │ -1.54129 │ -1.38819 │ 1.48327 │ -1.23406 │\n", + "│ 6667 │ 0.475008 │ 1.46697 │ 0.497403 │ -0.46861 │ 0.496553 │\n", + "│ 6668 │ -0.408154 │ -0.325323 │ 0.0850869 │ -0.182984 │ -0.46577 │\n", + "│ 6669 │ 0.886626 │ 0.487408 │ -0.0977307 │ 0.830857 │ -0.168597 │\n", + "│ 6670 │ -1.24394 │ 0.213697 │ 2.74965 │ -1.80285 │ 1.11401 │" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ "# Pkg.add(\"DataFrames\")\n", - "using DataFrames\n", + "# using DataFrames\n", + "using CSV \n", "\n", "cg10k_trait = readtable(\n", " \"cg10k_traits.txt\"; \n", @@ -540,22 +598,78 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true - }, - "outputs": [], + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "

13 rows × 8 columns

variablemeanminmedianmaxnuniquenmissingeltype
SymbolFloat64Float64Float64Float64NothingInt64DataType
1Trait10.00221138-3.204130.125013.47940Float64
2Trait20.00135253-3.511660.03351734.913420Float64
3Trait3-0.00129591-3.93844-0.0007821627.91630Float64
4Trait40.00230893-3.60840.2281653.127690Float64
5Trait5-0.00179039-4.148750.03103432.717180Float64
6Trait6-0.00119598-3.824790.0362422.589730Float64
7Trait7-0.00198906-4.272460.0698012.653780Float64
8Trait80.000614075-5.62549-0.03863015.80570Float64
9Trait9-0.00180965-5.381970.1065712.571940Float64
10Trait10-0.000437029-3.54851-0.09665076.537820Float64
11Trait11-0.000615918-3.26491-0.06804374.262410Float64
12Trait12-0.000588783-8.85191-0.14109913.21140Float64
13Trait13-0.000151238-5.5921-0.14102224.17440Float64
" + ], + "text/latex": [ + "\\begin{tabular}{r|cccccccc}\n", + "\t& variable & mean & min & median & max & nunique & nmissing & eltype\\\\\n", + "\t\\hline\n", + "\t& Symbol & Float64 & Float64 & Float64 & Float64 & Nothing & Int64 & DataType\\\\\n", + "\t\\hline\n", + "\t1 & Trait1 & 0.00221138 & -3.20413 & 0.12501 & 3.4794 & & 0 & Float64 \\\\\n", + "\t2 & Trait2 & 0.00135253 & -3.51166 & 0.0335173 & 4.91342 & & 0 & Float64 \\\\\n", + "\t3 & Trait3 & -0.00129591 & -3.93844 & -0.000782162 & 7.9163 & & 0 & Float64 \\\\\n", + "\t4 & Trait4 & 0.00230893 & -3.6084 & 0.228165 & 3.12769 & & 0 & Float64 \\\\\n", + "\t5 & Trait5 & -0.00179039 & -4.14875 & 0.0310343 & 2.71718 & & 0 & Float64 \\\\\n", + "\t6 & Trait6 & -0.00119598 & -3.82479 & 0.036242 & 2.58973 & & 0 & Float64 \\\\\n", + "\t7 & Trait7 & -0.00198906 & -4.27246 & 0.069801 & 2.65378 & & 0 & Float64 \\\\\n", + "\t8 & Trait8 & 0.000614075 & -5.62549 & -0.0386301 & 5.8057 & & 0 & Float64 \\\\\n", + "\t9 & Trait9 & -0.00180965 & -5.38197 & 0.106571 & 2.57194 & & 0 & Float64 \\\\\n", + "\t10 & Trait10 & -0.000437029 & -3.54851 & -0.0966507 & 6.53782 & & 0 & Float64 \\\\\n", + "\t11 & Trait11 & -0.000615918 & -3.26491 & -0.0680437 & 4.26241 & & 0 & Float64 \\\\\n", + "\t12 & Trait12 & -0.000588783 & -8.85191 & -0.141099 & 13.2114 & & 0 & Float64 \\\\\n", + "\t13 & Trait13 & -0.000151238 & -5.5921 & -0.141022 & 24.1744 & & 0 & Float64 \\\\\n", + "\\end{tabular}\n" + ], + "text/plain": [ + "13×8 DataFrame. Omitted printing of 2 columns\n", + "│ Row │ variable │ mean │ min │ median │ max │ nunique │\n", + "│ │ \u001b[90mSymbol\u001b[39m │ \u001b[90mFloat64\u001b[39m │ \u001b[90mFloat64\u001b[39m │ \u001b[90mFloat64\u001b[39m │ \u001b[90mFloat64\u001b[39m │ \u001b[90mNothing\u001b[39m │\n", + "├─────┼──────────┼──────────────┼──────────┼──────────────┼─────────┼─────────┤\n", + "│ 1 │ Trait1 │ 0.00221138 │ -3.20413 │ 0.12501 │ 3.4794 │ │\n", + "│ 2 │ Trait2 │ 0.00135253 │ -3.51166 │ 0.0335173 │ 4.91342 │ │\n", + "│ 3 │ Trait3 │ -0.00129591 │ -3.93844 │ -0.000782162 │ 7.9163 │ │\n", + "│ 4 │ Trait4 │ 0.00230893 │ -3.6084 │ 0.228165 │ 3.12769 │ │\n", + "│ 5 │ Trait5 │ -0.00179039 │ -4.14875 │ 0.0310343 │ 2.71718 │ │\n", + "│ 6 │ Trait6 │ -0.00119598 │ -3.82479 │ 0.036242 │ 2.58973 │ │\n", + "│ 7 │ Trait7 │ -0.00198906 │ -4.27246 │ 0.069801 │ 2.65378 │ │\n", + "│ 8 │ Trait8 │ 0.000614075 │ -5.62549 │ -0.0386301 │ 5.8057 │ │\n", + "│ 9 │ Trait9 │ -0.00180965 │ -5.38197 │ 0.106571 │ 2.57194 │ │\n", + "│ 10 │ Trait10 │ -0.000437029 │ -3.54851 │ -0.0966507 │ 6.53782 │ │\n", + "│ 11 │ Trait11 │ -0.000615918 │ -3.26491 │ -0.0680437 │ 4.26241 │ │\n", + "│ 12 │ Trait12 │ -0.000588783 │ -8.85191 │ -0.141099 │ 13.2114 │ │\n", + "│ 13 │ Trait13 │ -0.000151238 │ -5.5921 │ -0.141022 │ 24.1744 │ │" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "describe(cg10k_trait[:, 3:end])" ] }, { "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true - }, - "outputs": [], + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlgAAAH0CAIAAABuMsSDAAAABmJLR0QA/wD/AP+gvaeTAAAgAElEQVR4nOydeWAURfb4X/fck8mdyQUBBULCIQIiqyEgh6IiEF1RUVjQVVYEdgFRyQIGD+SUa/2xX5dL1FXDIRhcBbkRiHiAHAmQcEguyDn3Pd1dvz8axjgzmcz0TKZnJvX5AyZvuqte95uq19VV9R6BEAIMBoPBYNorJN8KYDAYDAbDJ9gRYjAYDKZdgx0hBoPBYNo12BFiMBgMpl2DHSEGg8Fg2jXYEWIwGAymXYMdIQaDwWDaNdgRYjAYDKZdgx0hBhOuUBQ1bdo0pVI5aNCgmpoaVqhWq8eMGZOQkDB27Fi1Wu1BiMFgWLg7QtwIMRh+WbNmjU6nq6ioyMnJWbhwIStctmxZ586db9682alTp+XLl3sQYjAYFoJziLX333//zJkz69evX7hwoVqt3rhxIwDk5+cbDIaVK1fOmTMnOjp6yZIlLQkdlJeXZ2RkyGQyDjrQNE0QBElyd+eRUQK/lJeXp6eny+VyXy+BoiiBQEAQRMieZbfbRSKRT6cEk/79+3/00Ud33323Xq8vLy+/5557ACArK6uoqCg7O/vSpUt5eXllZWUtCR0cO3bs+vXr48ePb7VGhmEQQgKBwEsNfT0eIUTTtFAoDP7xAoEgrNtgRkaGVCr16W644uv9DM0SOMDdEQaqEfbv33/jxo39+/fnoINGo5FIJNycKItWqxWJRHK5nHMJOp1OIBBERUX5U4JYLJZKpZxL4Jf+/fuvXr164MCBvhpCrVbLZDJfL7ypqSk6OlosFvt0VkNDQ3x8vK+tq66uTqlUhmz/mJiYOGXKlPXr13fp0uWjjz666667AEChUDQ0NMhkMrPZnJKSotPpWhI6yMrKSk9P37lzZ6s1ms1mu90eExPjpYYWi8VqtcbGxnp5vNVqNZlM8fHxXh5vs9kMBkNCQoKXx1MUpdFokpKSXL9SKBSh/NDjGbYX7d27t16vT0xM5FyO3W7XarVu70/QSqAoSq1WK5VKziVwgLvXraio+OKLL4YNG8Y2QlZYU1PTuXNnAGDfw3gQOtDpdCNGjLh06RIHHbRarUQi8ceF6HQ6kUgkk8nOnTtntVqjoqJ69uzpUwl6vV4gEPjjSvV6vVKpDF9HGGR0Ol1JSYlIJLr//vt9HeFFGDqdDiFUWlq6bt26KVOmnDx5EgAQQuxtYZ+s2SPdCh1ERUWtXLnSG/cjkUjsdruXjs1qtZ45cyY6OvqOO+7w8oqsVqtYLPbJEQqFQu+PpygKALw/HuMnGo3mwoULANClS5fU1FS+1WkR7o4wUI0wOjr6ww8/5Ob/RSKRnyNCsVgsEom+//77p154hYyKTZOTl86d9qkEiUTi54hQIpFIJBLOp7c3Xn71nz/8WmKsvGCxWHwdF0YYSqVy1qxZaWlpM2bMWLNmDStMT0+vqqrKzMysqanp0KGDB2Fb8+aiZatXr35wxPA9RV8Gp0ZMqPHMi9OKT5+ndI3vv7Ng+vTpfKvTItzf+TRvhCUlJayQbW8A4NoInYQOCIJISEggeaWmpgZ6PWiYuIlCBC8KcLZCO6Sy4rph/DqCFGzdunXv3r18q8MnDz/88JYtW6xW6/r16wcMGMAKx4wZs3nzZoTQ5s2b8/LyPAjbmqtXr1J3/onGed7aMRXXrhqe+xD6P863Iq3AvQsO8UaIiWwYgXjq+59Of3Uu34rwyZIlSw4dOpSSknLw4EF2tRoAFBQUnDt3LiMjo7S0dMGCBR6EGAyGhfur0SVLlvzlL39ZtmzZ3XffvWnTJlZYUFAwYcKEjIyM/v37f/rppx6EGIxf2C2mMW/TX07lWw8+SU1N3b9/v5MwLi7um2++8UaIwWBYuDtC3AgxQeZ86YXJL8+oqKzkWxEMBhNR4NmpZlj0Nxua0u7s3rlrd4Zh+NYG48y/P1z/qyjLRjmvt8JgMBh/wI6wGZobNoaofemrqutXOW+vDCMGDRpE3Gbq1FvvGEM9NlBKNyC83Z2NwWAw3oAd4R8RiiElk28lggFC6NKlS9XV1Xq9Xq/XOxbf4wBdGAwm4NA0bbPZXLfPhQjYEbZT6urqbDZbXl5eWlraxIkTHaFGdu3aNWPGDIlEMmPGDEeoEbfCYFJZWdl74OBPv9ga/KoxflJ3o2b58uV/f2N+5+w+//73v/lWB8MDdP31WbNnS2XyyS9O4VsX9wQ1nlsIYjab//Of/5w6dYqmuYeGCUdqa2vvvffeVatWderUafbs2TNnzvziiy+AU2wgs9n873//u0ePHmzYDu8xGAwURdnt9laP3L9//290jEXmHLcJIUav13tZl1Ao9DXEmsFgkEql3mz0lMlkQY6OCACDBg0qLi5mP7/88ssffvghAKjV6kmTJp04cSI3N/fjjz9mo6i4FbYper3eaDKCvr6kpi7/rfeAFKK07BsuPx5Me4DR1qLxq0Eks9M/8K2Le7iPCMNyhsmFuW++PX/Lnh3HztE+duLhTt++fQ8dOtS3b9+EhISlS5d+9913rJxDbCCCIBQKBdGWAAApiyFEf4y/Y9LcqK1PSkmb+eprbVq7lxoGmRB/uT1s9JP79h8AhiaUXRBlI0QSkEYHoV4MhgMcn2EdjZCNOuh4Fmbb244dO+bMmbN8+XI20YRbIe/877sD6zd9dPr0r9T9M+BqcXt7R3z69GmLxZKTkwMAYrHYEeONQ4AuqVQ6adIkhULha6w7u93uZdBtqVTqxtnoGxmxnHpsvtl+TqFQeC7BbDYrFApfB21Go1GhUIRm6B/Hy+2ysrIRI0asX7+evZO7du0qKipi32Pn5eWxzc2t0IHNZjt48KA3UXYtFovdbvcmImDZ+V9JRRINAPD7ujOKoiwWi4ezrFar1Wr1fExzbDabT8dTFNXS8SKRyPssGZgIg2MLD+AME8MwNTU1dNCZ9dobX9uzbuqcmsStRnv16tUrV66YTKbgKBP8RapGo/GJJ564ePGizWZ79913H3/8VgykcIsNRIKwnYZpZV9ub9y4saKiIjY2dubMmaycw8ttu92+e/dusxdYLBaLxeLNkQiguQtkP9vtlDfn8gLeMdWe4TgiDOAMk8FgePnll8+cOcNBDX+yT9A0A70fJk7dCgf8hyYrie4/5CGrqu7r3V+1mh+KzT7h/TOp2xKSkpL8CR3Ogdzc3IULF44ZM0ar1Y4aNeqDDz5g5Tg2ULjAvtxmPy9durRXr17sZ27ZJ9auXRvY7BMEQQCwg/jf/5XJpJ5raW/ZJyiK+sc//rF9+/bu3btv27aNfd0SIrO87QqOjjCAjTAmJmbbtm3Jyckc1GDf6XFzIbdeg9x+3/aH/y0645ra2FVD4uPjW1VMKpX6mX1CKpUGP4sCQRDTpk2bNm2akzwcYwPdrKn+8ssv+/fvf+edd/KtS/AI4MvtgHP48GHKbm/3S/FaZ82aNTqdrqKiYuHChQsXLmQDxobRBFPEwPHV6OnTpx3L1VwbIfiSfSJkYWyW/6xfX1BQ8NNPP/GtC6ZlKn89fubixOmv7du3j29VgkrIvtwuKSkZM36yxWr943sWjBs+//zz119/XS6XFxQUvPLKK6wwNLcwRTYcn9mMRuO4ceOOHDnStWtX10a4ePFi10boJAx9jKr6j66QwqN7U1JSBg4cyLc6mBawGqH7YFIa1BfLoUDIvty2Wq3CmCTQNLVpLZFBoNKb6/X6p59++vvvvzcYDP4sOLDb7V7uR/KyBPYVoEMhq83a2NjouQSKorRaLYeV2DExMZxfrXF0hCHbCANM7vPCn/7LtxIYjBsi6eV2uyVQ6c3lcnl+fn5sbCxJkt7M4LYEu6k3ICVcvXq1vLzcbDbD7xNPIBKJWi2coiiGYTjo4M+iX4KXoJqPPPJIXV0d+7mmpiYpKYlbinaGYTjv4qqqqaHkSWBoBGk02K0AAGIZmDQQ3wEar0PSHaCqgphkwmpIkItjYmI86AAAjhX2WVlZhYWFPmmi0+nEYjG3JT984WTBxMREiUTiqyG8NB9N00ajUWW0IMoOEgWYtRCdDNqbEK0EkxqEUgAgSCIxShId3eJONZqmSZL0RsPmFqyrq1MqlaG5fcJPuLVBhBBCyPMNsVqttQ2NDGUHUgAECQQJdiuQJAjFcXKJ51Ue3pTf6vEe2iBFUWq1WqlUell+W5Oenn7q1Km0tLTa2tpu3boZDAYAyMzM/PbbbzMzMy9fvvzYY4+Vl5e7FbpaUCwW+3T3XEEIMQzjj0dxlFBRWQUCEWO3QmwKGJpAGg1ARIElubWbjxDq0qVLkF//8jOdXVJSsmnTJm4LZEKZ8vLyxYsX861FMMAWDHewBUMBNr35q6++6prevNUJpgi24LvvvhvkSnlb19WrV6+OHTvyVXsbEZFDh5bAFgx3sAV5x8/05tiCgQIvcG7vlJSU/OlPfzIajeyfeA8TBhM0cHrzECGcnp4wAUer1T7//PMmk8khCZFIlRgMBhM0sCNsvyCEnn/++fz8/OZCvIcJABjf4VHbkpKS5vEcwi7wPQbDO/46QtwIw5dly5Z17dp13LhxzYUc9jBFGDRNN9ymqampwTusVisv2uIxPQbjP345QtwIw5fDhw/v3bvXNVAThz1MEYZAIEi5jVKpTPEObvt//ASP6d2CELK2DJuwwhUcdLs949eOE9wIXUEIWXwn+N7l4MGDR48eFYvFrIcjCOL48eMQWUHyIh48pncLQsjoO76mlcZEEtxXjQa2ET7+1Piq2gbOyviEmISvv9yalOSc7jwgIITYYAreY7FYgpx6AgAWLVq0aNEi9jNB/B5XgXOQvIixYLjAjuld1xxyHtNHjAVJkkxISHD7FUVRBEG09G24EzEWDD4cHWHAG+HRY8c1ectBkchNH5+QbproT9Ykz5Ak6evuAoFAIBKJ2kgfX+EcJC9iLBguOMb07J8EQRw7diw3N5dz9glswXAHW5AzHB1hwBshAEC3HIgPxms3UtxOU7m2RPMwe37tYcIWDCIBH9MDYAuGP9iCnOA4R7ho0SJ0GwBACOXm5kIIpIAJODRNZ2dn860FhiNFRUW9e/eOi4sbMmQIG7Mx4ikoKDh37lxGRkZpaemCBQs8CMOCvXv39uzZMy4urmfPnu0t01Yk4bS/INQIcGSZCMs+sXbt2s8//7ysrIxvRTBcqKysnDhx4r59+/r27btu3boXXnjhxIkTfCvVJgRsTB9iMAwzYcKEbdu2DRs2bOfOnS+88EJNTQ3fSmF8xnV/QagRgA31ro2wurp69+7djjwaboWhxpQpU1avXs1+fvHFF1etWgUAffr0efPNN3nVC+Mtrha8du3a+PHj77//fplMNnnyZPxAE+K4WpCiqE8//XT48OFGo1EikcTFxfGrIcYzbntRt/sLQg0cWeYWTz755K5duwDAarUWFRWNHz8eAIYNGzZ69Gi+VcN4hasFhw4dumHDBgCgabqgoOCZZ57hW0eMJ1wtKBaLR40aZTQaY2Ji8vLyWGtiQha3vajb/QWhBnaEtxg+fPiFCxfq6ur27dvXr1+/9PR0vjXC+EZLFjxw4MC9994bGxu7du1afjXEeKYlCyoUCoPBsGjRopkzZ/KrIcYzrhZsKXBHqIEd4S3EYvFjjz22e/fuwsLCiRMn8q1OMHC7DCF8g+S5WhAh9M9//vOdd94pLCxcunSpUIhzrYQ0rha8fv3666+/DgBRUVEvvvjixYsX+daxTYiYQJWuFmwpcEeogR3h74wbN+6zzz7bv3//E088wbcubQ67DOGDDz5QqVTvvPPOCy+8wMrDOkiekwWLi4t37dq1e/fu9PR0g8HApv/GhDJOFkxPT9+0adPRo0cRQlu3bu3Xrx/fCgaeCAtU6WTBlvYXhBrcHWGEjScA4KGHHjp16tTw4cNjYmL41qXNaWkZQlgHyXOy4JEjR8rKyuLj46Nvw7eCASbi26BYLN61a9err76amJhYWFgYeXOEkReoMkx7UY6OMCLHE1KptGfPnq7vRZsvi40YWlqGENaRKp0sOH/+fPRHvCmEYRjtbXQ6ndY77HZ7W16Zez3bQxt84IEHTp06pVKpiouLI29Hb+RFi22pFwVfOlKGYep8x58MMBxnTRzjCYPB4DSeKCoqYh9b8vLy2DlSt0InYuIS7DtmE2I55yvxHqvezROx3W4/f/58VVXVyJEjg6BDiMAuQ1i7du3MmTN//vln8CNIXsRYkCAIR8Q7kUgkEolIsvXnRW+OCSy4DbYEm33C7VcURbHZJ1y/8tLQASTggSojxoIkSSqVSg5nca6RoyNkxxMGgyEmJqb5/CfnZ5lD3+z64YcfuCnjMy8/6hp19+uvv37llVc+/PBDR9w4bjAMo9frfTqFvY3+VMqB69evr1u3bsWKFewyhMWLF7NyzkHyIsaCBEHI5be6EplMJpfLg+/kvAG3wZZgs0+4/YqiKKPR6DbGvUKh8LNeXwl4oMqIsSAE/8kS+YfBYHjvvfcGDBjA/imXy81mM/tDlMvlLQk7dOhQVVXlZ9UhyJkzZ+666y69j9y4cUOv1wdZVavVGh8ff+TIEYZh1q5dm5uby8pnz56dn5/PMEx+fv5rr73WkjCCLdinTx/Hn7W1tTRN86iPN+A22BwnCzpht9vr6+uDqY+XNO+KcRvs1atXkCvl6HVbWtaMs9kRBKHwkaioqOCv7G9pGUKERaqMYHAbjFRwGww+HB1hS8uaIy/odgTjdhlC+AbJa2/gNhhJoIgIVBm+cHSEeDyBwfALboMYTKDg/kaOHU84CSMp8j0GE+LgNojBBIRQXA6HwWAwGEzQwI4Qg8FgMO0afsIQy2SygoIChUIBADabTSQSsXtFA4jdTjEMDQBACgExAAAkCQwNAjFQVhBKgLKCQAQMBYSghQNsIBDePgABMAAEAcjDFpnGxsbAXkXI4mRBoVAYqH0/FEXRiACGBoEQaDsIhEDTQJAACAABIXD5iv3lsF9RIBABbZdIJNxqb7cW9L8NWq1WAAIcZSD0+5/sZ4Df/yRIoYAUCAT+1OgWbMHgY6coBhFe9KK3D6BpEIqBsrptp7xYkEB8xA+7cOHCgQMH2M8rVqx49tlnO3bsyKEcs9ksFAodcUCa8/bi5ap+48mTnzOZg6H+CikUMspuxIUD6NG5sHUOjF9FfFWAHvgb+VMh03UQNP5GEsCk9iBK9qDH5sHWOfD0CmL32yj3r+SpHcydfwJVNYloJnt4wunPFs6f27wii8VCkqTDOyqVymeffdanq9DpdGKxWCqVcrgDfOFkwSeeeKJz585uDeEBk8nEhm5pLtz99dcH68Xk1WJm4Hg4uhFyJhKndqEOvUmrDkwaptM9xKVDaNBkOPj/4L4JxPk9KOlOkgDQ3GC6DiJK9qAhU+DIhx07dsxISx3/9O+RqwwGg5db45tbsK6uTqlUhuaGej/h1gZtNhtN064b0i0Wy7w3CxiBBAnFIFYQdiMyakl5NCOQkGIp01RFiGVIKAaJgrCZkFEtSu36zLAB9957r1M5bOQXR0CDVqFp2mKxNE/dAB7bIEVRarWaQ9SSEMTVgmlpaa53wyfc3k8PfL7jqx+PHQaBGPrleduLdriLOPs1Gj2f2Pb6yvdXuD4MMQwjk8lefvllzlfBhSDvW3SlX79+p06d4nauWq02mUxOwsOHD7/zzjtxaZ2h4GdBRm+Y9CEMHC8YPBn+uplM7Q7/sQAAbLCRMUpYcFLQqQ9MXAf3TxDmTIQpn5Ip3WCDDQgC/mMmY1Ng3nFB577w3FrImSy871lYeKpz1l1O1Wm1WoPBwE1/RwnsZucg89VXX/Xq1Ss2Nnbw4MFlZWWsUKVSjR49Oj4+fsyYMSqVyoPQQb9+/Y4cOeJqiFZRqVSuF/763Hx44l1BYga8ugeEEnilkIhLh6FThfc8LrjzHnjmfTIuDeYeAYKElz4hlV3g/omCnAnCTn1g4joyRgnzjgNBwHNrs/r9qXmx9fX1drvdVw3DYkO9/3jfBo1Go0ajcZVrtVqxPJqMS4OUbtDlPjI1E8RyQXIXyOgj6J4LIikZlwbJ3aDrfWRqdxDLZHeP/OSTT1zLsVgsTU1N3mtutVobGxu9Pz5kN9T7CWtBX++GKzabraGhwfvju/buDwtOCp170a6eetGphaTyTthgI4Uiq9XqWiYvNuL+qFtUVNS7d++4uLghQ4aUl5ezQt4j39M0PXzEgwt/smh1ujatKNyprKycOHHihg0bbt68OXbs2MgI2fw7nfvzrUEwCM02iMGEHRwdYch2o4gNVpT3FkEGfvohkrh27dr48ePvv/9+mUw2efLksrIyVs4hBQzDMDdv3qQDBMME6l09CpRK3oCCPsUQsm3QO5Ber8cZIjEhAsfFMo5uFAAmT568dOlSVs458j0myAwdOnTo0KEAQNN0QUHBM888w8o5hGw2GAyzZs3KycnxdZpTo9HIZDKnCXOLxQyggFtuBTX71y2I/cf1CJqmm5qaHH+q1WqGYXwNZadWq0mS9GaOMDY2lvMKHW4EsA3SNP3LL7/06tWr1UqtVqvdbnfN3uAxA44b81Gaxhn/mLn/8PeF//3YqRyWVjVhsdlsLWWTcEtIZZ/AhA4cHWFgu9EXXnjh8OHDHNTQarUSicRsNrN/njhxoqamxn2/2VJfilr4382BCABohlGpVM3ler1eIBD4kwpLr9fHx8fzsljmwIEDb7zxxsiRIxctWsRKkO8pYGJiYjZt2qRUKt0G9feASCSSyWSOCy8vL79/2EOahjoYveD24kOi2b9uubUW0fUIgUCYnJz8+3EEER8f76sjRAiF7GKZALZBs9m8bt06p6x4bjGbzXa73ek23rx585kXptopO+Ep5cAfWhZj1qIeI0x2yilNhNVqNZlM3j9S2Gw2NrO0l8eHVPYJzC+//CISiXr06MGufeURv7ZPBKQblclk06dP57bSyW63SyQS9md97ty5Z1+aTnS9z/07qpb6UqKV750OJQnCSVWapgUCgZ8rtYIfdBshNG/evBMnThQWFnbv3t0h55wCxn9KSkqojn2RrL6Nyo9IAtIGFQrFxo0bXbPquGIymex2e/NYl7/8enbSCy9e0QFCqIVG1OLTjFgkdqrUarVKJBJvNGFhtw14fzxFUQRBeH88pu1gRPJHJ//dcqP88P7vcnJy+FWGY/8bwG5UIBAMGDCA22slyW2AdSfxadpJm+Dkl9wuCgA8OMRbrZkgnFSVSCQCgcCf12ISiST4Y47i4uJdu3adPHlSKBSyUzXsQxkbnXnx4sWuIZudhG0CQQJPe6HCjhB5lFn6/uqLNU2CRC7bnzDhS0lJyfnz5w0G33Kv/gGLXjfn+9h/PRg4pbjDsf9lu9Hdu3enp6cbDAbHpDeOfB8uHDlypKysLD4+Pvo2rByHbA4XQqgNynAyBI6E77rf+wYPfXldUX1DhIQv4OgI22k3ajM1aPW9B+YOuD+Xoii+tfGL+fPnO+2kYeU4BUy40E7bYAQRvut+EUJGTZP++U8JAT+xyQIOx8uYP3/+/PnzXeURHvleXWO2WEuHLyLff4hhGL61iSiqq6srKioQvqte007bYAQRwHW/CCGtVsvchrNKrZZgNBqnvvrPK5fLOVfhDAKnGjlfhT8TTOHtz2vr6lb/6/8JBMIRQwd3SE8PQo2ESIq6DcLzWIGlqamp/6BhOp2WzugXmBJNmuuV1fGpHadN+et7774TmDIxmIASwHW/Op0uLy/vwoULBoPBH0dot9v1er2HTbFnz5796tu9pqfXwNH9DiFy+uBydrMDnDdEIYQ0Gk1DQ4NDQlGUVqvloHxcXBzntRrh7Qjnvvn27goGKKvNbn9p8kS+1cFwxGQymcwW658mCW+cD0yJ6hq7SG7903MNf9zrggkpbtZUffbZZxMmTOBbET4JyLrf2NjY7du3Jycny2SyxMREzsrY7XapVJqUlNRcaDQaN2/eDADV1dUMw4BYDj0ean4A4fTBZZjQ7ADnJcQESSQkJKSkpDgkFEWJxeIgx4MNb0cIANDrIVBVF+39/PDRY2YzDwHEMSGKQASyWADsCEMVXf2vNXV/mzaj3TrCEFn364EPNmz5cP2Gmzdv6mI6gbYWgCQT0mk/9kyHLKG4U9hnqs+r4rOu3fEw5fKghAl9Rj4xvlOnTiaziW9FMD6z7+ChM2fPcjyZoYnU7nwkvwkVQmjdbwusWr3mwl0v6mgBfe+zdMd+0HWgfeAET4GewpaIcIQAREwKKLvyrUX4QdN0dnZ2c0nwl27/eOwI/H1XRLauiGfc089eq6nlW4twJZTX/TY1NV2+fJmi7JDRB0ThlCGOG345wlDoRjGcWbt2bU5OjiPcNkvoL93GNIf3NkiKucdUAoTspPi+B0ZcvHgxcBqFDaG5hen8+fP79u3rec999wwbVV1T03YVhRTcHSG/3ahKpTp58qRarQlIae2TPn36vPnmm05CDtknQpn6utrjx483X5MWSYT9owxDU0ZtyW83uK0SxAScCxcu5D37/NOvL62vvKZ/6wIp8C3VdvjCfbFMnz59unbtOmbMmObCoGWfeHzii2cuXTU21EDGk/6X5isIiH/OXyAUCKZPeyUuLi74CgSEYcOGuQo5LN02Go0FBQWFhYUWi8UnBTQaDZuZ0yFpYd01l+wTAADV578pObb3wOE1S98dMWIEAPga01Wr1QqFQm/2JykUCpEo2L1GoNogTdPnzp276667Wq3RfhvnLzy92fb41pskhfIYiqLYMlss31d9WoCtyO3xAoEgNKOrBxO9Xk/EpmhfKYJp4dqtcYO7IwxUN6rRaO67776Kigqfar9WXqaf/JlgfWvrzRydbIvZJ5y2vjgf53YHDKKpVVeiJCc25Q7KycrKEggEcrncB+3/iF6vT05O5iX7hCsclm6LRKLBgwdLpVJfs09oNJrS0lLa07Ynb7NPuMekQf3yRHa9WCyWyWQymcxXR8helDf9Iy99aKDaoNlsXr58+WsWOJUAACAASURBVOjRo1utkc0+4fgTIdSKn/udlvO6IGQwGNhBIZtNQiDwNpmozWYzGAzem5WiKL1e7zbLRExMDM4+EXwYu23Hjh2nTp164IEH+vTpw5caAd4+waEbjYuLKywsTEtL86kitqkQrW5sdxzQYvYJp60vzse1uAPm4dekZXsTExNTU1P9zD4RFRUVOi2Qw9JtsVj80EMPSaVSX335gsXvf/XtPpPp90Q8nOMUeDqRIEQikUQikUqlvjpC9qzwGihwzj7htHvMLc2zT5SUlDAM08qzyO/ftRzOniDi4uLY2q1Wq1Qq9Sn7hEQi8X7nHEVRAoHAmyvFBAeTqv7/HbtOfHlgBUI8OsIAt3C2xwQA127USdic4CchwrREMJdu/3btmunPK3CYnsDCrQ36ypUrV3JHjjaZTV6PCDEYNyCGtj84i8gawq8aAXaEobMDBsONUFi6jfGH4LRBvV4P8jg/xvC/Y9Zrc3IGvf322/4XheFMY2PjY+Oe+8fr/7RYfZvpjwwCPBQrKCiYMGFCRkZG//79P/30Uw9CTIjgFFcwaCGbV61adePmDegZ2FJbpLa29vjx4xKJZNSoUUGqkifCrg1SJi3qhaPY88ae/YemzXxVp27UJvWilXeR1FG+NeIBfx1hkLtRhNDs+W8dO3a8tp7/POaIoa9cuQIAHTt29GeOsB2ybNmy+Rt2oca6YFTGUGfPnl2wZGUTKOiKs2ajH6lEQxJeHmWuXLnicZUTJmzYtOWT65l5cKVYkNgJMu6G8vboCMNpFQAA2Gy2D1YsOT3wDbud/3SAZq1q6ty3ho8au23bNr51CRsoilq0dMWOnV/RPR4kxL6tMuWG9cblD7btrbz+m/HPK/B0VkD48ssvn5v8V5PJHMAyjx8//tFHHwWwQIwPJHZq59mVw8wRArtStMfw1teLtj20xWR+fgsMfIZvRcKJ/fv3L163+ZfrjRCsKJPIZrY99BpBersiH9MqZWVldIfWNx36QMPVQ2evTn1tfkp6R40GR8kIEqd/PfPa3Pzz5wOU8sU/dDpdbW0tX6EVws8RYsKXL7Zt/8+GTRCXBvEd+dYFE0oggLg0Sqyw2Gk8WRg0nhj/l5WnzVdq+J9momuvvPXe0s7dsv827e+8KBA2jrC+vr5f7oj0O7vTdMi1E6vVqtfrfY2r0t5obGycNPn5onKdzeZtHJAAgxhKIBk5dtyChXiBInfWf/LFijUftEnRpADvpQkCCKFF768dOXZcQ2Mj5D5PymL41giQvoF+/F3bM6vtPPXuYeMIL1++fK1Oo3q5KNT2LTH1vxW89XZCUvLkF//Gty6hjlAig245vFVvt9Am7X7o+e3B9rgcwH8oinr2penzCxZqGHFbtUKEqqqqHAmJMG2BzWZbmP/a/g5PWSIxsyA3guEIAxX5nhBJIDEjgIoFBEZbx4xbRk1YV3Lh4mtz81etWs23RoHHHws2NDQsXLhw6KN5A4aMsNlsbaShtxAC6J4LAGq1+s/PTX7gkTGDHxr152eea2xs5FmxNiYgbbC8vPzbnVtpIEEoCax6DmwgGPnIY2P+/NTHH3/SRlWEKYHqRauqqsrKyoAg4Z4/E0TIDYRomjIYDNevX7906dKlS5dQsFYSBONG+Bn5vr6+/qG8p6bNfiOk3z1e/eFSo2Xlz7p5S1Z36Jp9Z2Z2SGvrI/5Y8C9/m/7Ot6VHjxyu6PtiSEz/6BtLL1xMSe/4VWnj8V/OHEfd9ny332g0tn5iOOOPBWmaLr98ZVTen/vnPEDRdJv2nshiYOLSjpRc//Kb7w4fPrxy5craWpzvECBA+UPKy8vvvm9I7qhxDBOSOcwbr3/zv/8pU9PvH/7IwJGP9+zZ02wO5MpkDwQjtlmrke8vXrzIRr5nGObIkSMajcZoNHbt2rWhSbVz9zdlF0vPNCFGoSTNl6CmFACg6iwCBLXliLJBUyWYtYhAoKoGuwWqzjc/AOw2UFeDSQN2MaiqgLJCzTn2AEAI6i4juxXUNWBUI5KEpkqgrFB1FgCg+hwgBHVXkN0KmhtgUiHGDo0ViLLdPuA8IAbqroDdApobYGwi4tIgJdP2s+nGn5eQm55ftmxZbW1tbGwsQig2NjY1NbWxsTE9PX3voaMMg4Qk/PtfayQSCQDY7fYQDzLXqgXLysr69esnFAoZhtmxY8elS5dMJlNWVlZpaenVq1ch6ymidD8SCAAh0NaC1QiEAPT1wNDQ+BsggMbrwNCgb0BmPVhNoK0Fhob6awAIGiuAtoOhEZm1yGYBzQ1ADNRfBQBQVSDaBiY1ALplR4Sg7jIA3PoxmLVIIADKBqpKQAjqrwIguP4zJY1l4jJIRSJBCiG+A0MKps6cI2CoJ5/IO3fuXEJCQmpqalpamsFgqKmpGTBgQG1tLRuiMzs7GyF0uPinLZ9+JhXA4X17HHHGQzx3ga9t8Ny5c3FxcXK5/NT5i198uau2qgJEUkFiZzCcR4gByopsRoKyAgCiKbCbwUICQoAYsNvAbgLKBoAQQ4HdgkjBra8oG9hMQFkBsWdZwCYAhIBhgLKC1QR2KwAAQ4Esbv/BQ//btYMUisxm8x133KFSqUpKSrp169bU1JSRkTFo0CBWVZ1Od/XqVYTQ3XffbTAYGhoaDAZD//79GYZBCJEkSRAEQohhGIIgSJKkKEqr1Uokkps3byKExo4d62h6EWDBrKwsu91utVqPHz+u1+uvXbuWnp5+s7Z+/9HjNGX79VwpY7fqQUZN2gDLh7ddL4rsNo69aMNV6Hof1ftRYs9y/Qv/JVeMePb5KQatuusdnUQiUUbHDtnZ2Xa7vUOHDrW1tayxHnnkEUeIdn8sSARh7KlQKBoaGmQymdlsTklJ0el0zb/NzMwEgB9++AEArFbrgyMfphFhNhrkUrGNRoxQYtRpaCClErFer09ITKq7UZ3aIaOxvi4uIdGg04qlMqDtFAMyqUSn0ycqk2pr2APqY+MTTAadSMIegGQyqU6rTVQm11ZXpnbs1NRQHxMXb9TrRBIZwVB2BsllUp1Gm5icXFtTldIhQ9VQHxMXbzbohRIpwVA2CkXJZRqNOik5pbamKiU9Q93UoIiJMxt0ArFUAIyNoqOi5Bq1Oik5pa6mKiW9o6qxITo2zmLUS8QikiAsFmtMbGx9Y1NSSqqYhEPffct2o3q9PjExMSaG/ynrlmjVggzDHD16VCqVWq3WoQ89YjSZjXqtTCq1A0nZ7VJ5lF6jjktMVDc2xMbFm0xmkYAAgdhmtcgVCp1aFZ+YpFE1RcfEWKxWAQGEUGK3mOXRMZqmhoQkpUatUiiirDaKBCQQS80mQ0xsnKqhLjEpWavVyOUyO42AoUUSqdloiImLb6q7mahM1un0MomYAgJRdrFMbtBp4xISVA318QmJBqNRIhQwpJC22aRRt3TT1NcmJibpjOb4xES7XiWTK0iRWK9VK+Qys42SRyloqzkuLtZqtZrsNE0Im1sQQj53ga9t8GZtbWxcvM1kICRRao1aLJIQBIEArGajUCwRkgQiBbSdomlKLJEQBDCIsFlMIrFEQBCIJGmKoilKLJEAAIPAbjWLxBKSIAhSQFF2mqLEYjFBEAwQNotZKBILSQBSSFF2irKLRWKSFDAAdotZKCATExPMeq1IHtPUUCeTR4lkUXIB6t+vr81mq6ioMBqNNpphKKprlzsrKiqEEpndZr37rt6VlZU0Td9xxx0SiUStVtfX18fHxycnJ9tstt9++81isdgZkMuk+/buiSQLHj161GAwREVFPTTyYQagpqYmOTlFo9UJpHLGbm2pFzXqtGKZHFE2igGpRKzT65OUSp96UZNBL5JIgabsDJJLJVqNNiklxade1GLUC8RSEtE2ipbL5Rq1SpmSWnejOjmtg6qpMTom1mrUS8WimJhou8UsFArNVps8KooE+C5QFkRtj1wuN5vNCCGj0SiXy52+7dev36lTp7iV/NFHH504ccIf3f773/8ePXrUnxIKCwsPHDjgTwk7duzYv3+/PyW0Na1a8K233iouLva12HXr1p05c8bXs95///3Lly/7etbbb79948YNX8+aO3euTqfz9awQJFBt8MiRI5999pn39f70008bN270/vjz589/8MEH3h9/5cqVFStWeH/8jRs33n77be+PDx28sWBZWdmqVav8qeXatWvLli3zp4SKioolS5b4U0JNTc0777zjTwkcCMYbOc9JfNLT0//xj39wy+d39erVmJgYpVLJWbdr165FRUWlpKRwLuH69etSqTQ1NZVzCRUVFU8//fSDDz7IuYS2plULbty48auvvvLVEGVlZUql0vucOyylpaU7duyIjo726ayzZ8/u37/f14yJp0+fPnnypDePmW+99VZODn8LYlsjUG2wrq7OZDJt2bLFy3obGxvVavXWrVu9PF6tVtfV1e3evdvL4/V6fWVl5b59+7w83mw2l5eXHz9+3PWrCLAgRVHV1dV79uzhXIvBYKioqDhw4ADnEoxG47Vr1w4dOsS5BJPJdOXKlWPHjvl6oj8WDMar0VdffVUikSxevHjevHkURa1YsaL5tw0NDb/++mtb6xDi9O3bNzk5mW8tWgRbsFWwBcMdbMFwxx8LBsMRajSaCRMmnD17lo18z2b1xIQR2ILhDrZguIMt2KYEwxFiMBgMBhOyhO5yYQwGg8FgggB2hBgMBoNp12BHiMFgMJh2Tdg7wqKiot69e8fFxQ0ZMqS8vNync/0P3+dP7Q5KSkoiI8E9hwvx9QZyMxlnM0WMafzB890bNGgQcZsXXnjBg3Vcbed9yVOnTnUr8Vw+4YLn8iMGf363EdCjciHI+xYDS0VFhUKhKC4uNplMK1asyMnJ8en0uXPnTp8+3WKxTJ8+PT8/P8i1s2g0mnvuuSfcDYE4XQiHG8jBZJzNFDGm8QfPd49hmISEhOrqar1er9fr58yZ48E6TrbzqWSz2ewq8Vw+QkjfjDfffHPu3Lkeyg/kXeMVf363EdCjciO8G/nhw4dfeukl9nN9fX1iYqJPp3fv3v3ixYsIoYsXL3bv3j3ItSOEGIZ5/PHHt2/fHu69LbcL4XADOZiMm5kixjR+4vnu3bx5U6FQ3HPPPQqFIi8vr2vXrh6s42Q7n0quq6tzlXguv/lX586dGzFihN1u91A+1zsUWvj5uw33HpUzEdLIKYqaOnXqtGnTfDorKirKZDIhhEwmU3R0dJBrRwgtWbJkzpw5CKFw7239vBDvb6A/JvPJTBFjmkDh9u79+uuvw4YN+/XXX5uamiZNmiQUCj1YpyXbeVPy+PHjXSVelm+1WgcOHFhaWuq5fM53JqQI1O82THtUzoRfI8/KynJ6qbt///5+/frNnTu3+ROfN3gO3+clnGs/dOjQAw88YLPZUBj2ts2t4NOF+Gk+zibzqZawNo3/cLPRjRs3CILwYB23tvOy5Pj4eM+SlspHCC1evHjGjBkertdtaeEC55botgSWMO1R/SG8GznDMPn5+YMHDy4rK+Nwerdu3crLyxFC5eXlmZmZQa59/vz5TvO1x44d41AO73C+EA43kIPJONQSMabxH89379SpU46o942NjQKBwIN1nGznU8mpqamuEs/ls0KKojp16uQapb3V0sIR/3+3Yd2j+kN4O8Ljx49nZWWp1WrHlLhPp8+ePTs/P5+9+6+99lqQa29OxAw7fLoQDjeQg8n8NFPEmIYbLd29w4cPI4S+//775OTkCxcuWK3WmTNn9unTx9U67JHIxXY+lTx16lRXiefyWeG+ffsGDRrU/IpaKr8t7h6PcPvdRkyP6ivh3cgXLVrkzyJYtVo9atSoDh06jBkzRqPRBLn25kRMb+vThXC4gRxM5qeZIsY03Gjp7rEfGIZZt25d165dk5KSJk2aVFlZ6WodxylOtvOpZK1W6yrxXD4rfO6555zyLrVUfpvdQn7g9ruNmB7VV3CsUQwGg8G0a8J+Qz0Gg8FgMP6AHSEGg8Fg2jXYEWIwGAymXYMdIQaDwWDaNdgRYjAYDKZdgx0hBoPBYNo12BFiMBgMpl2DHSEGg8Fg2jXYEWIwGAymXYMdIQaDwWDaNdgRYjAYDKZdgx0hBoPBYNo12BFiMBgMpl2DHSEGg8Fg2jXYEWIwGAymXYMdIQaDwWDaNdgRYjAYDKZdgx0hBoPBYNo12BFiMBgMpl2DHSEGg8Fg2jXYEWIwGAymXYMdIQaDwWDaNdwd4aBBg4jbTJ06lRWq1eoxY8YkJCSMHTtWrVZ7EDooLy83m81OQoqiEEKcdYuYEsICJwva7Xb/y6RpmmEYPwthGIamaf+VCcgVhTJu2yCLn79h/03g580PyA8p9PFgQfDdiL5aDSFEUZT3xwdBJV/h6AgRQpcuXaqurtbr9Xq9fs2aNax82bJlnTt3vnnzZqdOnZYvX+5B6GD8+PEXL150EqrVaj8vW6PR+GqbgJeg1Wojvg+FP1oQIdTU1OR/mTqdzmaz+VmIzWbT6XT+K9PU1BTZDzRu2yCLn79hq9Wq1+s5nw4AKpXKH09mMplMJpM/CoQFHiwIvndlvt40mqZdRzieaWuVfIWjI6yrq7PZbHl5eWlpaRMnTnR0N7t27ZoxY4ZEIpkxY8bOnTs9CB0ghHQ6HRNKVFZWHj9+/McffwyaYpzt5w+BGtOHIGazubi4+OTJk1euXOFbF4xf1NfXFxcXFxcXGwwGvnXBtMi1a9eKi4s9jEpDHCG302pra++9995Vq1Z16tRp9uzZM2fO/OKLLwCgpqamc+fOAMAOAdmD3Qod6HS6MWPGOHVYTU1NFEUJhRzVAwCVSmWz2UQiEYdz7x083CJLMleWfLn1i4EDB3LWQa1WW61Wb3SIi4uTSCScK+KAY0wfGxsLAI5bzQ7fd+zYMWfOnOXLly9ZsqQlYSjzxsJFmz/fDgRxX+/uB/d8zbc6GO7kjZ/URAkt1ZeOHT7oT2PEtB01NTWPPPGMUVV/7szp7OxsvtXhAkdP07dv30OHDrGfly5d2qtXL/YzQoggCPaD492mW6GD2NjY7du3p6SkNBeSJBkfH++PIxQIBLGxsb46wnPnzjU0NDRU/YY+LIl5f3BCQoJSqayrqwMAuVzO+gzvEQqF0dHRYrHYp7OCg2NMX1ZWNmLEiPXr10ulUgDYtWtXUVERO3zPy8tjfZ5bYShz9epV02MLQSCmrn3Cty4Yv6itum5+91Lsv0fzrQimRUwmE0WKpMmd+FaEOxw9zenTpy0WS05ODgCIxWLHaCY9Pb2qqiozM7OmpqZDhw4ehCHI5cuXhzwyllB2QfD7hND6zR/PnDULgBj96COrVy4Xi8Wpqak8KhkoAjim12g09957b3V1NQAghBoaGvyfUdNqtVKplPMo2WqxAABoak4cO0qQJAFQWVkpEAi4lVZfX88wDPsw1xLx8fHskwQG037QarWlpaURMAvL0REajcZx48YdOXKka9eu77777uOPP87Kx4wZs3nz5sWLF2/evDkvL8+DMAQxGAwgj9P8/Tv42+/970+//GLLexfUNUV71u85dLRDivLyhXM8KhkoAjimj4uL2759e1paGnsASZL+PytIpVKZTMbZtUjYE1XV0PV++PtumBqVmprK+QUDQRApKSmeHSEG0w75y9SZe4p2IsqGFIlRimi+1eEOx8Uyubm5CxcuHDNmTIcOHdRq9bJly1h5QUHBuXPnMjIySktLFyxY4EEYZmjriLseNf/9G3ukrMQ+ffp0cXEx+9l1TA8ArmN6J2FzQsdJzJq3sM99Q04Un/xdRODNsmGP2aAbNGSoRBa1bds2vnUJJBRFTZs2TalUDho0qKamhhWG0YK1365dpeI6Ml3uC/dl1Rz7CIIgpk2bduXKlYaGho8//jgmJoaVx8XFffPNN9XV1bt373bMqLkVhj4Wg3bw4CEffbSFb0XaBKPR+MQTT1y8eNFms7mO6RFCrmN6J2Fo8q+l754fstBIu3HMBoNh586dO3fuPHnypOu34UgEr/t1gjJqqX/8T9Tn4ePHj2/btu3ChQt8axQY1qxZo9PpKioqcnJyFi5cyAo5bELjmZB5DuYMflhuEcqkQ3OPCNK6861ImxDJY/rMXELoZoHSzPw3J85d8pd/Ll/w7tLgKxVwAriXNzwQSW11V//zTfGkOW9t376db20Cw+eff/7666/L5fKCgoJXXnmFFXLYhEbT9JkzZ+xB548DQURRVPB1cODPPjTuyzLbBWTEPiiwY/pp06Y5ydnhuzfCcEIaff+IR69evWZ+9C2wmYq/fTchLUNEQtVvV0NzTa83RPa6XwBQqVQ6na55V4tsVvtjr0FdOY9aBZaKioovvvhi2LBhXbp0+eijj1ghhwVrZrN51apVY8eOdVsLG9PA+yX0RqMRIeSNX2GarRhACOn1eq1W600VbaFSTEwM5+aMHaFvIIZhLR1G73gxyKz9pd9MwcW/AwA0XLMl3Gl+8RPBguywntgI4LpfvV4/bty4n376ybUWlUpltVo59y8Wi8VqtXKI0GQyme4b/rBOb7JYLQ4havZtY2OjN+UYDAaCICwWS6tH+tONcob19KWlpevWrZsyZQr70p7DgjWFQrFx48akpKSWKvJpL5lUKkUIRUe3vvhF0GwBGkGQ8fHxHnQIjkrcwI7QFwyNNTdvJnfoZDPq/OkdMDzQcwQhvLUgiBBJIaEjv+r4TwDX/crl8nnz5rl9tqMoyp+9sGKx2Gq1cnhqpGlaVV9nXVUH02IcQsdMlEQi8bJMkiQJglAoFK0eyXl3jT8olcpZs2alpaXNmDHD8XI7rDehhSkR++rPV06cOPHzzz+3EuDUqEayWNuaRlLIJWANpk0pKirasmVLGA/xfCSA634FAkFmZqYoxPBw7QKBIODVkXzMgzz88MNbtmyxWq3r168fMGAAKwz3BWvhCHaEAAD/+9//Rj45cfZ7a03m1l+hYEKQPXv2PDdtzt8/Ogzh/LbTJyJ13W+7YsmSJYcOHUpJSTl48ODGjRtZYZguWLNYLL3v6rNy5Uq+FeECfjUKAHDz5k3oOdzUbRi5+22+dcFwwWg0CjvdrfvLRvi+vcRUc6z71Wq1o0aN+uCDD1h5QUHBhAkTMjIy+vfv/+mnn3oQYngnNTV1//79TsIwXbBGmXREr4f8zNjDF/6OCEtKSqKiohx/RuQeJgwmBIngvbwXLlw4efIkQi0sEbQaP9n21UNjnly99l/B1QvTGmEbvMIvvbVa7fPPP9880FzE7mHCYDBBoaysLOfBx8bPfNNmayEPYsWpa1GZB8zpR3/4ObiqYSIW7o4QIfT888/n5+c3F3LYChqOIGlMl55339G955kzZ/jWxV/wmB4TUmi1WohO0v/1c08HpWVDXIfv9u2PSUwZ8chjwVINE7Fwd4TLli3r2rXruHHjmgs57GEyGo2zZs1S/xGNRqP2Dy9LuH79+szZrxZu20ZT7HrRFpdaNE9JgYzqmgn/baJEdXV1/uugVqvtPCWyx2N6TLjScI3qPEA/ccONBhXfqnCn/QTJC3E4OsLDhw/v3bvXNT4Fhz1MIpFo9OjRMp6Y99aifx29eqi8sZWNEwDNdjEBAEBatkAil0gkAVGDl6XbARzT22y2YGgcYNCZM2d+/fVX3K2EKyIpyOP5VoI74R4k78CBA40NDXxrERgIbsE1FixY8N577zWXHDt2LDc3NzMz89tvv83MzLx8+fJjjz1WXl4OAK7CRx55hM12CwA1NTVJSUlOmedommZ3wnK9Lm9LaGhsNCAx2K0AAGIZmDQQ3wEar0PSHaCqgphkMDSBNBrsVgAE4igwqSC+IzReh8TOpK42JSnBKVVQVlZWYWEh+7mpqSlkE/MCwNKlSxsbG99//32C+P1noFAoGhoaZDKZ2WxOSUnR6XRuhc0tWFVVRdN0RkYG+ydN0/7vTWbz/3n/AzCZTCq9iYlOvmU7dQ3I44CyAW2DmBT3wqZKUiBENJWQEN/85XCXLl3+7//+r3nh9fX1SqUywvIRttoGWXw1hBMIITYzl/enWK3W2oYmRqEETQ0kdobGCkjMAHUNxCSDvhHkcWDRg0gKDA2IAWmMyKzq2CG9eQnN26Ber/dyQ33wqa2tzczMzMrKcgTJS05OBoCsrKyioqLs7OxLly6xIfTcCr20IPjenbKRzDxbjWGYispKIEggSCAFQNsBIRBJExRSb1ZjeVapuQVZDAZDKEaWWbRo0aJFi9jPzbtRL/MRlpSUbNq0ibV6JFFeXr548WK+tfAKdkzvunTbyzF9ZFuQza3oICLzEUa2BfnWwiv8DJIXwRZctGiRSvWHN95srFHPU0gKhSJUYo16v4epV69eHTuGfZgrJ3h5w8mNgwcPHj161PG7IQiCHdN7H94JWzBEKCkp+dOf/mQ0Gtk/1Wr1pEmTTpw4kZub+/HHH8fHx7ckxBbkF/+D5EWqBQmCaP6SBm6/WnASOsE58zYEJLJM85erEbCHqZ2waNEidBsAQAjl5uYCjksSbuDlTuFLYJNjRxIEQUh8x59noLB5esIEhzAN79Q+ac9bmDxD+07wU5HgIHmhAw6xhnEzpnc6IPTDO7VPArWFKcKgabqpqYn9zKZhslqtrZ4VGxvb0mKTNgIHyQsdsCPEYMISP5c7RTACgcCxhEQmk4XsqtH2lRw7tMGvRjGYsMSx3In1cARBHD9+HPAMEwbjO9gRYjBhCV7uhMEEilB5Nfr4U+OraoMUpEBMwtdfbk1KSgpOde0EbMEQgfMME7ZguIMtyJlQcYRHjx3X5C0HRWIQ6pJummix4AS8AQZbkEcCstwJWzDcwRbkTKg4QgCAbjkQH4zZC1Ic1LVh7QhswXAHW5APOIdEcAO2ICe4zxHu3bu3Z8+ecXFxPXv23LdvHyuMsLjpbmPDY8IIiqKmTZumVCoHDRpUU1PDtzoYnyFc4FujABPxIRGOHj3at2/f6OjoDyNpVgAAIABJREFUvn37fv/993yr4x6OjpBhmAkTJnzwwQcqleqdd9554YUXWHkkmbCl2PARQ3t4lFmzZo1Op6uoqMjJyVm4cCHf6mB8Rt+MN998c+7cuXxrFEjaQ0iEiRMnzp8/X6VSzZs3b+LEid6cwoYV9RU2Vjg3ODpCiqI+/fTT4cOHG41GiUQSFxfHysPXhFOmTFm9ejX7+cUXX1y1alVdXZ3NZsvLy0tLS5s4cSKbh6FVEEKW21itVot3BH9rV+Q9yrhaEAA+//zz119/XS6XFxQUvPLKK7wqiGkFtxZU3Oa3334rLi52xPqPDCIsJIJbC8bExGi1WoPBoNfrvdzQyTCM9o/odDqdTqf1CEVRnDXn6AjFYvGoUaOMRmNMTExeXt6GDRtYefia8Mknn9y1axcAWK3WoqKi8ePHs7HhN27cWFFRERsbO3PmTG/KQQiZfcefZxluRN6jjKsFAaCiouKLL75ISEgYNmyY95HpGd9pwwtrN7i1IIvNZnvppZf+9a9/+RNYOdQIYFbXEMGtBT/++OMpU6YkJCS89NJLn3zyiTflCASCpD+SmJiYmJiY5BF/Et759atSKBQGg2Ht2rUzZ878+eefIZxNOHz48IkTJ9bV1f3000/9+vVLT09PT093GxveMyRJOuaxGYYJ2XyE7KOMwWCIiYlx7MWGcH6UcbUgAOh0OoRQaWnpunXrpkyZcvLkyVbLoSiq4Y/pRpuamlpN5xYXFxfkAF2Rh1sLsqxcuXLgwIE9e/bkUb2A438GmFDDrQXnzp37xhtvzJo1a/Xq1fn5+QcOHOBbTTdwHBFev3799ddfB4CoqKgXX3zx4sWLrDx8o1qIxeLHHnts9+7dhYWF7IvslmLDRxLso8yiRYsc493wfZRxtSAAKJXKWbNmpaWlzZgxo6SkxJtyBAJByh9RKpUprcHLzyPCZnndWhAAaJr+8MMPvXwlE0ZEXkgEtxb88ccfZ8+enZaWNnfu3B9//JFfDVuCoyNMT0/ftGnT0aNHEUJbt27t168fKw9fEwLAuHHjPvvss/379z/xxBPQcmz4yCDyHmXAxYIA8PDDD2/ZssVqta5fv37AgAH8qhdYIm+WF9xZEAAOHTqUkZHRrVs3HhULJmGdAcbVgn369Nm0aZPBYPjkk0/uvvtuftVrCe5zhLt27Xr11VcTExMLCwsdc4RhbcKHHnro1KlTw4cPj4mJgWax4Tt06KBWq5ctW8a3goEkIh9lnCwIAEuWLDl06FBKSsrBgwc3btzIr3qBJfJmecGdBQFgy5YtI0eO9L4QhJDpNmaz2eQd/L7qiJisrq4W3Lx587fffpuWlrZjx46QbYPc5wgfeOCBU6dOOQnDOm66VCrt2bOnY0TfUmz4yMDxKPPbb79lZ2dv3ryZlYd1ChgnCwJAamqqa36GyCDyZnnBnQUB4LPPPvOpEHbxPfvZbrcTBOH40wMikUggEPhUEcYVVwtmZ2efOHGCR5W8IVSWYMXEJdh3zCbE8iDUZdW7mSOx2+3nz5+vqqry6dkzrAnsowy2IC8EcMFaxFiQJEnHmIld5RSaaZgCTsRYMPiEiiM89M2uH374IUiVvfxoQkKCk+zrr79+5ZVXPvzww9Bc5Bn6YAsGmevXr69bt27FihXsLO/ixYtZOec1h9iC4Q62IHcQH3To0KGqqoqXqtuUM2fO9OnTx/FnY2Oj1WrlUZ+2I4IteNdddzkJb968yTAML/p4wGq1xsfHHzlyhGGYtWvX5ubmsvLZs2fn5+czDJOfn//aa6+1JIxgCzZvgzqdTq/X86iPZ/bs2dOjR4/Y2NgePXp89913rFClUo0ePTo+Pn7MmDEqlaolYTuxIIter2e3QrUROB8hBhOWROSCtXZFRK77DVNC5dUoBoPxlchbsNaucKz7NRgMTut+i4qK2CW+eXl5bOgZt0JMoMCOMMDU1NzgWwUMBhMGROS634DAMIxKpWouMRqNqNliYLcoFArOc5PYEQYY1PohGAwGc4tIClQZKAiCiIqKai5hZ/KchE74E4eW+xxhUVFR79694+LihgwZUl5ezgrDN7wTBoPBBJOIjO4UEAiCkPgOSXJ3ZxzPrKysnDhx4oYNG27evDl27Fg8zRuO4EcZDIZHIjK6U5hCIMTlZd6RI0c+++wzdqFaQ0NDjx49GhsbASArK6uoqCg7O/vSpUt5eXllZWVuhZmZmYMHD2Z3udpsNpFIFITE0wghm90OAjFQVhBKgLKCQAQMBYQAEAMAQJLA0M0OsIFA6HwATYPQ7QEIADWpNYcO7Fc1NbI1NjU1hWz2icrKyl69eu3bt69v377r1q3btWsXG/0hPz/fYDCsXLlyzpw50dHR7Jy8q9CDBR3vcAKO3W5nCBIQAoRAILplBdoGpCAQQgYE4qa6G+Xl5efOnWteb21tbUpKSoTlRuelDTbHZrMhUggMBaQAGBoIEhAAICBIQDQIREDZbrVToRgo+6221trBer2+tLT07NmzbC16vT6UN9QfPXq0eXSn7OxsANBoNBMmTDh79iwbyIkNDuAq5MWCVqsVgAC2HoSafQYABA4FEOIchr6xsbG5BVkMBgNCKDo6mqvircDRETqgaXrGjBkkSa5btw4AFApFQ0ODTCYzm80pKSlsMltX4YULFxzJON555x2DwfDmm282L9ZoNMpkMn+GuiaTSSKRNI+ZVFNT89H2ry0jZsPWOTB+FfFVAXrgb+RPhUzXQdD4G0kAk9qDKNmDHpsHW+fA0yuI3W+j3L8Sp3agO/8EqmoSUUyHPsTZ3Wj0Atg6B55aTnz9Lhr0PHl6J3PHvaCtJe1mptfDcb98cun8r2yNarVaoVCIRKJWtQ1+Eh8/H2WaW3D58uWjR49m2zAA6PV6/3+vZrNZJBI5vfT/cNOWsth+0PgbaWhkBr8Iha/C+FXkt0uYux6FpkpSV8cMmQKFr8L4leS3y5jeD4OqmtTeZB74G2tQcs9yptdI0N4kVdXM0Jdh62vw1FLyu5VMjxGgbyAbf2OGTSO3z9244T+TJk1qXq83jrDVPE1tQVFR0fz586urq/v06bNx48bu3bsDgFqtnjRp0okTJ3Jzcz/++GM2KZirsLkFV6xY8eyzz3bs2NG1CqPRKJVKOcces9vtFEXJZDLXrxavWF2XPZb8eSvTYwRc+wmSu4LFQFp1THov4tJhNPgl2LMCnloK2+dC3kJizzKU+yL54+dMr5Fw9QdI7gZmHWk3MSlZxOXvUc5k2L8Gnlwi2DnvvXfe6tSp07PPPsvWEuKO0B+8tCD4bkSr1QoAbnukWbNmEYoERiAlJXKmsYKUyBihlJBFg64eaAqkCiSOIiirwKx++sknMjMzXTfde6OSUql0WJClrR2hXxvq9+/f369fv7lz59rtdlYil8vNZjNCyGg0yuVyD0IHffv2PXz4MP1HamtrrVYr7Qd1dXVs5ncHP/74Y2xmf/iPBQBgg42MUcKCk8JOfWDiOrh/gjBnIkz5lEzpChtsQBDwHzMZmwLzjgs694Xn1kLOZOF9z8LUQlJ5J2ywAUHC/xnIuDTIPyq4oz+MXwVDXhLeOw7ePtuhaw9HjfX19Waz2Rtt/bGCn1AUNXXq1GnTprF/RkVFmUwmhJDJZIqOjvYgdNC1a1elUllXV1dXV1dbW3v+/Pk6vykrK6usrHQSPvDIGJhaCA/+Q3D3KNhgY+0oSO0GL38OI2cL+zxyS7jeIkjrDn/7Lzw8R9B75G2DmgRpWfDSJ/DoG4JeI2CDDUgB/J9ekJ4Nf/0IHpsn6DEUNtgEYmlFRYVTvefPn6+trfWssMViCaLREEKooqJCoVAUFxebTKYVK1bk5OSw8rlz506fPt1isUyfPj0/P9+D0EG/fv1OnTrlthY/g0KYTCa1Wu32q+z+90H+98IOPeCvH0HWAzBmAQydKrzncXh+A5meDYsvgVgOG2wgksKaWjI2BeadELIWzBoCY9+EIVOE9z4Jf/k/smMveOccSBWwwSaOitFoNM1rCfEN9YHCgwURQg0NDTabzfvSPOxeJ0hSkNwFMu4W9BgGYrkgPh3SehDZQ4m4NCIqnkjtDt1yyIy7CHmcOD5106ZNQVApIHBcZoMQmjdv3okTJwoLC9nnUBYO4Z0IgoiJiXEa/JG34aae2xK8K83vh3ri94r8v4q25sCBA2+88cbIkSMXLVrESpDvK9ZiYmK2bduWnJwMt5+r2M/+IBKJZDKZVCptLhSLfk9h6hC6HYcRbj79/pEAN6cTt/9VKpVOIxiGYZKTk0Pt1ei1a9fGjx9///33A8DkyZOXLl3KykN/F9rFixd/+OEHrUbDow6YNgVRVjJ7EN9a+ABHR1hcXLxr166TJ08KhUKDwQAA7MsHdkZ38eLFrtO8TkIMvwTwUQbDC0OHDh06dCgA0DRdUFDwzDPPsHIOu9BMJlN+fv7WrVtda9FoNBRFcZ7ntlgs7ICyuXDIyFGmTveZ6+sB2Hkmb3E+lD33D1Kk0WgYhnH8bTAYvMw+4eUsBsZLaF3Ty9Nm7N6z76vthXzr0jocHeGRI0fKysrY6QcW9rcemkl8VCrVC9Nm3aypslgswa89NMGPMm5hxFEDH3hIICCXvTX/4Ycf5lud1gnImF4oFObm5rqdyZPJZDKZzJ8FXyRJOpVs1utMT60iL/T3dXmCy5DcdYxOSKXS5tVRFEUQhNtLc9XTR3UwnmAMjXTWMI3BzLciXsHREc6fP3/+/Pmu8tAM77Rt27a9F2ptyb1IOw77covwepQJGoxJWzJ4gezg6traWr51aYUAjunFYvHo0aOdXkSzSCQSqVTK2RGyDti55OYvmf154Uw0+xcAAGih5NEnn5WKBB+v/3dmZibczkfo9tJCAX+WO/GtuxcQYfNsETaK+olAeQd0vodvLUKI+fPnO00Xs/Kwzo4dGLrlCGJT+Faiddgx/e7du9PT0w0GAzush/a9C4026n7tO+Pc5etarZZvXVoH78YOHdqLI8RgIgzHmD76Nqy8vWefyBoilMfwrYRXOJY7yWSyyZMns1uVAGDXrl0zZsxgVzbt3LnTg9ABTdOlpaX2tqS8vDztzu7I+zlddgIXMW2qVXOazw37Co41igkDho564ucTR62IhAlPBaM+hj537ty3336bnJw8YMCAYNToO+E1PYFxJbDLnd577/+zd+bxURTZA3/dcx+5EwIJoBASWFQOxQsQAY9V5FRUdmURXV1ZZEUOFQUBEcP1U0FlZZVl3XW9UEBAVuRGLkUDBIKBADnIncncV/f0Ub8/GsY4SSbTPZM5MvX9I5/Jm66q1/2m+nXX8d4b999/f4sN2e12AAh8KZAQ4drHr/z888/O5B5QcbGpsE2vyLJci2/nIVHJh8TERBx0G9Nh4Xn+4M6t8HatfFmYFmTTNSXvbTz1j20Hb+qReXAX9h+YdiQky50SEhLWr1+fnp7eWitJSUmBex21Wo2a7V5PTEwkZb7+wt8ELwEAoFAoWtMqeJVCCHaEIYbn2NLSUgDo0aNHpHXpWOhSg1pYIQZEu5h753j0aVzxB+FpEROH4C1M0QOeIwwpdkO9wdR/6N29evVyu2Nj3TAGg4kI8bDcyWxsbD6jGYVgRxhSKDtSah1LzsuUUbpcG4OJIBUVFT/88APf0dPpBUjHX+7kMJ69UPbQQw9FWo+2CcoRchznDbUsEG1JfIrOFt8z5qE1f/+A49hwtosJFe+9995rr70WaS0wIcDpdA66Y+R9U2e62zmuBeOhX1y45OHHHv/xxx/btaEg6fhbmDiGyPpdpJUICOmOcM2aNYMHD/Yu+RWIth0wb6x6c48j4xyl5YJYWduBifJHmc8//3zuyveXnOBEBeLCRCcMw9jNRuvcw+0910uZDfsVA7756VxZWVm7NoTpMEh3hP369fPJnQSSdsC0O90HQkq3cDcaC8TEo4yy2/UwpoVNAhiBKH+U8bJjx44PP/yw+XLH9uKGUYqMa8PUFib2ke4IR4wYMXr0aB+hhB0wbrd7xYoV9t/icDjswSHUwLQVbLfZi0arbx6oha/8vaYIs98BasuyERi5DdWjDM/zFRUVweTMao1gdsheIbg3SWGpeiCI2GgcOmLiUQYACgsLH5n6zKvbzrB4dhATlYR4+4SEHTAkSXbu3Nknxw1xFcmaBFs+FAR/Fu3KiBEjmgslPMo4nc6ZM2cOGTIEABBCZrM5+Cj+FotFo9EI2TibiK98bips0QO16JTQrx9aKI5aOpZlWaPRaDab5XK5fzsmJSWFObUyAPTr1y8nJ2fMmDFNhVGYholhGEVKpmv0Qvjxq3C2GxNwHHfdddedO3fOK4nCWKMURR09ejR8L/RhJ8SOUMIOGJVK9fjjj/ukkHa73Xq93idBuSgoitLr9XKFwv87QQDx7L1ftBDq3k/Ner1eJpPp9fpgIveHH2mbeb/44ovQ5iOsrql96533ykovcWxT73I1d2C75SP0OVahUHTq1Ck68xFC6B5lbDbbXXfd1fRe7MVsNrvdbslPNkIaJpPJhPiW+6GoN+lA0jA1+ZK3Wq0NDQ0EQTidzjYrT05ODv+jzJo1az799NMW3+m/+uqrOXPmrFy5UnhqaVEYNua/9saad9cR3ftJK37ixImcnJxoXuMTYkcYPUl8Xnpt2Sf/+pBFJEy8OTwtNgXJFH96ahrw/OsLX+nbt2/4FZBMlGzmfX7egu/tqWCwyfWxEGU/mpD2KLNu3bqMjIzmtZEkmZCQIPlhzu120zRtsVgIsuXHCFGPFy0/thItH0IQZGJiYnp6OkEQPs/ZLRKRNEyx8k5fX1/Pdeol0VuoEwbfOXLjJ/8ZO3ZsaLUKISF2hNGTxGfP7l30jO2yT54NT3M+cLRrs+xWzcHVz9bVxZYjjJ5HGeg/GkoOgulymJprCavZuHnzZovFMmXKlGDGJ8KJhEcZgiBSU1Nb9ATkVaQpc+7cuTvv/r3TYpR1yWv7aLG05UMJgiBJUvgb+tZDQQinJ5577rnt27e32IqQrFhsYM+mCxc8Hg+IS6J8Neg2AFB21aAHHA5H01VawavUnGBSKwfbt31GNqIr4K9KF7agXC1w+2RFwacRa10q0fMoE3lqfim+VPHEyo8dP21/5JFHAnmriAai6FEG4OzZs+TvRsLv7kf/Wx62RmMdCe/0SqVy/PjxreUfFrIrB+4khMVf3trKy8sF1yv5ZopYz65du5KSku66666QqNQiwTzuxMZDLqb9iLZHmS++2nzg4PcXLl2CnuFpsHU8LuicZ3tqo/xU9M5tNAc/ysQ6Et7pFQrFyJEjW8s/LGRXDtzrsCyLEPLW9vvxj1TV1EB6T3GesMm4tbuu7D/VFz0gf+CBB0KiUsiJ0hEDTHzCsuwfHn14naFbvTEGEqtGCS0+ykRDXJKzZ89WV1fzOBiCSKIt1qjZUE/oUoOqgudQVt9o/h10QEfo8XgcDgffyio1TNRDwN3PEUp/YyCY6GfTpk03DR25eNn/0TQdEQV42v2PDf9+9vk5u3fvjogCkomeWKNms3nEmIlOlzMEN1PKvnnz5m49coKvqT3oIEOjVquV53m5XK7T6frffuel4jNMFJwaQ7uefWF+UmLiS89NmzBhQqTVwWDCgd1u/+STT7799lvPkCcRZZdHaLkTXXP+gPp24vjla7NPRnnvi7bpCS8lJSU/F53jeRSCFyaXhet+k/HyT2+//fakSZOC32ocWjrCG+GhQ4cyu16Tmd29U48+KZnZ504eZ94xk/LIX2ja3HAuZ0KBXVNeXh5pXTDS4RWaO+8be+vwe3bt2hVpXWKAGXPmzXz3ix2HCiI/JnrdPXDNjS6Xq7Gx0ZvkCCMKmUobusqQm/K8snRVFN4PO4IjLCwsJG77I/Pw/3mUetvcQ5FW57f0uJlMuybSSkQ1BoNh1apV+fn5ox969LEnnw42Klo7wLvtJwY+d9pMVFdXR1qXGMDhcHhu/RPXuXekFQEAgPqLb7+7Nqt7j6f/OiPSqsQYCKEDBw6EMvojQiCTqTK62e12qp0zkIgl8uOHoYSUQ1JmpJXAiGPy08/uraRR42UgCH7YU4D+E2mNWqLPcNmZLZFWInopLy8/deoUAAwbNizSuvwGZK1Bo+bxKV0ZOsoekaObS5cuvbbk9W27DxCJIb6jum3m+0c9kJXT5/cjhq1ftza0lUumYzlCTExRWVlZVVXV2GjgbpsBxXvlthp+8J/gv3+LtF4YEZw/f76srOzxac+5U3tSpSdze+XU1dfDPXdGWi+MRHieX//Rx4veWG53OFCXvqTTGNr6Wbed79Sr2qPauWdfaGsOhnAMjbZHCpiampotW7YsXPJG9943LH5jRUSyNwQKQg0NDefPn28eDyJWCJUF7Xb73r179+7d++mnn+7atavfLUPuf3LWqcIzIVQV0yLt0QfNZvOlS5duGTZy0itvN9RW2Sev5wjZLzkPWj1RN7jdAQhbIq2ysrJ58xfYbTakT2u/VkCf1mhzXpvbZ9GiRSdOnNi0adPq1asjtcAYwuMIQ5ICxu12z3jhlfsffHTMhIlFRUX3jH9kyqJ3899cU9nnQWNq74gkwQkQvq7k/977YOCQEVOfeqaoqOjChQuR1kg0wVjwH//4x4oVK9566y0A+Oucl8c9NeuBPzz5+Iw5j8x9w9JQY51ziNTFxnZ11lTzl+kz1PrE0RMmRloX0YSkDyKEFi5a9Mr8BY9PnTrjb891733DDTcPsZmM1unf/Bp1pMfNhCL6tr44TXsOHup53cBbh9zxyvwFy5bHXpib9k6kRVFUY2Pj/v37+w+6lSGUBCkLeRO/wUN5PMzlOsPStf8acd/oR5/+29wXXyorK9u0adPx48fbt+mWCMfQqP9wsRzHFRcX33DDDQDA8/zevXuNRmN9ff3AgQNTUlJ4nr98+fKcBUvsFpPZSfO3PQaH/vnN15tAkwiPfyBrEPbTEMhpAlMVMBRUngEAqCxEgKCuBBgPmKvAZQFGCaZKYGmoPi0cAAhB/QXE0GCuBqcZkSQYLwNLQ2UhAEDVaUAI6i8ihgZLDbhMiGegsQKxnqsHnAHEQ/1FYCiw1IDTiGgHNJYBx/zmgIZLvLmKHzKVddv27Pnkhhu2J3TuPujGgUoSvvz0Y2+gBJlMFrXhEEGMBTmO27t3L0Jo87d7vv7sYwAEiZlEdl9Uefrtdf80GM30vS/AxSMyl9n6wGIoGg6VhYhjobEM7I3IbfOaDzgWjOVXhVdMhjgWGsvBYUBu65WLXFmIWBaMFWBrQJT9qvA04hgwVoDdAO5mQlv9r0dWnUGsB0yXwVaPaOevQk4QNgDluCKsLmINFdyIGaxMceCHT7vlXacioeDHo1rtlTV1HcaCPM/v37/fbDaXlJRce+21J04XlZSWO13uisuX3TaLgdcCzxEeB0rpRvIk/8RH8M64K5eo5hfhB3/l6lEOsNWDx4Xc1itdr7YYEA+VhYAAqs8C4qHh4pXrTzvAVg9OM3JbwVwFDA21564cDE0OFixIO8BaD04jooSDKag7f/VgBNVFV7r2lYOdYK0HS41dlWZN6l1+avtxe6K67LCHpgFg7ty53njiHcmC33//vcFgyMjIEELIms1mnU4nnGBFRYWQpZUkSZvNZnV7Vr/zXmZaaklZhdVmA5eF6N4fTFWAELAMeJzAUAAAHAMeNwIeAIBngaEQ5QDEI8QDSwPlAI4FhID1AO0Cxg0IEMsC40YUCQCI58BzpQgIRTxOAACeQ4TMRqpItQyxzO8G3Ezw7LU5uT26ZWsTk+oNjXcPufXhhydWVlbabDaE0PXXXy+coNPpNBgMADBq1ChvEOBgLEiE4V1Kr9cbDAaNRuN2uzMzM202W9Nvc3NzAeDYsWMAQNP03ffexyJw2a06jbpLly5Go9FqtRKaJIah7XZ7alp6fU1V5+xujQ31yalpTptVqdEi1sPyoFGrbDZ7WkZ6XXVV5+xuhvr65NRUl8OuUGmAY1geaTRqm9WaltGprupy567djYaGxOQUl8OuUKmBYxkeaTVqm8Wa1qlTXXVlZnY3Y0N9YnIK5XTIVWqCZz0s0mk1Fos5vVNmXXVlZlY3s9GgT0ymnHaZUk0izsNyOp3WYjand8qsr6nq1CXbZDDok5I9LgepVJPA0R5Wr9dbTMb0zM5KEvZ99z9v6LzExMRoztYUuAUpihpx1z0gkxsMjWp9As94PCyXmJTUUFOVmd3NbGzU6BMQ6/F4uKTkpLqaqs7Z3cwmo0arQxxDe9jklGTBfGZjo1qrJ3iW8jDJKSmCySxGo1qnA+63QpNRrdUBz1K0rxBxLEXTKalpgr1sZqNSoyMQ56bolNTUuurKzKyuNrNJqdGSwLvcdEpqan11Vaes7KtC5HK5U9LSfnvkFWFHtaDQBzkE1VWVnbtkGRqN2sRkymlXqLU8QyFSriAJysPodFqbxZqant5QV5OZ1bWhtiY9s7PFZNQnJrsdNqVGy3koRMrlBFAeRq/X2azWlLR0Q11tZla2cLDZaExIunow7Qa5UkYg2sPqtFqbzZqSlt5YV9spK7u+pjqjcxezsTExOcVltyk1Oo52CQd7GE6rUdts9pTU1MaGuk5dhIOzzEZDYnKK025Ta3WM2wVypZxAHpbTaNR2uz0l5crBHdqCv6comgQ+PT09Ozvb7XZXVVUhhHr27Hm66KxcqbSbTdqEJNrtVGn1ToqmPSztdiqUahkJiJAxNEWQpEIuA0LGsizHMkqViiAIHiHa7VKptSQJCAiP2y1TKuUkAYTMQ1NAEEq5HGQylmU5hlEqVQRJ8Ahol0Ol0ZIEgQjSQ7lkcqWcJBBJsh4PAhCKcCzHeGiVWk0QBAKgXE6VWqPVJ6iBuenGG0+dPkPKFS6bZdCgQSRJVlVVmUwmHki1WrVr57ehsSBqf7RardvtRgg5nU6tVuvz7cCBAwsKCnyEr732Wk1NTTCN5ufitb6BAAAgAElEQVTnV1RUBFPDihUrSktLg6nhzTffLCkpCaaGKCFwC9I0PWvWrOBbXLduXfNfhVgKCgrWrVsXvDKzZs2iaTr4eiKIhD4oEORv+Pjx4+vXr5dcHCH0wgsvCPmZpfH111/v2LEjGAWiBMkWRAi98cYbly9fDrytHTt2fP3114EfX1tbu3jx4sCPD4NKYgnHG2Fubu7//ve/3NzcCxcuPPDAAyUlJU2/HT16tMVi8Q4xCRQWFubl5fmPNe6fM2fO9OzZU6fTSa6hqKjo2muvDSbnwC+//NK1a9fExMQ2j1y8ePHgwYMlN9TeBG5BnudPnDgxaNCgIFssKSlJS0tLSwtqut5oNBqNxry8YBMA/fzzzzfeeKP/UZcOY0EfAv8Nt0hjY6PZbBbeV6RRUFDQv39/yTmwKisrCYLo2rVrm0d2VAuC+Juh8PrYrVu3AI93u90lJSX9+/cP8Ph2UikYC4bDEc6ePVulUuXn57/yyissy65atarptwaD4eTJk+2tQ5QzYMCA4LO6tx/Ygm2CLRjrYAvGOsFYMByO0GKxPPbYY4WFhUIKmLAFv8eECmzBWAdbMNbBFmxXwuEIMRgMBoOJWqJ3uTAGg8FgMGEAO0IMBoPBxDXYEWIwGAwmrolqR1hUVBTM/oetW7def/31ycnJw4YN81lt3CbBR/YLpvUOQJC2g6AvYKhiM8a5HQWCsWYEu2E8227IkCHEVaZNm+bzbfML6/9aNa3tiSee8GOUFk0WeOWCqmKVDwHtt0UxSCwWy0033SRZw4qKCr1ef/ToUZfLtWrVqsGDB4sq/tJLLz377LMURT377LPz5s0Lc+uxTpC2Q6G4gEFaMFRqdACCsWYEu2E8247n+dTU1KqqKrvdbrfbhZ34TfG5sP6vlU9tc+bM8WOU5iYTVbnb7RarfEiuWJQ6Qp7nx48f/+WXX0q+me7fv/+pp54SPjc0NKSlpYkqnpeXV1xcjBAqLi7Oy8sLc+sxTfC2Q6G4gEFaMFRqxDpBWjOC3TCebVdbW6vX62+66Sa9Xj9u3Lj6+nqfA3wurP9r5VNbTk6OH6M0N5moyuvr68UqL/0yNSFKHeGyZcvmzJmDEAr+nZVl2WnTpk2fPl1UKZ1O53K5EEIulyshISHMrcc0IbQdCuIChsqCQaoR64TKmhHshnFou5MnT44YMeLkyZNGo3HKlCmTJk3yOaC1C9vitfKpTS6X+zGKH5MFUvmkSZMkKx8M0eIIe/fu7R2q3bdv35133unxeJDI7te0EoHdu3cPHDjwpZdeYhhGlD7+I/sFiOTWY4uQ2A6F1HwoRBYMXo2YI3hrRlU3jB/bNb/sAjU1NSkpKT5CrVbrDT3ovbCBXKuamhqCIPwYpTWTBVi5j6qtKR+qru0lWhxhU+bPn+8zkXno0CGxlfA8P2/evDvuuOP8+fMSdOjVq5cQa7ikpCQ3NzfMrccuIbEdCsUFDNKCoVIjpgnemhHshvFsu4KCgiNHjgifGxsbO3fu7HOAz4X1f618apPJZH6M0txkoirv3LmzWOUDuyRtEI2OsCmSB2QOHz7cu3dvs9lsv4qo4rNmzZo3b55gwrlz54a59Y5BMINpwV/AIC0YKjU6DNKsGcFuGM+2+/777zt16vTLL7/QND1z5sxp06YJ8v379wsffC5sa9dKON6ntn79+jU3Sms1o9YN0WLl06ZNE6t8SK5Yh3WES5cuDWZ9rNlsHjVqVHZ29pgxYywWS5hb7xgEc9bBX8AgLRgqNToM0s49gt0wnm3H8/zatWtzcnLS09OnTJlitVoFufci+FzY1q6V8MGntsuXLzc3Sms1o9YN0WLlVqtVrPIhuWI41igGg8Fg4pqo3lCPwWAwGEx7gx0hBoPBYOIa7AgxGAwGE9dgR4jBYDCYuAY7QgwGg8HENdgRYjAYDCauwY4Qg8FgMHENdoQYDAaDiWuwI8RgMBhMXIMdIQaDwWDiGuwIMRgMBhPXYEeIwWAwmLgGO0IMBoPBxDXYEWIwGAwmrsGOEIPBYDBxDXaEGAwGg4lrsCPEYDAYTFyDHSEGg8Fg4hrsCDEYDAYT12BHiMFgMJi4BjtCDAaDwcQ12BFiMBgMJq6JvCMsKSlxu91iS3Ecx/O8hOYQQizLSigIAAzDSCvIsixCSFrZ6CdAC0q+CJILSraX5IIxirQ+GExX4nme4zhpZSX3fei4li0pKbHb7WK7CcdxYq0gzegSunAwvxAJSHeELMtOnz49IyNjyJAh1dXVgtBsNo8ZMyY1NXXs2LFms9mP0MukSZOKi4vFtu50OiV0XQDweDw2m01CQQBobGyUVtBms3k8Hmllo58ALSj5IpjNZmldwmQySbhjIoSMRqOE5sJPZPsgy7IWi0Wa5m632+l0SisbTG+S9pOIfiZNmvTjjz+KPTWXy+VyuUQV4XneZDKJKgIAVqtV7CMIRVEOh0NsQ5KR7ghXr15ts9kqKioGDx68aNEiQbhixYprrrmmtra2e/fuK1eu9COMRWpqao4ePfrTTz9FWpG4oLGx8ejRoyUlJZFWJHqJoT544sSJo0ePXrp0KfxNY0JIYWEhRVGR1qIdQFIZOHDgqVOnEEI2m+3nn38WhHl5ecXFxQih4uLivLw8P0Iv/fv337x5MysSk8lktVrFlmJZ1ul0NjQ0SCjIsmxa9jWJeYMSUtLEFmxoaHA6nX4O4HlesiEizsCBAwsKCto8zGg0UhQVeLU3D/+9OqPbmIcebWhoYBhGgmJ1dXUcx4ktxfN8bW2thObCT6j6YIAW9MHj8RgMhkCO3L17ty6zu6ZLz8em/lmQOBwOq9UqtkUBk8nkdrullZX2k4h+Bg4cuHv3bpZlRZWy2Ww2my3w48vKyjQ6/cGDB0VqhxobG2maFlXE6XRaLBaxDUlGLtmDVlRUfPbZZyNGjOjZs+e//vUvQVhdXX3NNdcAgPD46Ufoxel0zpw5c8iQIaJat9vtMplMwrOJx+NxuVxI/JwTQshYXQFvH9Muvl7s0JnFYqFpWqlUtnZAUlKSSqUSq1LH5nLZJarf2J3ffZrVtetzM2YMGzZs3LhxkVYqughVH7TZbKNGjTp9+rSo1hmGsdvtgQzHnTt3DnKHunsOc7oPNTQ0AIDL5eI4Ttq7hcVi0Wg00vpLY2MjQogkWxgJw32wTex2u9sZvuHKcCLdEQqPEmfPnl27du3TTz/9ww8/AABCiCAI4YN3XqdFoZeEhIQvvviiU6dOolpXq9UymUyn04lVm6Zpp9OZmpoqtqDXdxIEIVZbuVyu0+lwNxONtQ7l3M6XHl+97VhhSRl2hD6Eqg/q9fo1a9akpaWJap1hGLlcHkgpvV5PECQAqFRK4Xi1Ws1xXGJioqgWBUiS1Gg0arVaQlmWZdPS0lp0hC0KMU0J5+qVMCPd9hkZGc8//3yXLl1mzJhRVFQkCLOysiorKwGguro6OzvbjzCmkbxUJ6pgQ7TUon0hZQjx/HX3dtxVt9IJVR8kSbJLly6ydsPrYwiCbL9WgkR4UMD44cnpz0VahfZCuiP8/e9//9FHH9E0/cEHHwwaNEgQjhkzZsOGDQihDRs2eJ/fWxTGKh63m0XJaRl9ru8faVWCJYaWWmBaJE77YIejqKio6eBW1D2MXqXs4sWItBsGpDvCZcuW7du3LzMzc+/evevXrxeECxcuPH36dLdu3c6ePbtgwQI/wliFcfMMzb76syf212B/+umnL7zwglarXbhw4V//+ldBuGXLlhkzZqhUqhkzZmzevNmPsCkURfGhplkjKORNhJB2tFPrxGkf7FhYrdapU6c23cYQtQ+jNOUGTaJ3NrojIX2OsHPnzrt37/YRJicn79ixIxAhJuKEaqmF1WodOXJkRUWF/+YsFovb7fazYqgpW7ZscTmdkPHr1KzHwxgMhoBPDgBAWNMkdu4HIWQ0GmUymahSycnJ4Z8Dxn0w1kEITZ06dd68eQ8//LBXuGXLlq1btwrPnePGjVu2bFlrQi8sy/7www+DBw8W9bulaRoAAuySAMAjHgBoDyMUDByPxyOsGhWlG8OIa0ihUEie6JXuCDGxTqiWWiQlJX355ZeZmZn+m1MoFAGuGPrqq69mv7GGcjoBwDtzo1Qq22yiORkZGRIcIQBIaAuDEcuKFStycnImTpzYVCjhYZSiqC1btjzzzDOiHKHwGiq2gwib0EQVcTqdBEGI2lPvdruF1ViBF9Hr9YE7dR+wI5QE7TTZXZndc24ZdOP2zV9GWhuJNF1qsXr1akEorKrIzc1tvtTCR9h+uFwuWY9BYPqmvRvChBkPTRsMBo1Gg1emAMD+/ft37tzZ/J1e2rrfFStWpKaminKECoUCABISEgI8ngACEL9j53elpaXeOelAQAglJCSI8lIul4thmKSkpMCLBANeMSwJu4FhmIYH8i9WVEVaFengpRaYsGK4tHXr1uxres6a+2KkVYkK9u7de/DgQaVSKXg4giAOHz4M0b72HrlslkhNircf2BFKhZRDclaklQgKvNQCE1YstTBwLPNgPtvR7qISWbp0qTeyCQAghIYOHQpR/jDa4VyggHRHOGTIEOIq06ZNE4RRu/AX0xxhqYXFYjl48GCvXr0EobCqoqqqatu2bd5xiRaFGIwUEDp16uSCVxd99913kVYlSonqh1FeYr6RKEeiI0QInTt3rqqqym632+127wxT1C78xWA6HjH5MHrh8GkDs3pv8c69+yOpRpTRdEUlfhgNPxIdYX19vcfjGTduXJcuXSZPnuxNbCRtFxoGgxFL7D6MEt37w+9GRlYHDKYpEleN1tXV3XzzzW+99Vb37t1nzZo1c+bMzz77DKQG3X7xxRe//FLc2ksh6LaEtGQ0TbtcLrGL1hwOR7PnaAQAPMcF8nxtsVg8Ho+fnQN6vV5YwYXBBIj3YfT8+fN33XXXBx98IITflLALjabp//znP3l5eaIUYBjG6XQGEvOToigECOA3ne6XM2dWrlx544033nbbbaLadTqdHMexknICO51OjUbT4oYBtVotarE+piMh0fADBgzYt2+f8Hn58uXXXXed8FnCwl+FQjF8+HCNRiNKAYZhZDKZ2FIAQJIkQkhsweEPTLh44cJvZQQAECQRSFUURfmPl48D/mLEEsKHUf5q5ilRCjRd6CGaS8eO11l/Wr912n01t956a9jaDUpnTMdFoiM8ceIERVGDBw8GAKVS6b3FS9iFplQqR40aJTaWvMfjkclkEiLQEwTBcZzYgheKCt1LimFWl2a1kYFUpVKp1Gp1tGWfGDJkyNGjR4XPzzzzzLp16wDAbDZPmTLlyJEjQ4cO/fe//52SktKaMMzwPO92u9VqNd6CJhDCh1GNRvPSSy8Fvp9MgGEYnucDKaVWq4nfvg4CxxJ5Q7mUbgqlWWy7LMtKzj7hcrkSEhLwc6cEGhoaaE9HTMkLAJLnCJ1O54QJE4qLiz0ez+uvvz5+/HhBHtULf0MO4s1G499mzVm//p+RVkU0MTbDZDccOnhAp9OdOnUqAq1HJULOd+Fz84dRiNJdaJhYpbS0FHXMrRMAAITkEYb333//rbfeslqto0aNevfdd4XUYhaL5bHHHissLLzxxhs//vhjYXVTc+F9991XX18vVFVdXZ2eni72bUnY0SnhyU4YGBFbsLyiAqV0BeNlSMkGSw0kdAJHI2iTwWUBpUZN8F06X4nI1bt3788//9ynuMlkirZ8hHV1dbm5ub179/bOMAlJFnv37r1169Y+ffqcO3dOmH9avnz5m2++mZmZqVAoGIapr6/v2rVr06qqqqq0Wm2br4kcx5EkGcj7nMPhMDooRLtAqQGEgKFAqQHEkzzXuVN64PEpAm+xKbm5uStXrhQbYk2lUokNTxokhw4dmjhx4oEDB3Jycl588UWapt9//30AmD17tkqlys/Pf+WVV1iWXbVq1fLly4WvUlNTTSaT8KFpVdL6IEKI5/lAztrhcDTa3VfvNAgIEngOZDIg5YlyPk1kclCe54WFsq0d0GIfFKivr5cQdS/iLF++/IsvvvBzQHV1dWpqqtgZH1F3UZqma2qEQXWUlZUl6tfSpsl86N2794YNG8IZWUaiIwySrl27/vOf/xSb3jb6KSkpyc/PLyws9JFHoSM8derU7NmzvTNMHo9HmGHS6/VCECy3252ZmWmz2Z599tnExMRHHnkk0iqHg5KSkqVLl27atEmsI9RqtWFe7hT4w2i8WbDFPigQo44wDi147NixcDrCiK2Suu6663xeLDoAMdTBRM0wde3adeDAgRHRM8yQJEmSZGJiYvRv1SIIYvr06dOnT/eRt5hoIq4sGGkV2gVswfZtNPxNYqIBUTNMGAwG04HBjjBOEbXcCYPBtAc7d+7s27dvcnJy3759d+3aJQijKAxQ3IAdYZwydOjQRYsWjRkzJjs722w2r1ixQpBHS0hDDKajw/P8Y4899u6775pMpiVLljzxxBOCPPKLtOMPHEkhThE1w4TBYEIOy7Iff/zxyJEjHQ6HSqVKTk4W5P7DAGHag2DfCIuKinQ6nfffaA/4i8F0OALpg5goRIgl4nQ6ExMTx40b9+GHHwpy/2GA4gGe5x0Oh8PhsItBWtQ9gaAcodVqnTp1qsvl8kqidDs2BtNBCbAPYqIWvV7vcDiWLl06c+ZMQeI/DFA84M1ULIpgWpQ+NIoQmjp16rx58x5++GGvUELAX4HxD0+qrDNIVkYUShK2b/oiPT29nepv/tvlrtJaEQn7viPI22ve+e9XW8PTFgnw6gszx44dG57mYovA+6AP2ILRQHl5+dq1a1etWqXT6f785z/n5+cLcv8xKQU6tgUJgtDr9QzD6PX68LQo3RGuWLEiJydn4sSJTYUSAv4KHDx02DJuJejTJOsTOOp/Tqao9gqax3Gc0Wj0EVosFpqm/YRESUpKitR2+6KioltvvdXpdAr/BhJW9Ezx+ROqvtB/dBjUUx54T9jOgWlO4H3Qh45tQY7jWjtxg8HAsmyLO9VSUlKkxS+VTFZW1j//+c/Ro0cPGzbsiy++8G4TFFZu5+fn+1m53bEtGH4kOsL9+/fv3Llz9+7dPnIJAX9/pddgSAnHxjVS2Y4uRyaTNY+YI5fLoy2yjEBrA2tfffXVnDlzVq5c2epEfWZueFLKkae/DkMrsYioPtgCHdeCMpmsSxff+PhXlCHJ6Ikso1Qqt2zZMnv27LKysj59+mzYsEGQL1y48LHHHuvWrZsQG6jV8h3XguFH4g9i7969Bw8eVCqV3sHcw4cPQ8cK+NvQ0DB58uQuXbp07dr1L3/5i91uj7RGIcY7sNZUGNNZlDmO69OnT1NJB145IqoPxgrNLdiasGNw5513FhQUmEymo0ePes8xppPRNzfW1q1br7/++uTk5GHDhpWUlERKMf9IdIRLly5tmtkLITR06FDoWNknnnzyyR49elRUVFy8eDElJWXx4sWBlLJYLO2sV8iQPLAWnaxZs2bw4MHnz59vKpSwcoTneYvFYhKJhBzRQSKqD8YELVqwRSEmOmlurMuXL0+ePPnDDz+sra0dO3asd69ktBHiIYIWt2PHxB7tp59++u233xY+//nPf37rrbcOHjw4d+5cpVKpVqvnzZu3adOmQOoRArpHP8LAWvORz5hYsdbcWADQr1+/V1991edICS+4BEFotVqdSKInuXmMdjdoxYItCjERJ0ALlpaWTpo06fbbb9doNI8//njUPtCEwBE2zV/R4kt9TLzpP/TQQ1u2bAEAmqa3bt06adKkQYMGLV++3GKxNDQ0LF68OLZej9okpgfWmhsLAEaMGDF6tO/aAQkvuARBCMFXRRHZaac2+2C0EbgFWxRiIk6AFhw+fLiwP5LjuIULFz766KMR0bZNomLSOBoYOXLkL7/8Ul9fv2vXroEDB2ZlZX300UfCk/Xtt9/es2fPVJFZ06KcmB5Ya26s1o6MiRfcOCRwC2KiE1EW3LNnz80335yUlLRmzZqwaSiKaBnPiThKpfKBBx7Ytm3bgQMHJk+eDAA6nW7Lli3CnoeDBw/27t070jqGg0BXrEWU5sZqjUC2ZGHCT+AWxEQnAVoQIfTKK68cOXLk888/z8vLC6eGosBvhL8yceLETz75ZPfu3RMmTACAF1988ZlnnrHZbLW1tfPmzXvuuecirWB7EXMDa9DMWK0REy+48UmAFsRELYFY8OjRo1u2bNm2bVtWVpYQNS2cGgYOdoS/cs899xQUFIwcOVLI9L1q1SqTydStW7e77rrrySeffPDBByOtIOZXfIzVGjGxciQ+CdCCmKglEAseOHDg/PnzKSkpCVcJp4aBI90RBp5JK1b2cqnV6r59+3pf89PS0rZu3Wq1Wn/55Zenn346srphfPAxlpemb7cQOy+40ojpbHYBWtCPEBNZArHg/Pnz0W8Jr46BInGOUMiktXHjxhEjRmzevPmJJ56orq6GVuKSBBKsJDE5lflqFqHUBnMyAULbW7g7MAxz5syZysrKe++9Nww6xDSpqamqPR8rLv8YhrbY0p9g/PU+QmwsENkHfcAWjHWwBUOLREcoKpNWIFGA9+3YcuzYMcmnIY5n7m++BHT79u1//etf161b5yciKEbgjUULBvQN29Kh+2+55RYfETYWBJfNDlsw1sEWDC0SHaGQScvhcCQmJnq3oEEQQbdzcnJycnKkKRMSHnzwwZDMAiIEzcejLBaLx+PxE2tUr9crFIrgWxfFzp07Z8+eXVNTk5WVtXr1auHJLpCg2yqVKrIr/UJlrJhGVB/0oWNbkOf51gIiOhwOtVrd4qZPjUYT/qgIW7dunT9/flVVVb9+/davXy+sq8R9MPwEtVgmwExa8bWXiwCNeMK/HVsYWHv33XdNJtOSJUu8oY9wNrvYAmeza47YPHbBZ7OTRmvhx3AfDD8S77/l5eUvvPACAAiZtIqLiwV5Rwq6LQ0CQN0MlUrVXNgUmUwWZj29A2tOp9NnYC12g26HBMF/iCX8qwBE9cG4Qshm1yI6na61r8L/Otha+DHcB8OPRNuLyqQVSHotTJgJZmCtY8PzvNlsFjtSHf6MksFks8NEA8OHDx8+fDg0Cz+G+6DQBxmGETWkodfrJU9YSp8jDDyTVkwEK4lPhIG1NWvWzJw586effgI8sAYgk8nS09ObJ5WMNoLNZoeJDvbs2fPiiy/ee++9S5cuFSS4DwqB7xmG0el0gZcK5p1eekkhk5aPUNi2FYiwo8Jy3O7du3v16tWjR49I6+KP8vLytWvXrlq1ShhYy8/PF+Q4JlkMEXgfxEQhrYUfw32QIAghkH3YRllwZJkQ46bosX/6y3//+99IK9IGwsDawYMHEULNB9ZwTDIMpr1pLfwY7oPhJzJBtzUazcKFC/V6PQB4PB6FQhGRVVvNYRiGJ0jgOJDJgWNBJgeOAbkSWM9v/soUwLFAygBxQJCAeAASABmNRqTQ0Df/MdLn0TaBD6xpNJqvvvqqqKhIGLFpPnkWEgvStAeEGhAPBOn3AwIAhVze5lJb7xBT4DQ2NorXPdoRLOgnFVxo+yDP8wzLAUEAQgAEgPevlyZCgrhi1l//Ct1KBoDkJCF2HVkMWdAbfswrEZZcSeuD8gB6hGQQQs1TT7fT61pELEhEJObNL7/8smfPHuHzqlWr/vCHP3Tt2lVUDRRFkSQpYWqUZVmPx6PV+oawcTqd2//37fkLl0w9RpBnvuUHPUQc/xwNfZI4+AG6dxZ8uxLGLoItr8IDL8N3b8LQJ4gfP0fX/5648D3q2o80lPL6DJJx8xxLpnRB19782i3qpgkqTSaTTqcL82KKUOF2u4WMYq0RoAVdLpdSqWxtHP+lV16lkruTSjWqOoMy80hShurOo855JM8hQynq3Jtk3MhUhTJ7k5RVhTx/nTxRWFDgB7vdrtfrxd7f09PTR44c2blzZ1GloplQWdAHjuMoimo+i3P27NmPvz3s1mWStlpen0G6zLwqgWTcPCknCeA5llSoedpJapN4h5FIzETmGkjrDoYyyMwl6s6h7OvIqjN89wEKl2ncDZ3vuOMOn/pdLpdCofCzmikjI+MPf/hDi1/V19dnZGRENnmkBAKx4NixY3Nzc0WdGk3TEJg/KykpWbvuA0DoyrMLkARwq996K8CGnE6nqLXxGRkZ48aNYxgmbGERI/NG2Ldv3759+wqfP/roo0mTJt14442iarDZbDKZTNRUqgBN006ns3lkmWkzZh5zpZNMGVx3D3HxMAwYQ5zahm5+GI58BLf/Cb57E4Y9BVtehdsfgz3vwICxcGo79BlB1J5FPW4mOZrPyCXdFp6hibRuURpNTyoajcZ/5o0ALej/aeDV15dRGT0IbSKqOw+dckiZnGssh069SM7DWWogM5d0W3iHEXXOI201St71yCOP3Hrrrf5blHbXQwjV19eLKhLlhMqCPjAMY7Va09PTfeTffPPNFwVV7vTfkTVn+E65pOkyn5BJui28XE0QCBgPoUkAu5FIyQJDGZHVBxEkXDsIGBp63gKUBXJuJ+wNkDtUZiq7886+M2bM8KnfbDZrNBq1Wi1K25gmEAtOmDBhxIgRol6ghbADgQTC3rt379oP/vmrIyRkBILAE/IYjcaEhARR7y0ulyvwg4MH5yNsQt5Qsq6Yj7QWGEyMYrVa//y3uUWFJyg+OdK6YDAikD5EsHXr1uuvvz45OXnYsGElJSWCMKazT2AiAkLojfxlNE1FWpHYI9r64I8//rjzaMF5TV5oFv0jZLFYqqurw/xygIlDJDpCUcGBcMQgjB/ef//9xRu20LTvVLwfWA81b/HS5Sv/r/20in6isw/KE9KgU2iCBnMNlxbnr+jRKy/612BjYh2JjlBUcCD/EYM4jjt//jwTaXhe0phoK/OBHMeFo/XgiJ73CcW1N4pa0uI2NRxokG/+366QaxJDhLAPIoSMRiMfNKFdecfbGrnxSxW3/wEhFFQwvx4AACAASURBVLxubRJCzTExh8Q5QlHBgfxHDHK5XAsWLLjnnntEKWC322UyGcuyYjX3eDxOp7PplHJhYeGERx+jaBpGvyqmH/96bNPuL3ykadpqtXqFNpuN4zg/c8WJiYlhzmYivE/s2rVrwIABa9eufeKJJ44cOQJBZJQMN936g+VopJWIJCHsg3a7/dFHH/VGKw0QhmHsdntT52exWIR/BdnVvwi8vaVZT/GlSZGrAmS32w0Gg1ditVqF+L2itBUwGo0A0OL6qeTk5Bhd1x0hUGFhYf/+/SOtRmgIarFMgMGB/EcMSkhIWL9+ffO1Z/5RKpWSV42q1eqmq0YPHz5sv+mPsnP7AEDMQnvi1z/Eb6QIQKvVNj0jkiSjbfuE930CAB5//PHly5cLcgkZJSmK+vDDD1etWuW/RafTiRBiGMZHTtM0avXV+le8d9MrN0mGvlBWcfvwu9eszPeuQG7eooTMHgghp9Pp3d0cIGq1OvxRmyFEfTAxMXHjxo2ZmZmimmYYRq1WN/2dp6SkCK0QTfrFFYlwxG97Sgs0KXJVQCQmJjbVTalUBrNqNBa3T0Sc5uNASKFbnL9yyxefRESfkCOx64oKDhQDEYNUOiBDeRf7/vvv8/Pz77vvPrFL0sNGCN8nBM/R5rAYukpzuZQTMFfZWNmZ8jqz2dxaDa21KE3PaKOj9UFMFPPvTz73FRHARXsXEYHEu78QHOiHH36Qy+XCs7MQJgZnnwAAVHJoL6CD679KTk6OWkcoEJL3CY1G8/zzz7e5G4lhmBZfi9VqNdEs9EhzCAB09T1DOJLQJitUpFarba1pl8uVkJAg4Y1QKCiqVPjBfRATNpwdfeGuREcYeHCg1oQdGYRQ37vkjrpI6+EP/D4R68RLH0TIbDZXVVWlpqY2DwiFwYQEiWPl8+fPR79FkAuR76uqqrZt2+aNjtOiEBNZRAX8xVGAo5Co6oMGg+HWkff/cepT7lC/OjD1pQuWvNEzt/fGjRtDW3OUwHFcnz59mkqicDe2xWLxkSAETqe4efRoBk8axyne94mEqwjyhQsXnj59ulu3bmfPnl2wYIEfIQbj5ezZs8VVjcbrH+T4EE8cIYeRfWiF6paJoa02SlizZs3gwYN94qFH4W7sM2eKfEU8t+/b7WFWo/3AjjBOiar3iWBgGEbCLhpMyJGptKAXt/Yb069fv6bR+QUk7ARtbzjeZ2UAAl9JbBPXsUbLyspGPzK5tqIUbnsq0rpgpOC2Gu++595bBg/94dCBSOuCwYhmxIgRzYUSVm5brdbRo0eXlpaKCrotTIgEE8EuwPD0ZrPZ7Xb7SRjSHLfbzTAMRYmIvBjMTtC4doQ7duy4KMv26OhIKxK/FBcXHzt2jOc10oqzThsas4BqOBBSpTCYSCJtJ+iaNWsyMjJEOUJhL2YwC6QzMjICOYwkSQnZJxiRaZiC2R4a1NBoTEzz+odMSAeVvj1q5h2m11at6Xvjrd7Mi5jm3Pn70Z9/e4ANSYzmuCQa+uChQ4cOHjzIcniAOjQIi7QBoPnKbR+hF4Ig1Go12T7s2Lmrtb1Nf//739upUQkEc82lF46Vad7W8Hg8zaOchBC29kJD7zGXuQQhqlN0EvHbqNNmYbP7hbDCuCIa+uCpU6dGPfKnZf/a7Ha3Y/4Qzm6at2T57wbe8t1337VfK1FCtK3c/uBf/2l5l68mcd2/PwunJu2HdEcYK9O8LVJYWJiamT1n7tx29YWQfg2p980AHD1Ew200JJgbG5YtW1ZRURFpRcJNNPRBl8slS+lC3zYlhHU2h2korb/+4csoJZofK0NFzKzcjvroS4EjfY4wVNO8Tqfz+eef37p1q6jWhaDbHo+I3D0CNE27XK7jx48T192FqKaDOWKNin7903ooYSH8mMVi8Xg8fiZy9Xq9qJnkkNCvX7+cnJwxY8Y0FUqINRphjBVVBvOS1f+45ZZbhJ9Z/BCqPuhyuV5++eXPP28WRssvDMPYbDafuNsC6NfO4fNFix+bHfWboNtXSb+WNF0QehNN09KWRVitVrlc3uIwWkT6oIDPBRQWafsc06IwbLRmLA9NeScvY5oQL5aRMM2rUChGjx6t0YhbLsEwjEwmE1sKAEiSRAgplcqgTUe08KlZKGHiaoxgjUbjp+sGOcAtjVDdRu12+8SJE48fP+6/ObPZTFGU74T51VtA8+wEVwXo1wNbfOxAiEjOUiYkWq3WxsbGpmVNJhNBEGKvLULIZDKJjaAd/vwhrSGhD8rlciGdk6iG5HI5wzAqlYpoFkC71c7VZtBt4Sui5c7F2RrnLX5Du+LNd1Ytu//++0VpK6BWq1uLwx6RPhgTWC3mlj0hz10oOklRlIT7cLQRYkcoIUCXUqm8++67xcaS93g8MplMQgR6giA4jlMoFC35LPEQbRUlCIVCoVKpJCeOCTMSbqNarfbll19uc30Xx3FNY40aDIap02fSPu/0za7krzdZ4tdcH6jZ4Sztem3l2yOO/LBq+RveshRFJSUlSXCEQkFRpUSt1mtXpPXBsWPHiu1NMpmMpmmlUtnC7//X1CyhhGmscg+ZRpfsoShKWvYJoSdinycKyt3K/gqEQK7++OOP//KXv4RXo9AT4h9EtE3zNsdqtT4xY/ai/JUShlXjAQkr1mQyWV5enkIkGzduPHDZ7XWrbd40/R9AG2tOQ9ddBw+LVSNURM+9Nfr7YFAkZ5OaxEgrgbkK20H2noW490bzNK/H45m3aOm4hyd9e7jgorxrs1gJGIDw3kbJjJ6hqgoAICOnztA4asLDp06dCmW1sUbY+qDRaLxn/KOjxk90OsObmoDnysrKCgoKmua+xrQfjYZGP9/OnvuCyWQKmzLtRLBDo9E/zevl9OnT7/xjvbvrIJmagtTuYKsJT7scx7EsG/357QRiOHeBtdbqpA4WXqitrR0wYECktQkfkeqD5eXlF6vqHUOnkz99Edqa/UMZLi9ZfWrJspUbP/nP2LFjw9l0HLJt27YyP+uxZQqnw7Fs5f+tWp4fRqVCT7xEltm+ffvOnTtBnQDd+0NRmFwgAHgaqyZP/hMA+uabb0aNGhW2dgMnhh5l2kaTpEjpDACFhYUGgyE7OzvAyBcYaZAKJYR9gxCiHPRDK5MK/hvmduMTl8sFiG/1a5IEQvXZV18vf+P16Jkml0C0TGy0K2VlZQ9Pnrru6GWaDve8IGdvRE9/nHBDC+sz45kjR458++23PN96B5MK4tg9e/YMuXf0mIl/aHMhKwaD8U9RUbO8Ez4QUF1d9fe//z0s6rQXceEIEUIKXRJ/44ORVgRzhbvuvW/XqdL2cITuxup3//WpW6ZV9hx0+PDhb775JuRNYKxW6/DRD419+A8ulztSOiDWs3379jVr1sRhIIWwQVHU8jYDaCAEPHfu3LmysrKwKNUudPyh0bKysp9//pnH0SyjhtraWoQQdMppj8p52s31G0OeP+iuL/vou+rd3x+76aab0tLSpK22xzTnwoULP/zww08nC123PU0e/Xek1HDVlv7rJ7n8650///xzbm7ulClTrr322kgp01Gpra3lCEUAwUbQp1//r6Ki4sMPP+zSpUs4NAs1HfmN0Gg0njx58qYhw//80utuMek82gOWZT0eT6wsmWk/vvvuux55fcOwdwWxDNvnrqqa2mt79sLvhcHj8Xhmz1805K77brj1jr++/DrF8JAc0Vsez3F3PMUSsv9eYJet3XDp0qVIKtNB2blzZ4BHWs3GHTt21NSEb/lFaAmHIwxP9gmbzXb8+PEjR45UV1e/tPiN/rffmd2r7x0PTDQ3Njgmrm6nRgOEsjSMf/BBtVp98uTJyGoijVBZ8IkZsx/545+YIU9AeGIyMW6kSVb1GTp/6coHxk7YuHHj5cuXw9Fu9BG8BSsrK99//x9HNQM9hMI55vWQaygRBHDbH1Wdrjl//vxPP/0kJNjrkIQ/h8/OnTuffX5uQIciHnEcAKxbt06nT4jF3RThcITtF7KZZdnS0tLS0lKXyzX5mefuevjxO+4f37P3dSvz3zh900zaanDODPSJpl3hnFb03PaEnjdUV1eXlpbGXEb14C3odrtnvfjy5//6wJaaKz6sa1DQDeUlZNa3+w4+PmPOyy+/vGDBgq+++urUqVMWiyWcakSWYCzIsuzLS/KnPjODI+TQ87Z20jAYXKb6OUvfuv2eB9Iys7QJifPnz3/vvfdOnDhRVlbWvlH1w0jYAt+bTCabzbZq1aqnp01HgXdVjgWldv1/PnFxkNGtlzIxZcGCBXa7vf30DC3hmCP0H7KZ47ji4uIbbrgBAHie37JlS3l5Ocdxw4cPP3nypMPhUKvVAwYMaGhosNvtxcXFarVaqVQqlKpPNm2z2h1V50+DSp+UlOSkPeyDy4mNcz2Pr4f1U0CdAABQcw4QgoZLwLFgqgSWBmsdol1gb0RuG4AMEQhoB9jqgaHAXIM4BkyXgefAUAYIQf1FQAC15wAAas4CANSeA4SgsQw4BsxVwFBgrUeUA5xGRDuA9SB7I3icYK1HLA3masSxYKxAPA+GUpfVMnbsWNAmpyUnqRVk0akTOp1OuAgymSx6QpM0R5QFv//++/Lycoqizp4vcTicRpMZsR7gue9+qQOEgOeRzQAIgLIjRyMAgNsOCiUCALcNEAcIwGVDchkAApcV8QwgBE4rou2AeHBZEOUAhMBpBtqJEA9OM/K4gefBYQKGAp4DhwlYGgEChxFxHgRyUKhBrmRI9WffHQLnNkKm5J0meVInGUd375nbJS3p4YcmHD52vLSqlkRcnx7dSJK8/fbbKYoym819+/YVwvIJBsrJyTEYDGfPnuU4bvr06YqrYZo7kgUPHDhgNptLSkpyc3MLCgoUCsWq1e9y/ccRHgoay4FjoLECsR4wVSKWBlPVlb7jcYOtDmjnlR5hNyC3/UpHI+WIdoDMDYCAZYFnwG1FCiVQdmQzAOUAWz14nGCrB9oN1jrkcYOlFhgKLDVX+hHrAeNlxDFgrEAcC43liOegsUzoqpzbzoxeQG5f4hm3hPx25YqPNvMOI7IZQKYYPW6CwWylKYqjXX+b9nRRUVF6enppaWlOTo5cLn/uuec6jAVLSkqGDh3K8zzP85s2bUpLS+vcuTP8dn+UcIJCxESEUEFBQVVVVV5e3u9+9zsAKCoqeunl+WabAyHkpmgATlyYPJ4DkgCe43mO58hlb65Z897fc3JyLC5P7+6drTa71enO7pw5buzoCyUl6enpW3d8+/qiV7Va7alTp0aOHOnVk+d5giAoimJZ1ps0+PTp0zqdrkePHl4buVyuM2fO/OlPf/KGBQ7GgkQYZq30er3BYNBoNG63OzMz02azNf02NzcXAI4dOwYANE2PuOc+hgeedirlMheLlCoNwVI8y5BKNeVyUxwvA0QSBMexSKHmPLRSo6VcTo0uweWw6ROTbRZTcmqaydCQntm5oba6U5dsQ11NeqdMU2NjcmqqzWrV63VOp1ujUdEeViknWR5IxIFcyTMemVLN0m6lRke5HFp9gsNmTUxOsZiMKWkZRkNdRmZWfXVlZnZ2Y31danons8mYnJxss9n1eq3LTamVSg/LyUnggCR5VqhQrlJ73G61Tuey23WJiVcqNDamZnTSKGT7vvufN1ht9IRsbhFRFrz73t+bTCZCJrdYbTqdDuRKh9mYkJjIETKX3arX6xEhdzmsep2eI0jK6dDptBwiKLdTq9VxCDxul1arZXnkoSmtRstwHOOhtRotw7Ecw6o1ag/D8Byn0ahpmuERr1apaNoDBKiUKoqmSIJQqlSU203KZHK5gqbccplcoVK6nU6FQiFXqlwOu1KpvPJBpZIrlC6rSafTIYXKYbVqtWqZXGk3NyYlJhIyudVs6n5ND4fVxAORmJTitJlvu+22H374QaFPVpDE3o5rQYZDNdWVXbK61tbVpWdkNjYaUtPTTY2NKWlpFpMpOSXVYjEnJiXZrNbEpCS73Z6g1zscTp1O63K5tVq1y01p1GqKptVKJeVhVAq5h+UUcpLhkIIElgcSEAeEHBBPyAjEIlJOcAzIlYj1EHIVYmhSqeI8tFylZmhKoVIzlFup0XrcLqF7qrV6yuXQ6BLcTrtWn+B02HX6RIfdmpCYZLdaEpNTbGZzYkqq1WxMTk0zGxtT0jLMDbVdunSuratPz+xsqKvr1LmLSk52JAtyHHfs2DEhAOyddw7PyemZmZkJAAghb6xgb8RgwREeO/6z3W6/pmt2nz69EULHfy64ePEiAhKAB4K4Gt73qoNAcPXz1QDAV/8DAECo2XwHAoKUkzK5Si0H3uFyA0GqFWR2dle7w84zjMPlvvfukWazubS0fMSIO+FqIGjhA8/zHMcplUpB1cPHfuickd6zZ0/viTQ0NFy4VHpg397QWBC1P1qt1u12I4ScTqdWq/X5duDAgQUFBWLr/PLLL3ft2iVBmaKionfeeUdCQYTQjBkzpBVcs2bN2bNnpZWNBkJiwXfeeaeoqEhC60uWLKmurpZQcO7cuU6nU2wpmqZnzZoloblopj36YHl5+bJly6Tps2fPni+++EJa2X/84x8///yztLIvv/yyxWKRVjaytGnBp59+uqGhQVSdW7du3bp1q6giDQ0NCxcuFFUEIbRq1aqLFy+KKnLgwIFPPvlEbEOSCcfQqP/I91lZWc8995xWqxVVZ0VFhVKplLBU12Kx1NXVbd++XWxBADh+/LhPGtsAKS4u/uSTT/wkNFi8ePHgwYMl1BweQmLBc+fO/fe//xWb1QEATp8+/d1334n9hQBAQUHBiRMnxAa84Hn+xIkTbe8j/i3xYEEfHA5HeXn5vn37JOhTW1tL0/T69esllC0pKUlLS0tLS5NQ9sSJE0ePHm3xvSHWLbhp06Zz586J2iYkRNJ/7733Ai9CUdS5c+eEwYPAOXv27KZNm7zjnIFQX1/vcrk++uijwIsEY8FwDI3Onj1bpVLl5+e/8sorLMuuWrWq6bcGgyFG11KGkAEDBnTq1CnSWrQKtmCbYAvGOtiCsU4wFgyHI7RYLI899lhhYaEQslnCOwEmsmALxjrYgrEOtmC7Eg5HiMFgMBhM1BK9y4UxGAwGgwkD2BFiMBgMJq7BjhCDwWAwcU1sO8KioiJvcJbA2bp16/XXX5+cnDxs2LCSkpJAigQT6E9Ccx0G/+c+ZMgQ4irTpk0ThP4vtZ9vJbTlRx6IMhhpv+0I9iZpd4woh+O4Pn36NJUEfoX9//59kGY4UU0Ecy5BEbYdiyHHYrHcdNNNYk+hoqJCr9cfPXrU5XKtWrVq8ODBgZR66aWXnn32WYqinn322Xnz5rV3cx0D/+fO83xqampVVZXdbrfb7cJmYdTWpW7tW2lttSZvszkMCuK3HaneJO2OEeWsXr36lltu8TmpAK9wm79/HyQYTlQTwZxLkMTqb4Ln+fHjx3/55Zdif9b79+9/6qmnhM8NDQ1paWmBlMrLyysuLkYIFRcX5+XltXdzHQP/515bW6vX62+66Sa9Xj9u3Lj6+npB7v9St/attLZak7fZHAYF8duOSG+SfMeIcvbt2yeEB2kqDPAKt/n790GC4UQ1Ecy5BEms/iaWLVs2Z84chJDknzXLstOmTZs+fXogB+t0OpfLhRByuVwJCQnt3VwHo8VzP3ny5IgRI06ePGk0GqdMmTJp0iRB7v9St2kIUW21Jg+8OQwS/9uOSG8K/o4RzficVIBXuM3fvw8SDCe2CcnnEiQx85vo3bu3dyx33759d955pzfPraiyArt37x44cOBLL73EMEwgrfsP9NcmYpuLXaRd6pqampSUFOGz/0vt/1uxbbUpD9LuHY/guxKKRG8Se8eIZpqbADU7KT9XuMXiqPV+EWC1gRBIE0jMuYSQmPxNzJ8/32em89ChQwGW5Xl+3rx5d9xxx/nz5wNvsVevXiUlJQghITdN4AWlNdcx8H/uBQUFR44cET43NjZ27txZ+Oz/Urf2rbS2WpO32RwGBfHbDn9vCuaOERP4OI8Ar3Cbv38fJBhObBNI6rkESUyuGl26dKnPVRs6dGiAZY8ePbply5Zt27ZlZWU5HI4AU1qPGTNmw4YNCKENGzaMGzcucFWlNdcxaO3cDxw4AABOp3PChAnFxcUej+f1118fP3688G2Ll1oo0tq3kttqTd5mcxgI4rcd/t4UzB0jFgnwCrf2+w+y2mCaCEmjUmgnBxs2xJ7C0qVLJVwBs9k8atSo7OzsMWPGiErjIq25jkFr5y584Hl+7dq1OTk56enpU6ZMsVqtwrctXmpv2dYMIa2t1uRtNodBQfy2I9ubOmQf9DmpAK9wa7//1pBgOLFNSD6XIMGxRjEYDAYT18Tk0CgGg8FgMKECO0IMBoPBxDXYEWIwGAwmrsGOEIPBYDBxDXaEGAwGg4lrsCPEYDAYTFyDHSEGg8Fg4hrsCDEYDAYT12BHiMFgMJi4BjtCDAaDwcQ12BFiMBgMJq7BjhCDwWAwcQ12hBgMBoOJa7AjxGAwGExcgx0hBoPBYOIa7AgxGAwGE9dgR4jBYDCYuAY7QgwGg8HENdgRYjAYDCauwY4Qg8FgMHENdoQYDAaDiWuwI8RgMBhMXBN5R1hSUuJ2u5tKeJ7nOK7NghzH8Twf5sNYlkUIhfkwDAaDwbQf0h3hkCFDiKtMmzZNEJrN5jFjxqSmpo4dO9ZsNvsRepk0aVJxcXFTicvlcrlcbSrgdDp9PGiLOBwOiqLaPMxms3k8njYPs1qtDMO0eZjFYmFZts3DTCZTIN4Xg8FgMO2HREeIEDp37lxVVZXdbrfb7atXrxbkK1asuOaaa2pra7t3775y5Uo/wiBhWfbEiRMhqQqDwWAw8YxER1hfX+/xeMaNG9elS5fJkyfbbDZBvmXLlhkzZqhUqhkzZmzevNmPMEi+//77seMnhKQqDAaDwcQzcmnF6urqbr755rfeeqt79+6zZs2aOXPmZ599BgDV1dXXXHMNAAivgMLBLQq9uFyuV1999dNPP/VKnE4nQsj/mGFlZSWPwGaztTkCabPZFApFm+OZwtAoTdNtHsbzvFKpbPMwAFAoFG0eJpfLZTKZVqtt82AMBoPBtAcSHeGAAQP27dsnfF6+fPl1110nfEYIEQQhfPAueGlR+KsGcvnAgQObugG5XA5teRGZTCYc2ab/UCgUAR4mEP7DZDIZSUZ+1RIGg8HEJxId4YkTJyiKGjx4MAAolUqVSiXIs7KyKisrc3Nzq6urs7Oz/Qi9KJXKBx98UKvVeiU8zyOEmkqao1KpCAK0Wq3/wwCAYRiFQtHmYTRNazQatVrt/zC3263Vatt8I3S5XIG85Gk0Gq1WKzh1DAaDwUQEiS8iTqdzwoQJxcXFHo/n9ddfHz9+vCAfM2bMhg0bEEIbNmwYN26cHyEGg8FgMNGAREc4dOjQRYsWjRkzJjs722w2r1ixQpAvXLjw9OnT3bp1O3v27IIFC/wIMRgMBoOJBiQOjRIEMX369OnTp/vIk5OTd+zYEYgwSMxmM+Px2Gw2nU4X2poxGAwGE1fE6hqNl199jaao8vLySCuCwWAwmNgmVh2h0+kEhGOyYDAYDCZYYtURYjAYDAYTErAjxGAwGExcgx0hBoPBYOKa2HaEgWRrwmAwGAzGD7HsCJXajRs3RloJDAaDwcQ2wTrCoqKipjv5JOQjlA4Ry14cg8FgMNFBUL7EarVOnTq1aRLdsOUjxGAwGAwmJEh3hAihqVOnzps3r6kwbPkIMRgMBoMJCRJDrAHAihUrcnJyJk6c2FQoIR+h2+1evXr12rVrvRIhH2Fb7SMAYBjGbrf7P87hcCgUijaX1TgcDpZl20xb6HA4AKDN7BMOh4MkyTazTzgcDqVSKZPJNBqNkHwKg8FgMGFG4s13//79O3fu3L17t4+8eerB5cuXKxSK22+/nSAIhJBMJhs4cGDTIo2Njfv37x86dGjTSgBAqKc1NAqZWkF+8803R48e9a9qILUBAM/zBEEEcpj/3IG5ubkfffQRcRX/tQV4GAaDwWDaD4mOcO/evQcPHvS+GBEEcejQoaFDhzZPPVhZWTlt2rRHHnkkZCpHMSUlJfn5+Xq9nqIovV7f5huhy+XS6/U4HyEGg8FEEImOcOnSpUuXLhU+C696wmch9WB+fn7T1INdu3b1eQvsqOBE8xgMBhNzhPjGjVMPYjAYDCa2CMECjaYLW9oj9SAGg8FgMO0HHsrDYDAYTFyDHSEGg8Fg4hrsCDEYDAYT12BHiMFgMJi4JgLRTN5e885/v9oanrZIgFdfmDl27NjwNIfBYDCYmCMCjvBM8fkTqr7Qf3QY2lIeeK+ysjIMDWEwGAwmRolQfMvMXPjdyDC0Q57+OgytYDAYDCZ2wXOEv8JxXJ8+fZpKdu7c2bdv3+Tk5L59+/5/e/caE1e57gH8XTPMDAMzAwMou9MibikwrWxaMPqBHmugSnKodKQaooE0Ia1JNc2u1lh3sO32WA7Emlg5hqSmlOyIzUFrwhma6j6tW7ECzdZdiaUtt9gqFJBrBzr3y3rPh2XnjHNZMw5zdf6/T/adh2e9wwf/mVmL9zl//ny0NgYAAOETfBB6DYlwzeANv9bW1rKystHRUecKy7J1dXXvvffe0tLSm2++2dDQEMXtAQBAmAQZhL5CIi5m8D7//PPHjx/n/nv37t3vvPMOIaS4uPjw4cOuZXa7vbOzs6KiwmAwSCSS9PT0KOwVAADCLMh7hM6Q0Ov1riHR3d2t1Wq5GbwajaalpSV0Ww2Zp59+urm5+eWXX7ZYLFqt9ujRo4SQ8vJytzKxWFxVVaXX6xUKBcMwfX19gTSnlOr1eoPBIBQK/U6f4CJWKBQmJydjHiEAQFQE+T9fXyHBP4M3RlRUVNTX18/Ozn7zzTclJSUqlYqnWCaT6fX61tbW/fv3f/vtt36b018LsPi3vQEAAAidVT0sw4VEU1PTFy2LqgAADIdJREFU/v37uRXPwbwxSCwWb9++vaenp6urq76+3lfZjz/++OqrrxJCUlNTd+/ePTw8HEhzgUAgl8tlMpk8AM4yfBwEAIiWIIPQV0hwg3kJIc7BvLHpmWeeOX369IULF2pqanzVqFSqU6dOffXVV5TSjz76KEFGKgIAJJogg9BXSHCDeSmlroN5Y9ATTzxx+fLliooKhULhq0YsFnd3dx84cCAzM7Orq+vkyZOR3CEAAERG8PcIuZC4efOmWq3u6Ojg1o8cOVJXV5eTk1NaWtrZ2Rm6fYZYcnLyxo0bPb8Xdbtd99hjj12+fDmC+wIAgEgL/taU15AIZDBvRkaG5PNO0cQ/g7504Ow3viVPFbkt2my2oaGhycnJysrKCOwBAABiWRSe0fjPvx7avLEwUlf790ceecRt6ezZsy+88MKJEyfEYnGktgEAADEqCkEokUh4ntWMgJ07d+7cuTOKGwAAgNiBs0YBACChIQgBACChIQgBACChIQgBACChIQgBACChBR+EWq22qKgoPT1969atY2Nj3GL8ziMEAIDEFOSfT0xMTNTX158/f37z5s1tbW0NDQ39/f3k7jzCTz755JVXXjl27FhLS4tUKv3444+vXbuWlJRkt9sJIW4HTFutVpFIxB3VHSCr1eo8AoZhmNj5c8CFhYVobwEAAH4bJrgZQL29vadPn+aO35yfn9+wYQOXAYWFhVqtVq1Wj4yMaDSa0dFRk8l0//3379mzJzs7e3Z2tr29/fXXX3dt1dLSUlJSsm3bNucKl3MSicTX1d9597+m7854SpWn/cehv/Bs1Ww2BzIa0GQyiUQiv1MgjEYjN0HQV8E999xTW1u7uLioUCj8XnR+fj4jI4PbHk9PAAAInyCD0MnhcOzbt08gELS1tRFCZDLZ/Py8VCo1mUzZ2dkrKyu+Fp3UanVubm5XV5dzxWAwUEplMpmvi26tePzqteuEEMIwmfeuGb/yL54drqysiEQiqVTK/0Z0Op1UKuVJX87S0pJMJvP7GfS3BqFMJvNbDAAA4bCqk2U+//zzgwcPVlZWNjU1cSte5xHyDylMSUlpaWlRKpXOFZFIRCmVy+W+risQ/v+2GYHA9We9FAsEIpEoJSXF79uRSqXJycn8NSzLyuVyv0HocDjS0tL8ZpvNZlMqlfgsCAAQRUEGIaW0sbGxv7+/q6uroKDAuc7NI8zPz3edR+h1MVSsJkNoGwIAQEIJ8qnRgYGB7u7unp4elUql1+v1ej237nUeYRiHFFJ2ZX4mlA0BACDBBBmEvb29o6OjSqVSfhe3fuTIkStXruTk5Fy7du3QoUM8i6GxqvubAAAAq35YZvVKS0vb29tLS0udK3q9nv8eYVrW3SduKCV2C/9bWF5eDuQe4e3btwO5R7i4uBjIPcKFhYVA7hHOzc1lZmbiHiEAQBTF5ckyRqMx2lsAAIDfibgMQgAAgFCJvyC0Wq2sg3VdcT6qAwAA8FvFXxBOTEyw1CUIBUl9fX3R2w4AAMS3+AtCd2LpqVMd0d4EAADEq/gPQtbxj6/7o70JAACIV/EfhJRlmVUdFAcAAIlsVUHocDjUarXritd5hBhSCAAAMSv4IGxtbS0rKxsdHXVd5OYRzszM3HfffceOHeNZBAAAiAXBf6lYXFycl5dXXV3tutjd3a3VaiUSyb59+zQaTUtLi69FJ7vd3tfX9+CDDzpXLBYLpdTX6S1Wq9VthVJqsVh87dNisbAs6/f0FovFIhAI/M4Htlqt3PYCKWNZNpAybh6hQBD/X1MDAMSh4IOwvLzcc3Fqaio3N5cQwn0E5Fl0MpvNH374YX19vXPFaDRSSn0Fg8lk8lw0GHzOoDAajdxcJ/63w13Uc0qU54UYhrHZbH7LApkG7BzzG8iMQwAACIcQP2YSxDxCmUx24sSJjIwM54pYLOY5a/TmzZtuKwzDuP64Gy6Q/J41yjBMIGeNchvzG1osywZy1qjdbucG8/KXAQBA+IT46zhu9CAhxHMeodti0D777LNVdgAAAHAKcRBGYB7h3Nyc24r5zrLdbl9lWwAASEwhDsIIzCO80HvxV/+mrMWw7PbwKgAAQIBWe4/Q7SGU9PT0c+fOudV4XQza8vKK+5JU8ZfGQ2e13aG6BAAAJI44e2R/bm5uZtr9uVPCCPz8mQIAAIAPcRaEs7OzXtctFgvPX1AAAAD4EmdB6J3d9o/er1/Y9+do7wMAAOLP7yIIWTsRMH//3/MLCwvR3goAAMSZ30UQEkIomZ+55Xn6GgAAAL84C8J3333X+wsOG0kSX7p0KbLbAQCAuBdnQfi30//t/QVKCCHT09OR3AwAAPwORCIIQzWP0GazUZ55DozgpYONtbW1er0+6EsAAECiiUQQhmQe4eTk5P1/fIB/hARrt575n7P3/XG9NCV1ZGQkuAsBAEBCYfzOJ1q9wsJCrVarVqtHRkY0Go3bcWibNm06ePBgbW0tIYRl2TNnzmRlZSkUCoPBMDQ0JJVKBwYGvu4fmJi8RQkhDEMoQ5xDAyklhBLCEMItUsIICOsgQpFQKExLTsr+wxq90cjabfnr11dWVl69erWmpmb9+vUXL17U6/W5ubnj4+NlZWVarXbbtm3r1q2TSCQTExMrKytFRUXEZW6Gc3qGTqeTyWRCodBt3flrZBiGYZilpaXl5eWVlZXi4mLXJlyZxWK5evXqc889t7S0lJmZKRQKhUIh5hECAERFJIJQJpPNz89LpVKTyZSdnb2y8qsz0vLz8wkh3HMuFovl3x59dOOGDUqlcmpqanR8XCIWT/88a7dYKCO4GzWEkF/yh/xyc/CXdYYwhFBCnYtUyDAOlkokIrudvSdTuXh7+aGSTWq1+tPP/s4wTFpa+s8/z5SWbB64dGnro4+qVCqBQHB9eNhmtW/a9CdngLkym80ikSgpKYl7lfvtMQzDzeB1hpnZbL5x44bJbC4tKSF3I9D5Izqd7vsrQxe/6jUYDEqlUigUKhQKzCMEAIiKEM8j9Ip/HqFcLm9vb8/KyuL+efPGjXPnzjkcjh07dvC3PXPmjFKpfPzxx/nLPvjgg7y8vC1btvCXvf/++w8//HBpaSl/2fHjx6uqqgoLC/nLmpub33rrrZycHP6yw4cPv/TSS5mZmfxlAAAQPpH4RJifn//pp5/m5+ePj49v3759bGzM9dUnn3xSp9O5Ds69desWpdRvivz0009isXjNmjX8ZT/88INcLr/33nv5y8bGxrKysngG/HKuX7++bt06hULBXzY0NPTAAw+kpqbylw0ODm7cuFEikbzxxhtlZWX8xQAAEA6RCMIDBw5IJJLm5ubGxka73f7222+7vjo/Pz84OBjuPcS4zZs3+41qAAAIh0gEoU6nq6ur+/7770tLSzs7O9PS0sJ9RQAAgABFIggBAABiFh7ZBwCAhIYgBACAhIYgBACAhBajQbhlyxbmrr1793oWBHh+KX8fh8OhVqv9tvUs82yr1WqLiorS09O3bt3q/PsQz25ey/y+WQAACJ9YDEJK6cjIyK1bt+7cuXPnzh2vo5cCOb+Uv09ra2tZWZnbeW+ebT3LPNtOTEzU19efPHlyZmZmx44dDQ0NXrt5LQvkzQIAQBjR2DMzMyOTyR566CGZTKbRaGZnZz1rCgoKhoeHKaXDw8MFBQVB9Pniiy/Onj3r9hvwbOtZ5tn2yy+/3LNnD/fq3NxcZmam125eywJ5swAAED6xGISDg4Pl5eWDg4OLi4u7du169tlnPWtSU1ONRiOl1Gg0yuXyoPu4BaGvtq5lPG3tdvvevXtffPFF/m6uZYFsEgAAwidWgtB5eqfb+vT0tFKp9KxPSUkxmUyUUoPBkJKS4re/rz5uV/TV1tdHZ9e2Fy5cKCkpee2112w2G083zzK/mwQAgPCJxKHbgXAdH/jdd9+ZzWbu7E2xWCyRSDzrVSrV5ORkfn7+1NTU2rVrvfYMpE9I2lJKGxsb+/v7u7q6CgoKfHXzWhbEJgEAIIRi8WEZg8FQU1MzPDxstVqPHj361FNPedZUV1d3dHRQSjs6OjQaTdB9QtJ2YGCgu7u7p6dHpVLp9Xq9Xu+1m9eyIDYJAAChFNXPo96xLNvW1paXl5eVlbVr167l5WXPmtu3b1dVVa1du7a6ulqn0wXdx+034Kuta5ln26amJq+/VbduXssC2SQAAIQPzhoFAICEFotfjQIAAEQMghAAABIaghAAABIaghAAABIaghAAABIaghAAABIaghAAABIaghAAABIaghAAABIaghAAABIaghAAABLa/wG2jEm26H1ojAAAAABJRU5ErkJggg==" + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "Y = convert(Matrix{Float64}, cg10k_trait[:, 3:15])\n", "histogram(Y, layout = 13)" @@ -577,43 +691,52 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 21, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "WARNING: using VarianceComponentModels.mean in module Main conflicts with an existing identifier.\n" + "WARNING: Base.I is deprecated, run `using LinearAlgebra` to load linear algebra functionality.\n", + " likely near /Users/juhyun-kim/.julia/packages/IJulia/GIANC/src/kernel.jl:41\n" ] }, { - "ename": "UndefVarError", - "evalue": "UndefVarError: Y not defined", - "output_type": "error", - "traceback": [ - "UndefVarError: Y not defined", - "", - "Stacktrace:", - " [1] top-level scope at In[20]:2" - ] + "data": { + "text/plain": [ + "(:Y, :X, :V)" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ - "using VarianceComponentModels\n", + "using VarianceComponentModels, LinearAlgebra\n", "\n", "# form data as VarianceComponentVariate\n", - "cg10kdata = VarianceComponentVariate(Y, (2Φgrm, eye(size(Y, 1))))\n", - "fieldnames(cg10kdata)" + "cg10kdata = VarianceComponentVariate(Y, (2Φgrm, Matrix(1.0I, size(Y, 1), size(Y, 1))))\n", + "fieldnames(typeof(cg10kdata))" ] }, { "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true - }, - "outputs": [], + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "VarianceComponentVariate{Float64,2,Array{Float64,2},Array{Float64,2},Array{Float64,2}}([-1.81573 -0.94615 … -1.02853 -0.394049; -1.2444 0.10966 … 1.09065 0.0256616; … ; 0.886626 0.487408 … -0.636874 -0.439825; -1.24394 0.213697 … 0.299931 0.392809], Array{Float64}(6670,0), ([1.00547 0.00656224 … -0.000121918 -0.00555374; 0.00656224 0.99614 … 0.00176999 0.00683183; … ; -0.000121918 0.00176999 … 0.998578 0.000962983; -0.00555374 0.00683183 … 0.000962983 0.999599], [1.0 0.0 … 0.0 0.0; 0.0 1.0 … 0.0 0.0; … ; 0.0 0.0 … 1.0 0.0; 0.0 0.0 … 0.0 1.0]))" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "cg10kdata" ] @@ -627,15 +750,31 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true - }, - "outputs": [], + "execution_count": 24, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " 53.245058 seconds (29 allocations: 1021.427 MiB, 0.24% gc time)\n" + ] + }, + { + "data": { + "text/plain": [ + "(:Yrot, :Xrot, :eigval, :eigvec, :logdetV2)" + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# pre-compute eigen-decomposition (~50 secs on my laptop)\n", "@time cg10kdata_rotated = TwoVarCompVariateRotate(cg10kdata)\n", - "fieldnames(cg10kdata_rotated)" + "fieldnames(typeof(cg10kdata_rotated))" ] }, { @@ -656,16 +795,134 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true - }, - "outputs": [], + "execution_count": 29, + "metadata": {}, + "outputs": [ + { + "data": { + "text/latex": [ + "\\begin{tabular}\n", + "{l | r | l}\n", + "name & size & summary \\\\\n", + "\\hline\n", + "Base & & Module \\\\\n", + "Core & & Module \\\\\n", + "Main & & Module \\\\\n", + "Plots & 22.513 MiB & Module \\\\\n", + "PyPlot & 790.274 KiB & Module \\\\\n", + "Y & 677.461 KiB & 6670×13 Array\\{Float64,2\\} \\\\\n", + "cg10k & 1022.983 MiB & 6670×630860 SnpArray \\\\\n", + "cg10k\\_trait & 1.499 MiB & 6670×15 DataFrame \\\\\n", + "cg10kdata & 679.508 MiB & VarianceComponentVariate\\{Float64,2,Array\\{Float64,2\\},Array\\{Float64,2\\},Array\\{Float64,2\\}\\} \\\\\n", + "cg10kdata\\_rotated & 340.136 MiB & TwoVarCompVariateRotate\\{Float64,Array\\{Float64,2\\},Array\\{Float64,2\\}\\} \\\\\n", + "maf\\_cg10k & 4.813 MiB & 630860-element Array\\{Float64,1\\} \\\\\n", + "missings\\_by\\_snp & 4.813 MiB & 1×630860 Array\\{Int64,2\\} \\\\\n", + "mp & 28.748 MiB & 6670×630860 SparseArrays.SparseMatrixCSC\\{Bool,Int32\\} \\\\\n", + "people & 8 bytes & Int64 \\\\\n", + "snps & 8 bytes & Int64 \\\\\n", + "startupfile & 57 bytes & String \\\\\n", + "Φgrm & 339.423 MiB & 6670×6670 Array\\{Float64,2\\} \\\\\n", + "\\end{tabular}\n" + ], + "text/markdown": [ + "| name | size | summary |\n", + "|:----------------- | ------------:|:-------------------------------------------------------------------------------------- |\n", + "| Base | | Module |\n", + "| Core | | Module |\n", + "| Main | | Module |\n", + "| Plots | 22.513 MiB | Module |\n", + "| PyPlot | 790.274 KiB | Module |\n", + "| Y | 677.461 KiB | 6670×13 Array{Float64,2} |\n", + "| cg10k | 1022.983 MiB | 6670×630860 SnpArray |\n", + "| cg10k_trait | 1.499 MiB | 6670×15 DataFrame |\n", + "| cg10kdata | 679.508 MiB | VarianceComponentVariate{Float64,2,Array{Float64,2},Array{Float64,2},Array{Float64,2}} |\n", + "| cg10kdata_rotated | 340.136 MiB | TwoVarCompVariateRotate{Float64,Array{Float64,2},Array{Float64,2}} |\n", + "| maf_cg10k | 4.813 MiB | 630860-element Array{Float64,1} |\n", + "| missings_by_snp | 4.813 MiB | 1×630860 Array{Int64,2} |\n", + "| mp | 28.748 MiB | 6670×630860 SparseArrays.SparseMatrixCSC{Bool,Int32} |\n", + "| people | 8 bytes | Int64 |\n", + "| snps | 8 bytes | Int64 |\n", + "| startupfile | 57 bytes | String |\n", + "| Φgrm | 339.423 MiB | 6670×6670 Array{Float64,2} |\n" + ], + "text/plain": [ + "name size summary \n", + "––––––––––––––––– –––––––––––– ––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––\n", + "Base Module \n", + "Core Module \n", + "Main Module \n", + "Plots 22.513 MiB Module \n", + "PyPlot 790.274 KiB Module \n", + "Y 677.461 KiB 6670×13 Array{Float64,2} \n", + "cg10k 1022.983 MiB 6670×630860 SnpArray \n", + "cg10k_trait 1.499 MiB 6670×15 DataFrame \n", + "cg10kdata 679.508 MiB VarianceComponentVariate{Float64,2,Array{Float64,2},Array{Float64,2},Array{Float64,2}}\n", + "cg10kdata_rotated 340.136 MiB TwoVarCompVariateRotate{Float64,Array{Float64,2},Array{Float64,2}} \n", + "maf_cg10k 4.813 MiB 630860-element Array{Float64,1} \n", + "missings_by_snp 4.813 MiB 1×630860 Array{Int64,2} \n", + "mp 28.748 MiB 6670×630860 SparseArrays.SparseMatrixCSC{Bool,Int32} \n", + "people 8 bytes Int64 \n", + "snps 8 bytes Int64 \n", + "startupfile 57 bytes String \n", + "Φgrm 339.423 MiB 6670×6670 Array{Float64,2} " + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "using SnpArrays, JLD, DataFrames, VarianceComponentModels, Plots\n", - "pyplot()\n", - "@load \"cg10k.jld\"\n", - "whos()" + "using JLD\n", + "@save \"cg10k.jld\"\n", + "varinfo()" + ] + }, + { + "cell_type": "code", + "execution_count": 45, + "metadata": {}, + "outputs": [ + { + "ename": "ErrorException", + "evalue": "unrecognized HDF5 datatype class 4", + "output_type": "error", + "traceback": [ + "unrecognized HDF5 datatype class 4", + "", + "Stacktrace:", + " [1] error(::String, ::Int32) at ./error.jl:42", + " [2] jldatatype(::JLD.JldFile, ::HDF5.HDF5Datatype) at /Users/juhyun-kim/.julia/packages/JLD/1BoSz/src/jld_types.jl:709", + " [3] read_array(::JLD.JldDataset, ::HDF5.HDF5Datatype, ::Int64, ::Int64, ::Tuple{Int64}) at /Users/juhyun-kim/.julia/packages/JLD/1BoSz/src/JLD.jl:411", + " [4] read_array(::JLD.JldDataset, ::HDF5.HDF5Datatype, ::Int64, ::Int64) at /Users/juhyun-kim/.julia/packages/JLD/1BoSz/src/JLD.jl:408", + " [5] read(::JLD.JldDataset) at /Users/juhyun-kim/.julia/packages/JLD/1BoSz/src/JLD.jl:372", + " [6] read_ref(::JLD.JldFile, ::HDF5.HDF5ReferenceObj) at /Users/juhyun-kim/.julia/packages/JLD/1BoSz/src/JLD.jl:502", + " [7] jlconvert(::Type{SparseArrays.SparseMatrixCSC{Bool,Int32}}, ::JLD.JldFile, ::Ptr{UInt8}) at /Users/juhyun-kim/.julia/packages/JLD/1BoSz/src/jld_types.jl:434", + " [8] read_scalar(::JLD.JldDataset, ::HDF5.HDF5Datatype, ::Type) at /Users/juhyun-kim/.julia/packages/JLD/1BoSz/src/JLD.jl:398", + " [9] read(::JLD.JldDataset) at /Users/juhyun-kim/.julia/packages/JLD/1BoSz/src/JLD.jl:370", + " [10] read(::JLD.JldFile, ::String) at /Users/juhyun-kim/.julia/packages/JLD/1BoSz/src/JLD.jl:346", + " [11] #39 at ./none:0 [inlined]", + " [12] iterate at ./generator.jl:47 [inlined]", + " [13] collect_to!(::Array{Tuple{String,Any},1}, ::Base.Generator{Array{String,1},getfield(JLD, Symbol(\"##39#41\")){JLD.JldFile}}, ::Int64, ::Int64) at ./array.jl:656", + " [14] collect_to!(::Array{Tuple{String,AbstractArray{T,2} where T},1}, ::Base.Generator{Array{String,1},getfield(JLD, Symbol(\"##39#41\")){JLD.JldFile}}, ::Int64, ::Int64) at ./array.jl:667 (repeats 2 times)", + " [15] collect_to_with_first!(::Array{Tuple{String,Array{Float64,2}},1}, ::Tuple{String,Array{Float64,2}}, ::Base.Generator{Array{String,1},getfield(JLD, Symbol(\"##39#41\")){JLD.JldFile}}, ::Int64) at ./array.jl:643", + " [16] collect(::Base.Generator{Array{String,1},getfield(JLD, Symbol(\"##39#41\")){JLD.JldFile}}) at ./array.jl:624", + " [17] (::getfield(JLD, Symbol(\"##38#40\")))(::JLD.JldFile) at /Users/juhyun-kim/.julia/packages/JLD/1BoSz/src/JLD.jl:1234", + " [18] #jldopen#14(::Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}}, ::Function, ::getfield(JLD, Symbol(\"##38#40\")), ::String, ::Vararg{String,N} where N) at /Users/juhyun-kim/.julia/packages/JLD/1BoSz/src/JLD.jl:246", + " [19] jldopen at /Users/juhyun-kim/.julia/packages/JLD/1BoSz/src/JLD.jl:244 [inlined]", + " [20] load at /Users/juhyun-kim/.julia/packages/JLD/1BoSz/src/JLD.jl:1233 [inlined]", + " [21] #load#13(::Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}}, ::Function, ::String) at /Users/juhyun-kim/.julia/packages/FileIO/YJO7Z/src/loadsave.jl:118", + " [22] load(::String) at /Users/juhyun-kim/.julia/packages/FileIO/YJO7Z/src/loadsave.jl:118", + " [23] top-level scope at In[45]:1" + ] + } + ], + "source": [ + "#using SnpArrays, JLD, DataFrames, VarianceComponentModels, Plots\n", + "#pyplot()\n", + "#@load \"cg10k.jld\"\n", + "load(\"cg10k.jld\")\n", + "varinfo()" ] }, { @@ -684,11 +941,43 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true - }, - "outputs": [], + "execution_count": 35, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Trait1\n", + "(σ2a[trait], σ2e[trait]) = (0.25947604776534905, 0.7375938878381869)\n", + "Trait2\n", + "(σ2a[trait], σ2e[trait]) = (0.18588150448759697, 0.8137880281704691)\n", + "Trait3\n", + "(σ2a[trait], σ2e[trait]) = (0.3196093756535387, 0.6795300238388639)\n", + "Trait4\n", + "(σ2a[trait], σ2e[trait]) = (0.2657824469749975, 0.7304994606945179)\n", + "Trait5\n", + "(σ2a[trait], σ2e[trait]) = (0.28143198005766734, 0.7169747061765026)\n", + "Trait6\n", + "(σ2a[trait], σ2e[trait]) = (0.28300551312417344, 0.7168800753377119)\n", + "Trait7\n", + "(σ2a[trait], σ2e[trait]) = (0.21565421413116753, 0.7816910320891622)\n", + "Trait8\n", + "(σ2a[trait], σ2e[trait]) = (0.19408878271207622, 0.8058201577783586)\n", + "Trait9\n", + "(σ2a[trait], σ2e[trait]) = (0.24746236011764494, 0.7512222977091667)\n", + "Trait10\n", + "(σ2a[trait], σ2e[trait]) = (0.0992417256213372, 0.9007769787053679)\n", + "Trait11\n", + "(σ2a[trait], σ2e[trait]) = (0.16457266481403124, 0.8343110221526249)\n", + "Trait12\n", + "(σ2a[trait], σ2e[trait]) = (0.08224956591863997, 0.9166483378364418)\n", + "Trait13\n", + "(σ2a[trait], σ2e[trait]) = (0.05687679106196685, 0.9424058676223442)\n", + " 0.152341 seconds (258.64 k allocations: 16.272 MiB, 8.63% gc time)\n" + ] + } + ], "source": [ "# heritability from single trait analysis\n", "hST = zeros(13)\n", @@ -699,8 +988,7 @@ "# enviromental effects\n", "σ2e = zeros(13)\n", "\n", - "tic()\n", - "for trait in 1:13\n", + "@time for trait in 1:13\n", " println(names(cg10k_trait)[trait + 2])\n", " # form data set for trait j\n", " traitj_data = TwoVarCompVariateRotate(cg10kdata_rotated.Yrot[:, trait], cg10kdata_rotated.Xrot, \n", @@ -715,17 +1003,27 @@ " h, hse = heritability(traitj_model.Σ, Σcov)\n", " hST[trait] = h[1]\n", " hST_se[trait] = hse[1]\n", - "end\n", - "toc()" + "end" ] }, { "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true - }, - "outputs": [], + "execution_count": 36, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "2×13 Array{Float64,2}:\n", + " 0.260239 0.185943 0.319885 … 0.164757 0.0823403 0.0569176\n", + " 0.0799434 0.08689 0.0739664 0.0887138 0.0944375 0.0953072" + ] + }, + "execution_count": 36, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# heritability and standard errors\n", "[hST'; hST_se']" @@ -742,19 +1040,187 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true - }, - "outputs": [], + "execution_count": 38, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Trait1Trait2\n", + "(Σa[i, j], Σe[i, j]) = ([0.258529 0.173943; 0.173943 0.184525], [0.738519 0.58639; 0.58639 0.815122])\n", + "Trait1Trait3\n", + "(Σa[i, j], Σe[i, j]) = ([0.259934 -0.0139395; -0.0139395 0.320025], [0.737149 -0.120342; -0.120342 0.67913])\n", + "Trait1Trait4\n", + "(Σa[i, j], Σe[i, j]) = ([0.259307 0.222267; 0.222267 0.265791], [0.737759 0.600077; 0.600077 0.730492])\n", + "Trait1Trait5\n", + "(Σa[i, j], Σe[i, j]) = ([0.259264 -0.147125; -0.147125 0.282102], [0.737798 -0.254669; -0.254669 0.716327])\n", + "Trait1Trait6\n", + "(Σa[i, j], Σe[i, j]) = ([0.259171 -0.129519; -0.129519 0.283267], [0.73789 -0.231373; -0.231373 0.716628])\n", + "Trait1Trait7\n", + "(Σa[i, j], Σe[i, j]) = ([0.258784 -0.140357; -0.140357 0.21535], [0.738269 -0.197894; -0.197894 0.782002])\n", + "Trait1Trait8\n", + "(Σa[i, j], Σe[i, j]) = ([0.259467 -0.0336174; -0.0336174 0.194107], [0.737603 -0.12623; -0.12623 0.805802])\n", + "Trait1Trait9\n", + "(Σa[i, j], Σe[i, j]) = ([0.261524 -0.204467; -0.204467 0.246378], [0.735632 -0.308118; -0.308118 0.752288])\n", + "Trait1Trait10\n", + "(Σa[i, j], Σe[i, j]) = ([0.259326 -0.0987008; -0.0987008 0.0962217], [0.73774 -0.304854; -0.304854 0.90378])\n", + "Trait1Trait11\n", + "(Σa[i, j], Σe[i, j]) = ([0.259194 -0.137578; -0.137578 0.162813], [0.737868 -0.36076; -0.36076 0.836058])\n", + "Trait1Trait12\n", + "(Σa[i, j], Σe[i, j]) = ([0.26168 -0.145305; -0.145305 0.0806851], [0.735514 -0.0421192; -0.0421192 0.918294])\n", + "Trait1Trait13\n", + "(Σa[i, j], Σe[i, j]) = ([0.260776 -0.108277; -0.108277 0.0513909], [0.736359 -0.114756; -0.114756 0.947913])\n", + "Trait2Trait3\n", + "(Σa[i, j], Σe[i, j]) = ([0.185992 0.14505; 0.14505 0.321413], [0.813678 0.0987898; 0.0987898 0.677805])\n", + "Trait2Trait4\n", + "(Σa[i, j], Σe[i, j]) = ([0.18554 0.074367; 0.074367 0.26577], [0.814123 0.22144; 0.22144 0.730512])\n", + "Trait2Trait5\n", + "(Σa[i, j], Σe[i, j]) = ([0.185854 -0.0115021; -0.0115021 0.281443], [0.813815 -0.0368393; -0.0368393 0.716964])\n", + "Trait2Trait6\n", + "(Σa[i, j], Σe[i, j]) = ([0.185903 -0.00350261; -0.00350261 0.283064], [0.813767 -0.0207933; -0.0207933 0.716823])\n", + "Trait2Trait7\n", + "(Σa[i, j], Σe[i, j]) = ([0.185498 -0.0301226; -0.0301226 0.215413], [0.814164 -0.00124563; -0.00124563 0.781929])\n", + "Trait2Trait8\n", + "(Σa[i, j], Σe[i, j]) = ([0.186397 0.032688; 0.032688 0.194625], [0.813285 -0.0321045; -0.0321045 0.805301])\n", + "Trait2Trait9\n", + "(Σa[i, j], Σe[i, j]) = ([0.184426 -0.084907; -0.084907 0.246288], [0.815228 -0.0814978; -0.0814978 0.752373])\n", + "Trait2Trait10\n", + "(Σa[i, j], Σe[i, j]) = ([0.18604 -0.123261; -0.123261 0.0992567], [0.813659 -0.273285; -0.273285 0.90084])\n", + "Trait2Trait11\n", + "(Σa[i, j], Σe[i, j]) = ([0.184894 -0.116422; -0.116422 0.165892], [0.814757 -0.297755; -0.297755 0.833033])\n", + "Trait2Trait12\n", + "(Σa[i, j], Σe[i, j]) = ([0.185297 -0.0908872; -0.0908872 0.0814927], [0.814404 0.0456529; 0.0456529 0.917465])\n", + "Trait2Trait13\n", + "(Σa[i, j], Σe[i, j]) = ([0.185348 -0.07104; -0.07104 0.0546791], [0.814367 0.0740412; 0.0740412 0.944638])\n", + "Trait3Trait4\n", + "(Σa[i, j], Σe[i, j]) = ([0.319575 -0.15468; -0.15468 0.265006], [0.679563 -0.303309; -0.303309 0.731254])\n", + "Trait3Trait5\n", + "(Σa[i, j], Σe[i, j]) = ([0.320016 0.184736; 0.184736 0.282719], [0.679136 0.336276; 0.336276 0.715725])\n", + "Trait3Trait6\n", + "(Σa[i, j], Σe[i, j]) = ([0.320608 0.166929; 0.166929 0.285124], [0.678565 0.297633; 0.297633 0.714823])\n", + "Trait3Trait7\n", + "(Σa[i, j], Σe[i, j]) = ([0.319614 0.167317; 0.167317 0.215458], [0.679526 0.346897; 0.346897 0.781883])\n", + "Trait3Trait8\n", + "(Σa[i, j], Σe[i, j]) = ([0.321532 0.0573397; 0.0573397 0.197221], [0.677674 0.0445311; 0.0445311 0.802757])\n", + "Trait3Trait9\n", + "(Σa[i, j], Σe[i, j]) = ([0.319777 0.138268; 0.138268 0.246547], [0.679369 0.266317; 0.266317 0.752112])\n", + "Trait3Trait10\n", + "(Σa[i, j], Σe[i, j]) = ([0.319905 -0.0779601; -0.0779601 0.100192], [0.679245 -0.141562; -0.141562 0.899839])\n", + "Trait3Trait11\n", + "(Σa[i, j], Σe[i, j]) = ([0.318882 -0.0177352; -0.0177352 0.164446], [0.680233 -0.114434; -0.114434 0.834435])\n", + "Trait3Trait12\n", + "(Σa[i, j], Σe[i, j]) = ([0.321958 0.0844644; 0.0844644 0.0863388], [0.67727 0.034186; 0.034186 0.912603])\n", + "Trait3Trait13\n", + "(Σa[i, j], Σe[i, j]) = ([0.32405 0.109731; 0.109731 0.0613165], [0.675287 -0.00619989; -0.00619989 0.938011])\n", + "Trait4Trait5\n", + "(Σa[i, j], Σe[i, j]) = ([0.26593 -0.216116; -0.216116 0.283173], [0.730347 -0.376697; -0.376697 0.715289])\n", + "Trait4Trait6\n", + "(Σa[i, j], Σe[i, j]) = ([0.266395 -0.200793; -0.200793 0.284547], [0.729899 -0.346915; -0.346915 0.715387])\n", + "Trait4Trait7\n", + "(Σa[i, j], Σe[i, j]) = ([0.264731 -0.182912; -0.182912 0.21441], [0.731526 -0.326257; -0.326257 0.782928])\n", + "Trait4Trait8\n", + "(Σa[i, j], Σe[i, j]) = ([0.266914 -0.0976281; -0.0976281 0.196119], [0.729401 -0.150498; -0.150498 0.803836])\n", + "Trait4Trait9\n", + "(Σa[i, j], Σe[i, j]) = ([0.270267 -0.227463; -0.227463 0.247665], [0.726155 -0.415849; -0.415849 0.751008])\n", + "Trait4Trait10\n", + "(Σa[i, j], Σe[i, j]) = ([0.265756 -0.0339156; -0.0339156 0.0987618], [0.730536 -0.227666; -0.227666 0.901251])\n", + "Trait4Trait11\n", + "(Σa[i, j], Σe[i, j]) = ([0.265858 -0.0963336; -0.0963336 0.163013], [0.730427 -0.273145; -0.273145 0.835846])\n", + "Trait4Trait12\n", + "(Σa[i, j], Σe[i, j]) = ([0.268394 -0.141452; -0.141452 0.0796851], [0.72801 -0.0831954; -0.0831954 0.919263])\n", + "Trait4Trait13\n", + "(Σa[i, j], Σe[i, j]) = ([0.266334 -0.0971798; -0.0971798 0.0541478], [0.729966 -0.226082; -0.226082 0.945138])\n", + "Trait5Trait6\n", + "(Σa[i, j], Σe[i, j]) = ([0.281794 0.281033; 0.281033 0.282364], [0.716628 0.660607; 0.660607 0.717505])\n", + "Trait5Trait7\n", + "(Σa[i, j], Σe[i, j]) = ([0.281023 0.232223; 0.232223 0.211943], [0.717383 0.67449; 0.67449 0.785343])\n", + "Trait5Trait8\n", + "(Σa[i, j], Σe[i, j]) = ([0.281551 0.164011; 0.164011 0.192737], [0.716865 0.221187; 0.221187 0.807144])\n", + "Trait5Trait9\n", + "(Σa[i, j], Σe[i, j]) = ([0.284085 0.244479; 0.244479 0.240901], [0.714415 0.508795; 0.508795 0.757606])\n", + "Trait5Trait10\n", + "(Σa[i, j], Σe[i, j]) = ([0.281993 -0.0460624; -0.0460624 0.100628], [0.716433 -0.0574194; -0.0574194 0.899407])\n", + "Trait5Trait11\n", + "(Σa[i, j], Σe[i, j]) = ([0.280652 0.0199425; 0.0199425 0.163738], [0.717732 -0.0349079; -0.0349079 0.83513])\n", + "Trait5Trait12\n", + "(Σa[i, j], Σe[i, j]) = ([0.281652 0.0612882; 0.0612882 0.0820476], [0.716765 0.0533362; 0.0533362 0.916851])\n", + "Trait5Trait13\n", + "(Σa[i, j], Σe[i, j]) = ([0.282494 0.0696137; 0.0696137 0.0570685], [0.715955 0.0537392; 0.0537392 0.942222])\n", + "Trait6Trait7\n", + "(Σa[i, j], Σe[i, j]) = ([0.283018 0.220794; 0.220794 0.214128], [0.716868 0.58124; 0.58124 0.783191])\n", + "Trait6Trait8\n", + "(Σa[i, j], Σe[i, j]) = ([0.283018 0.183995; 0.183995 0.192405], [0.716869 0.436926; 0.436926 0.807474])\n", + "Trait6Trait9\n", + "(Σa[i, j], Σe[i, j]) = ([0.285057 0.234311; 0.234311 0.24283], [0.714902 0.477263; 0.477263 0.755728])\n", + "Trait6Trait10\n", + "(Σa[i, j], Σe[i, j]) = ([0.283699 -0.0433873; -0.0433873 0.101183], [0.716209 -0.0593852; -0.0593852 0.89886])\n", + "Trait6Trait11\n", + "(Σa[i, j], Σe[i, j]) = ([0.281626 0.027639; 0.027639 0.163178], [0.718219 -0.0520113; -0.0520113 0.83568])\n", + "Trait6Trait12\n", + "(Σa[i, j], Σe[i, j]) = ([0.283196 0.0569533; 0.0569533 0.0819944], [0.716699 0.0481832; 0.0481832 0.916905])\n", + "Trait6Trait13\n", + "(Σa[i, j], Σe[i, j]) = ([0.283862 0.0600111; 0.0600111 0.0571748], [0.716057 0.0544601; 0.0544601 0.942116])\n", + "Trait7Trait8\n", + "(Σa[i, j], Σe[i, j]) = ([0.214129 0.0883281; 0.0883281 0.192399], [0.78319 -0.0565873; -0.0565873 0.80748])\n", + "Trait7Trait9\n", + "(Σa[i, j], Σe[i, j]) = ([0.219059 0.217228; 0.217228 0.243722], [0.778422 0.463004; 0.463004 0.754868])\n", + "Trait7Trait10\n", + "(Σa[i, j], Σe[i, j]) = ([0.216477 -0.042081; -0.042081 0.101258], [0.78089 -0.0859952; -0.0859952 0.898786])\n", + "Trait7Trait11\n", + "(Σa[i, j], Σe[i, j]) = ([0.214308 0.0205244; 0.0205244 0.163203], [0.783006 -0.0479474; -0.0479474 0.835656])\n", + "Trait7Trait12\n", + "(Σa[i, j], Σe[i, j]) = ([0.215192 0.0752708; 0.0752708 0.0802075], [0.782157 0.0353123; 0.0353123 0.918686])\n", + "Trait7Trait13\n", + "(Σa[i, j], Σe[i, j]) = ([0.216198 0.0740251; 0.0740251 0.0547527], [0.781186 0.0399163; 0.0399163 0.944534])\n", + "Trait8Trait9\n", + "(Σa[i, j], Σe[i, j]) = ([0.194551 0.112282; 0.112282 0.246832], [0.805384 0.185461; 0.185461 0.751837])\n", + "Trait8Trait10\n", + "(Σa[i, j], Σe[i, j]) = ([0.194399 -0.0154035; -0.0154035 0.0995883], [0.805519 0.0117323; 0.0117323 0.900435])\n", + "Trait8Trait11\n", + "(Σa[i, j], Σe[i, j]) = ([0.193837 0.022072; 0.022072 0.164393], [0.806067 -0.0267843; -0.0267843 0.834489])\n", + "Trait8Trait12\n", + "(Σa[i, j], Σe[i, j]) = ([0.193914 -0.00259164; -0.00259164 0.0821237], [0.805992 0.0333257; 0.0333257 0.916773])\n", + "Trait8Trait13\n", + "(Σa[i, j], Σe[i, j]) = ([0.193913 0.00330925; 0.00330925 0.0569126], [0.805993 0.0386552; 0.0386552 0.94237])\n", + "Trait9Trait10\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(Σa[i, j], Σe[i, j]) = ([0.246837 -0.00323396; -0.00323396 0.0989732], [0.751834 0.074992; 0.074992 0.901043])\n", + "Trait9Trait11\n", + "(Σa[i, j], Σe[i, j]) = ([0.247395 0.0305136; 0.0305136 0.164575], [0.751288 0.153633; 0.153633 0.834308])\n", + "Trait9Trait12\n", + "(Σa[i, j], Σe[i, j]) = ([0.249928 0.0844216; 0.0844216 0.0880992], [0.748829 0.108018; 0.108018 0.910884])\n", + "Trait9Trait13\n", + "(Σa[i, j], Σe[i, j]) = ([0.248998 0.0925603; 0.0925603 0.0580327], [0.749747 0.0992651; 0.0992651 0.94131])\n", + "Trait10Trait11\n", + "(Σa[i, j], Σe[i, j]) = ([0.0923059 0.0990858; 0.0990858 0.164638], [0.907657 0.475503; 0.475503 0.834248])\n", + "Trait10Trait12\n", + "(Σa[i, j], Σe[i, j]) = ([0.0957712 0.0574496; 0.0574496 0.0785646], [0.904228 0.0843534; 0.0843534 0.920326])\n", + "Trait10Trait13\n", + "(Σa[i, j], Σe[i, j]) = ([0.100109 -0.0266483; -0.0266483 0.0578511], [0.899941 0.164674; 0.164674 0.941451])\n", + "Trait11Trait12\n", + "(Σa[i, j], Σe[i, j]) = ([0.163544 0.0571549; 0.0571549 0.0784378], [0.835325 0.145549; 0.145549 0.920432])\n", + "Trait11Trait13\n", + "(Σa[i, j], Σe[i, j]) = ([0.164571 -0.00169939; -0.00169939 0.0575331], [0.834326 0.200725; 0.200725 0.941755])\n", + "Trait12Trait13\n", + "(Σa[i, j], Σe[i, j]) = ([0.08398 0.0685417; 0.0685417 0.0559416], [0.91494 0.573206; 0.573206 0.943343])\n", + " 3.627466 seconds (2.89 M allocations: 223.634 MiB, 3.17% gc time)\n" + ] + } + ], "source": [ "# additive genetic effects (2x2 psd matrices) from bavariate trait analysis;\n", - "Σa = Array{Matrix{Float64}}(13, 13)\n", + "Σa = Array{Matrix{Float64}}(undef, 13, 13)\n", "# environmental effects (2x2 psd matrices) from bavariate trait analysis;\n", - "Σe = Array{Matrix{Float64}}(13, 13)\n", + "Σe = Array{Matrix{Float64}}(undef, 13, 13)\n", "\n", - "tic()\n", - "for i in 1:13\n", + "\n", + "@time for i in 1:13\n", " for j in (i+1):13\n", " println(names(cg10k_trait)[i + 2], names(cg10k_trait)[j + 2])\n", " # form data set for (trait1, trait2)\n", @@ -768,8 +1234,7 @@ " Σe[i, j] = traitij_model.Σ[2]\n", " @show Σa[i, j], Σe[i, j]\n", " end\n", - "end\n", - "toc()" + "end" ] }, { @@ -783,11 +1248,78 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true - }, - "outputs": [], + "execution_count": 39, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "This is Ipopt version 3.12.10, running with linear solver mumps.\n", + "NOTE: Other linear solvers might be more efficient (see Ipopt documentation).\n", + "\n", + "Number of nonzeros in equality constraint Jacobian...: 0\n", + "Number of nonzeros in inequality constraint Jacobian.: 0\n", + "Number of nonzeros in Lagrangian Hessian.............: 78\n", + "\n", + "Total number of variables............................: 12\n", + " variables with only lower bounds: 0\n", + " variables with lower and upper bounds: 0\n", + " variables with only upper bounds: 0\n", + "Total number of equality constraints.................: 0\n", + "Total number of inequality constraints...............: 0\n", + " inequality constraints with only lower bounds: 0\n", + " inequality constraints with lower and upper bounds: 0\n", + " inequality constraints with only upper bounds: 0\n", + "\n", + "iter objective inf_pr inf_du lg(mu) ||d|| lg(rg) alpha_du alpha_pr ls\n", + " 0 3.0244169e+04 0.00e+00 1.00e+02 0.0 0.00e+00 - 0.00e+00 0.00e+00 0 \n", + " 5 1.6834042e+04 0.00e+00 4.07e+02 -11.0 3.66e-01 - 1.00e+00 1.00e+00f 1 MaxS\n", + " 10 1.4744248e+04 0.00e+00 1.12e+02 -11.0 2.36e-01 - 1.00e+00 1.00e+00f 1 MaxS\n", + " 15 1.4701415e+04 0.00e+00 1.25e+01 -11.0 1.08e-01 -4.5 1.00e+00 1.00e+00f 1 MaxS\n", + " 20 1.4700955e+04 0.00e+00 6.34e-01 -11.0 1.66e-04 -6.9 1.00e+00 1.00e+00f 1 MaxS\n", + " 25 1.4700954e+04 0.00e+00 2.63e-02 -11.0 7.01e-06 -9.2 1.00e+00 1.00e+00f 1 MaxS\n", + " 30 1.4700954e+04 0.00e+00 1.09e-03 -11.0 2.90e-07 -11.6 1.00e+00 1.00e+00f 1 MaxS\n", + " 35 1.4700954e+04 0.00e+00 4.49e-05 -11.0 1.20e-08 -14.0 1.00e+00 1.00e+00f 1 MaxS\n", + " 40 1.4700954e+04 0.00e+00 1.86e-06 -11.0 4.96e-10 -16.4 1.00e+00 1.00e+00h 1 MaxSA\n", + " 45 1.4700954e+04 0.00e+00 7.67e-08 -11.0 2.05e-11 -18.8 1.00e+00 1.00e+00h 1 MaxSA\n", + "iter objective inf_pr inf_du lg(mu) ||d|| lg(rg) alpha_du alpha_pr ls\n", + "\n", + "Number of Iterations....: 49\n", + "\n", + " (scaled) (unscaled)\n", + "Objective...............: 4.4662368766169078e+02 1.4700954216526392e+04\n", + "Dual infeasibility......: 5.8683114125458765e-09 1.9315987885869595e-07\n", + "Constraint violation....: 0.0000000000000000e+00 0.0000000000000000e+00\n", + "Complementarity.........: 0.0000000000000000e+00 0.0000000000000000e+00\n", + "Overall NLP error.......: 5.8683114125458765e-09 1.9315987885869595e-07\n", + "\n", + "\n", + "Number of objective function evaluations = 50\n", + "Number of objective gradient evaluations = 50\n", + "Number of equality constraint evaluations = 0\n", + "Number of inequality constraint evaluations = 0\n", + "Number of equality constraint Jacobian evaluations = 0\n", + "Number of inequality constraint Jacobian evaluations = 0\n", + "Number of Lagrangian Hessian evaluations = 49\n", + "Total CPU secs in IPOPT (w/o function evaluations) = 0.019\n", + "Total CPU secs in NLP function evaluations = 0.055\n", + "\n", + "EXIT: Optimal Solution Found.\n", + " 0.093736 seconds (46.45 k allocations: 5.051 MiB)\n" + ] + }, + { + "data": { + "text/plain": [ + "VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}(Array{Float64}(0,3), ([0.28137 0.280153 0.232512; 0.280153 0.284916 0.220363; 0.232512 0.220363 0.212921], [0.717042 0.661484 0.674207; 0.661484 0.714964 0.581648; 0.674207 0.581648 0.784373]), Array{Float64}(0,0), Char[], Float64[], -Inf, Inf)" + ] + }, + "execution_count": 39, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "traitidx = 5:7\n", "# form data set\n", @@ -809,20 +1341,32 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 40, "metadata": {}, "outputs": [ { - "ename": "UndefVarError", - "evalue": "UndefVarError: trait57_model not defined", - "output_type": "error", - "traceback": [ - "UndefVarError: trait57_model not defined", - "", - "Stacktrace:", - " [1] top-level scope at util.jl:156", - " [2] top-level scope at In[10]:1" + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + " MM Algorithm\n", + " Iter Objective \n", + "-------- -------------\n", + " 0 -1.470095e+04\n", + " 1 -1.470095e+04\n", + "\n", + " 0.534608 seconds (1.06 M allocations: 52.739 MiB, 5.97% gc time)\n" ] + }, + { + "data": { + "text/plain": [ + "VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}(Array{Float64}(0,3), ([0.28137 0.280153 0.232512; 0.280153 0.284916 0.220363; 0.232512 0.220363 0.212921], [0.717042 0.661484 0.674207; 0.661484 0.714964 0.581648; 0.674207 0.581648 0.784373]), Array{Float64}(0,0), Char[], Float64[], -Inf, Inf)" + ] + }, + "execution_count": 40, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ @@ -840,11 +1384,153 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true - }, - "outputs": [], + "execution_count": 41, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + " MM Algorithm\n", + " Iter Objective \n", + "-------- -------------\n", + " 0 -3.024417e+04\n", + " 1 -2.040172e+04\n", + " 2 -1.656053e+04\n", + " 3 -1.528554e+04\n", + " 4 -1.491023e+04\n", + " 5 -1.480677e+04\n", + " 6 -1.477849e+04\n", + " 7 -1.477005e+04\n", + " 8 -1.476675e+04\n", + " 9 -1.476478e+04\n", + " 10 -1.476318e+04\n", + " 20 -1.475020e+04\n", + " 30 -1.474023e+04\n", + " 40 -1.473255e+04\n", + " 50 -1.472662e+04\n", + " 60 -1.472201e+04\n", + " 70 -1.471839e+04\n", + " 80 -1.471554e+04\n", + " 90 -1.471326e+04\n", + " 100 -1.471142e+04\n", + " 110 -1.470993e+04\n", + " 120 -1.470871e+04\n", + " 130 -1.470770e+04\n", + " 140 -1.470686e+04\n", + " 150 -1.470616e+04\n", + " 160 -1.470557e+04\n", + " 170 -1.470506e+04\n", + " 180 -1.470463e+04\n", + " 190 -1.470426e+04\n", + " 200 -1.470394e+04\n", + " 210 -1.470366e+04\n", + " 220 -1.470342e+04\n", + " 230 -1.470321e+04\n", + " 240 -1.470302e+04\n", + " 250 -1.470285e+04\n", + " 260 -1.470270e+04\n", + " 270 -1.470257e+04\n", + " 280 -1.470245e+04\n", + " 290 -1.470234e+04\n", + " 300 -1.470225e+04\n", + " 310 -1.470216e+04\n", + " 320 -1.470208e+04\n", + " 330 -1.470201e+04\n", + " 340 -1.470195e+04\n", + " 350 -1.470189e+04\n", + " 360 -1.470183e+04\n", + " 370 -1.470178e+04\n", + " 380 -1.470173e+04\n", + " 390 -1.470169e+04\n", + " 400 -1.470165e+04\n", + " 410 -1.470162e+04\n", + " 420 -1.470158e+04\n", + " 430 -1.470155e+04\n", + " 440 -1.470152e+04\n", + " 450 -1.470149e+04\n", + " 460 -1.470147e+04\n", + " 470 -1.470144e+04\n", + " 480 -1.470142e+04\n", + " 490 -1.470140e+04\n", + " 500 -1.470138e+04\n", + " 510 -1.470136e+04\n", + " 520 -1.470134e+04\n", + " 530 -1.470132e+04\n", + " 540 -1.470131e+04\n", + " 550 -1.470129e+04\n", + " 560 -1.470128e+04\n", + " 570 -1.470127e+04\n", + " 580 -1.470125e+04\n", + " 590 -1.470124e+04\n", + " 600 -1.470123e+04\n", + " 610 -1.470122e+04\n", + " 620 -1.470121e+04\n", + " 630 -1.470120e+04\n", + " 640 -1.470119e+04\n", + " 650 -1.470118e+04\n", + " 660 -1.470117e+04\n", + " 670 -1.470116e+04\n", + " 680 -1.470116e+04\n", + " 690 -1.470115e+04\n", + " 700 -1.470114e+04\n", + " 710 -1.470113e+04\n", + " 720 -1.470113e+04\n", + " 730 -1.470112e+04\n", + " 740 -1.470112e+04\n", + " 750 -1.470111e+04\n", + " 760 -1.470110e+04\n", + " 770 -1.470110e+04\n", + " 780 -1.470109e+04\n", + " 790 -1.470109e+04\n", + " 800 -1.470108e+04\n", + " 810 -1.470108e+04\n", + " 820 -1.470108e+04\n", + " 830 -1.470107e+04\n", + " 840 -1.470107e+04\n", + " 850 -1.470106e+04\n", + " 860 -1.470106e+04\n", + " 870 -1.470106e+04\n", + " 880 -1.470105e+04\n", + " 890 -1.470105e+04\n", + " 900 -1.470105e+04\n", + " 910 -1.470104e+04\n", + " 920 -1.470104e+04\n", + " 930 -1.470104e+04\n", + " 940 -1.470103e+04\n", + " 950 -1.470103e+04\n", + " 960 -1.470103e+04\n", + " 970 -1.470103e+04\n", + " 980 -1.470102e+04\n", + " 990 -1.470102e+04\n", + " 1000 -1.470102e+04\n", + " 1010 -1.470102e+04\n", + " 1020 -1.470102e+04\n", + " 1030 -1.470101e+04\n", + " 1040 -1.470101e+04\n", + " 1050 -1.470101e+04\n", + " 1060 -1.470101e+04\n", + " 1070 -1.470101e+04\n", + " 1080 -1.470101e+04\n", + " 1090 -1.470100e+04\n", + " 1100 -1.470100e+04\n", + " 1110 -1.470100e+04\n", + "\n", + " 0.746208 seconds (135.30 k allocations: 13.562 MiB, 2.06% gc time)\n" + ] + }, + { + "data": { + "text/plain": [ + "VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}(Array{Float64}(0,3), ([0.281394 0.280169 0.232564; 0.280169 0.285001 0.220511; 0.232564 0.220511 0.213147], [0.717018 0.661467 0.674155; 0.661467 0.714877 0.581502; 0.674155 0.581502 0.784149]), Array{Float64}(0,0), Char[], Float64[], -Inf, Inf)" + ] + }, + "execution_count": 41, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# default starting point\n", "trait57_model = VarianceComponentModel(trait57_data)\n", @@ -861,11 +1547,22 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true - }, - "outputs": [], + "execution_count": 42, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "2×3 Array{Float64,2}:\n", + " 0.281842 0.285036 0.213725 \n", + " 0.0777056 0.0772501 0.0840114" + ] + }, + "execution_count": 42, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "h, hse = heritability(trait57_model.Σ, Σcov)\n", "[h'; hse']" @@ -882,11 +1579,63 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true - }, - "outputs": [], + "execution_count": 43, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "This is Ipopt version 3.12.10, running with linear solver mumps.\n", + "NOTE: Other linear solvers might be more efficient (see Ipopt documentation).\n", + "\n", + "Number of nonzeros in equality constraint Jacobian...: 0\n", + "Number of nonzeros in inequality constraint Jacobian.: 0\n", + "Number of nonzeros in Lagrangian Hessian.............: 16653\n", + "\n", + "Total number of variables............................: 182\n", + " variables with only lower bounds: 0\n", + " variables with lower and upper bounds: 0\n", + " variables with only upper bounds: 0\n", + "Total number of equality constraints.................: 0\n", + "Total number of inequality constraints...............: 0\n", + " inequality constraints with only lower bounds: 0\n", + " inequality constraints with lower and upper bounds: 0\n", + " inequality constraints with only upper bounds: 0\n", + "\n", + "iter objective inf_pr inf_du lg(mu) ||d|| lg(rg) alpha_du alpha_pr ls\n", + " 0 1.3111983e+05 0.00e+00 1.00e+02 0.0 0.00e+00 - 0.00e+00 0.00e+00 0 \n", + " 5 8.2228529e+04 0.00e+00 6.03e+02 -11.0 2.42e+00 - 1.00e+00 1.00e+00f 1 MaxS\n", + " 10 1.2570490e+05 0.00e+00 9.38e+02 -11.0 6.72e+01 -5.4 1.00e+00 1.00e+00h 1 MaxS\n" + ] + }, + { + "ename": "PosDefException", + "evalue": "PosDefException: matrix is not positive definite; Cholesky factorization failed.", + "output_type": "error", + "traceback": [ + "PosDefException: matrix is not positive definite; Cholesky factorization failed.", + "", + "Stacktrace:", + " [1] chkposdef at /Users/osx/buildbot/slave/package_osx64/build/usr/share/julia/stdlib/v0.7/LinearAlgebra/src/lapack.jl:50 [inlined]", + " [2] sygvd!(::Int64, ::Char, ::Char, ::Array{Float64,2}, ::Array{Float64,2}) at /Users/osx/buildbot/slave/package_osx64/build/usr/share/julia/stdlib/v0.7/LinearAlgebra/src/lapack.jl:5075", + " [3] eigen! at /Users/osx/buildbot/slave/package_osx64/build/usr/share/julia/stdlib/v0.7/LinearAlgebra/src/symmetric.jl:638 [inlined]", + " [4] eigen(::Symmetric{Float64,Array{Float64,2}}, ::Symmetric{Float64,Array{Float64,2}}) at /Users/osx/buildbot/slave/package_osx64/build/usr/share/julia/stdlib/v0.7/LinearAlgebra/src/eigen.jl:383", + " [5] TwoVarCompModelRotate(::VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}) at /Users/juhyun-kim/.julia/packages/VarianceComponentModels/Mh6LK/src/VarianceComponentModels.jl:119", + " [6] logpdf at /Users/juhyun-kim/.julia/packages/VarianceComponentModels/Mh6LK/src/two_variance_component.jl:62 [inlined]", + " [7] eval_f(::VarianceComponentModels.TwoVarCompOptProb{VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}},TwoVarCompVariateRotate{Float64,Array{Float64,2},Array{Float64,2}},Array{Float64,2},Array{Float64,1},VarianceComponentAuxData{Array{Float64,2},Array{Float64,1}}}, ::Array{Float64,1}) at /Users/juhyun-kim/.julia/packages/VarianceComponentModels/Mh6LK/src/two_variance_component.jl:731", + " [8] (::getfield(Ipopt, Symbol(\"#eval_f_cb#6\")){VarianceComponentModels.TwoVarCompOptProb{VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}},TwoVarCompVariateRotate{Float64,Array{Float64,2},Array{Float64,2}},Array{Float64,2},Array{Float64,1},VarianceComponentAuxData{Array{Float64,2},Array{Float64,1}}}})(::Array{Float64,1}) at /Users/juhyun-kim/.julia/packages/Ipopt/vosfb/src/MPB_wrapper.jl:64", + " [9] eval_f_wrapper(::Int32, ::Ptr{Float64}, ::Int32, ::Ptr{Float64}, ::Ptr{Nothing}) at /Users/juhyun-kim/.julia/packages/Ipopt/vosfb/src/Ipopt.jl:128", + " [10] solveProblem(::Ipopt.IpoptProblem) at /Users/juhyun-kim/.julia/packages/Ipopt/vosfb/src/Ipopt.jl:346", + " [11] optimize!(::Ipopt.IpoptMathProgModel) at /Users/juhyun-kim/.julia/packages/Ipopt/vosfb/src/MPB_wrapper.jl:141", + " [12] macro expansion at ./simdloop.jl:67 [inlined]", + " [13] #mle_fs!#27(::Int64, ::Symbol, ::Symbol, ::Bool, ::Function, ::VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}, ::TwoVarCompVariateRotate{Float64,Array{Float64,2},Array{Float64,2}}) at /Users/juhyun-kim/.julia/packages/VarianceComponentModels/Mh6LK/src/two_variance_component.jl:942", + " [14] (::getfield(VarianceComponentModels, Symbol(\"#kw##mle_fs!\")))(::NamedTuple{(:solver, :verbose),Tuple{Symbol,Bool}}, ::typeof(mle_fs!), ::VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}, ::TwoVarCompVariateRotate{Float64,Array{Float64,2},Array{Float64,2}}) at ./none:0", + " [15] top-level scope at util.jl:156", + " [16] top-level scope at In[43]:3" + ] + } + ], "source": [ "# initialize model parameters\n", "traitall_model = VarianceComponentModel(cg10kdata_rotated)\n", @@ -903,11 +1652,150 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true - }, - "outputs": [], + "execution_count": 44, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + " MM Algorithm\n", + " Iter Objective \n", + "-------- -------------\n", + " 0 -1.311198e+05\n", + " 1 -8.001595e+04\n", + " 2 -5.806836e+04\n", + " 3 -4.926167e+04\n", + " 4 -4.611163e+04\n", + " 5 -4.511722e+04\n", + " 6 -4.482795e+04\n", + " 7 -4.474405e+04\n", + " 8 -4.471603e+04\n", + " 9 -4.470277e+04\n", + " 10 -4.469345e+04\n", + " 20 -4.462308e+04\n", + " 30 -4.456930e+04\n", + " 40 -4.452802e+04\n", + " 50 -4.449620e+04\n", + " 60 -4.447148e+04\n", + " 70 -4.445209e+04\n", + " 80 -4.443672e+04\n", + " 90 -4.442442e+04\n", + " 100 -4.441448e+04\n", + " 110 -4.440635e+04\n", + " 120 -4.439966e+04\n", + " 130 -4.439409e+04\n", + " 140 -4.438943e+04\n", + " 150 -4.438549e+04\n", + " 160 -4.438215e+04\n", + " 170 -4.437928e+04\n", + " 180 -4.437682e+04\n", + " 190 -4.437469e+04\n", + " 200 -4.437283e+04\n", + " 210 -4.437121e+04\n", + " 220 -4.436978e+04\n", + " 230 -4.436853e+04\n", + " 240 -4.436742e+04\n", + " 250 -4.436643e+04\n", + " 260 -4.436555e+04\n", + " 270 -4.436476e+04\n", + " 280 -4.436406e+04\n", + " 290 -4.436342e+04\n", + " 300 -4.436285e+04\n", + " 310 -4.436234e+04\n", + " 320 -4.436187e+04\n", + " 330 -4.436144e+04\n", + " 340 -4.436106e+04\n", + " 350 -4.436071e+04\n", + " 360 -4.436039e+04\n", + " 370 -4.436009e+04\n", + " 380 -4.435982e+04\n", + " 390 -4.435957e+04\n", + " 400 -4.435935e+04\n", + " 410 -4.435914e+04\n", + " 420 -4.435895e+04\n", + " 430 -4.435877e+04\n", + " 440 -4.435860e+04\n", + " 450 -4.435845e+04\n", + " 460 -4.435831e+04\n", + " 470 -4.435818e+04\n", + " 480 -4.435806e+04\n", + " 490 -4.435794e+04\n", + " 500 -4.435784e+04\n", + " 510 -4.435774e+04\n", + " 520 -4.435765e+04\n", + " 530 -4.435757e+04\n", + " 540 -4.435749e+04\n", + " 550 -4.435741e+04\n", + " 560 -4.435734e+04\n", + " 570 -4.435728e+04\n", + " 580 -4.435722e+04\n", + " 590 -4.435716e+04\n", + " 600 -4.435711e+04\n", + " 610 -4.435706e+04\n", + " 620 -4.435701e+04\n", + " 630 -4.435696e+04\n", + " 640 -4.435692e+04\n", + " 650 -4.435688e+04\n", + " 660 -4.435685e+04\n", + " 670 -4.435681e+04\n", + " 680 -4.435678e+04\n", + " 690 -4.435675e+04\n", + " 700 -4.435672e+04\n", + " 710 -4.435669e+04\n", + " 720 -4.435666e+04\n", + " 730 -4.435664e+04\n", + " 740 -4.435662e+04\n", + " 750 -4.435659e+04\n", + " 760 -4.435657e+04\n", + " 770 -4.435655e+04\n", + " 780 -4.435653e+04\n", + " 790 -4.435652e+04\n", + " 800 -4.435650e+04\n", + " 810 -4.435648e+04\n", + " 820 -4.435647e+04\n", + " 830 -4.435645e+04\n", + " 840 -4.435644e+04\n", + " 850 -4.435643e+04\n", + " 860 -4.435641e+04\n", + " 870 -4.435640e+04\n", + " 880 -4.435639e+04\n", + " 890 -4.435638e+04\n", + " 900 -4.435637e+04\n", + " 910 -4.435636e+04\n", + " 920 -4.435635e+04\n", + " 930 -4.435634e+04\n", + " 940 -4.435633e+04\n", + " 950 -4.435633e+04\n", + " 960 -4.435632e+04\n", + " 970 -4.435631e+04\n", + " 980 -4.435630e+04\n", + " 990 -4.435630e+04\n", + " 1000 -4.435629e+04\n", + " 1010 -4.435628e+04\n", + " 1020 -4.435628e+04\n", + " 1030 -4.435627e+04\n", + " 1040 -4.435627e+04\n", + " 1050 -4.435626e+04\n", + " 1060 -4.435626e+04\n", + " 1070 -4.435625e+04\n", + " 1080 -4.435625e+04\n", + "\n", + " 3.378533 seconds (131.51 k allocations: 65.569 MiB, 0.70% gc time)\n" + ] + }, + { + "data": { + "text/plain": [ + "(-44356.24416253091, VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}(Array{Float64}(0,13), ([0.272112 0.190023 … -0.128464 -0.0976418; 0.190023 0.216814 … -0.0687833 -0.04341; … ; -0.128464 -0.0687833 … 0.116994 0.0900933; -0.0976418 -0.04341 … 0.0900933 0.105876], [0.725183 0.570497 … -0.0589748 -0.125486; 0.570497 0.783023 … 0.0235685 0.0464638; … ; -0.0589748 0.0235685 … 0.882056 0.551829; -0.125486 0.0464638 … 0.551829 0.893642]), Array{Float64}(0,0), Char[], Float64[], -Inf, Inf), ([0.0111615 0.0131072 … 0.0128913 0.0127584; 0.0131071 0.0151656 … 0.017153 0.0171365; … ; 0.0128911 0.017153 … 0.0174174 0.018212; 0.0127593 0.0171355 … 0.0182119 0.0188], [0.0112229 0.0133031 … 0.013004 0.0127777; 0.0133033 0.015806 … 0.0178516 0.0177826; … ; 0.0130042 0.0178517 … 0.0179556 0.0187634; 0.0127776 0.0177822 … 0.0187633 0.0193475]), [0.00012458 7.242e-5 … -3.70019e-7 -1.40553e-5; 7.23929e-5 0.000171795 … -2.04581e-5 -3.19854e-6; … ; -3.8356e-7 -2.04665e-5 … 0.000352066 -1.46068e-5; -1.40476e-5 -3.19013e-6 … -1.46022e-5 0.000374325], Array{Float64}(0,13), Array{Float64}(0,0))" + ] + }, + "execution_count": 44, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# reset model parameters\n", "traitall_model = VarianceComponentModel(cg10kdata_rotated)\n", From c6d35e3f7bdc3e0bed37d3e01d54a581c7d262c9 Mon Sep 17 00:00:00 2001 From: Juhyun Kim Date: Tue, 29 Jan 2019 18:18:20 -0800 Subject: [PATCH 46/56] update with JLD2 package --- docs/heritability_v0.7.ipynb | 323 ++++++++++++++++++----------------- 1 file changed, 167 insertions(+), 156 deletions(-) diff --git a/docs/heritability_v0.7.ipynb b/docs/heritability_v0.7.ipynb index b326236..6a821c1 100644 --- a/docs/heritability_v0.7.ipynb +++ b/docs/heritability_v0.7.ipynb @@ -16,7 +16,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 37, "metadata": {}, "outputs": [ { @@ -43,7 +43,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 38, "metadata": {}, "outputs": [ { @@ -85,7 +85,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 39, "metadata": { "collapsed": true }, @@ -96,16 +96,16 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 40, "metadata": { - "scrolled": true + "scrolled": false }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - " 0.196193 seconds (466.92 k allocations: 43.171 MiB, 11.74% gc time)\n" + " 0.080738 seconds (70.17 k allocations: 23.096 MiB, 12.02% gc time)\n" ] }, { @@ -140,7 +140,7 @@ " 0x03 0x03 0x03 0x03 0x03 0x02 0x03 0x00 0x03 0x03 0x03 0x03" ] }, - "execution_count": 4, + "execution_count": 40, "metadata": {}, "output_type": "execute_result" } @@ -159,7 +159,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 41, "metadata": { "scrolled": true }, @@ -170,7 +170,7 @@ "(6670, 630860)" ] }, - "execution_count": 5, + "execution_count": 41, "metadata": {}, "output_type": "execute_result" } @@ -188,13 +188,13 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 42, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "6670×630860 SparseArrays.SparseMatrixCSC{Bool,Int32} with 5524131 stored entries:\n", + "6670×630860 SparseMatrixCSC{Bool,Int32} with 5524131 stored entries:\n", " [5688 , 1] = true\n", " [6445 , 1] = true\n", " [109 , 3] = true\n", @@ -222,7 +222,7 @@ " [6511 , 630860] = true" ] }, - "execution_count": 6, + "execution_count": 42, "metadata": {}, "output_type": "execute_result" } @@ -240,7 +240,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 43, "metadata": {}, "outputs": [ { @@ -250,7 +250,7 @@ " 2 0 132 77 0 27 2 2 6 27 2 … 6 4 5 11 0 0 4 29 0 5 43" ] }, - "execution_count": 7, + "execution_count": 43, "metadata": {}, "output_type": "execute_result" } @@ -268,7 +268,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 44, "metadata": {}, "outputs": [ { @@ -303,7 +303,7 @@ " 0.1324128564961521 " ] }, - "execution_count": 8, + "execution_count": 44, "metadata": {}, "output_type": "execute_result" } @@ -314,7 +314,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 45, "metadata": {}, "outputs": [ { @@ -323,7 +323,7 @@ "([0.00841726 0.124063 … 0.364253 0.5], 0.24536516625042462)" ] }, - "execution_count": 9, + "execution_count": 45, "metadata": {}, "output_type": "execute_result" } @@ -336,25 +336,14 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 46, "metadata": {}, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "IOPub data rate exceeded.\n", - "The notebook server will temporarily stop sending output\n", - "to the client in order to avoid crashing it.\n", - "To change this limit, set the config variable\n", - "`--NotebookApp.iopub_data_rate_limit`.\n" - ] - }, { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlgAAAH0CAIAAABuMsSDAAAABmJLR0QA/wD/AP+gvaeTAAAgAElEQVR4nO3deXxU1f3/8TOZLJCFQCJrILGAylbKWhfiQlG/CoQQl1YEEYstliJI+aloAatQVr8qYi11iaI2AomEqGAQgYCAFhQqioSoZY0hgewLmeXO/f0x7XzTmTtDlsm9Mzmv54M/Jvdzl8+cuTNv7tyZOyZVVQUAALIKMboBAACMRBACAKRGEAIApEYQAgCkRhACAKRGEAIApEYQAgCkRhACAKRGEAIApEYQAgCk1paD8PTp0+Xl5f5dp91u56J0QgiHw6EoitFdBASbzWZ0CwFBURSHw2F0F8ZTVdVutxvdRUAIolfLthyEf/zjHz/88EP/rrOiooK9XAhRX19fU1NjdBcB4cKFC0a3EBCqq6stFovRXRjPZrNVVlYa3UVAKCsrC5b/G7XlIAQA4JIIQgCA1AhCAIDUCEIAgNQIQgCA1AhCAIDUCEIAgNQIQgCA1AhCAIDUCEIAgNQIQgCA1EKNbgA6uXDhwtmzZz2n19fX3/PrGaeOHdFcasS1yQf3f9rKrQGAkQhCWdw0Nu3M+XJTiNltusNmrS45I161aizz3d76vEV6NAcAxiEIZXHq+/yahV+JmM7uheICsfhqIzoCgIDAOUIAgNQIQjRHfn7+FT8d2iG+q+e/2Mu6vvrqq0Y3CACNxVujaI59+/ZVJ15Xf8dyz1L4B3+qra3VvyUAaB6CEM1kCm8vOnTVmh6pfzMA0Gy8NQoAkBpHhAAQQJYtW7Zx40aju/ADRVHMZvfvazXGVVddtX79er/340PQB+Hhw4eTk5M5KdUqLladOF14eb/BnpWqilLbFbfo3xHQ5p05c+b222+/++67jW7EGAUFBUuXLtV5o8EdhOfOnXv66afr6uqMbqSNKj110RR+6p43NUqbFpiFqnc/gBx69uw5dOhQo7swRkiIASfsgjgI6+vrZ8yYsWbNmpycHKN7abNMYe1Ez59qFCI76t4LALSKIP6wzMMPPzx37tzExESjGwEABLEgPiLMzMx87bXXnLdNJtPFixfbtWtnbEsQQgiHsn//fm+Pxfjx43v27KlzRwDgQxAHYUVFhfOGyWRSVc5XBQr7yUObTqgflh7yLKlHPurduzdBCCCg+D8IFUUZOHBgfn6+Z2nUqFH79+933p4xY8batWubtJ7y8vKpU6fu27cvOTl53bp1nTp18m/n8A/Fpvx80sWbZ3tWYsvG6t8OAPjm5yBcvXp1RkbG8ePHPUuqqubn5589ezY2NlYIERrqa9Oa61mxYkVSUlJWVta8efNWrly5bNky15o1V6IoypEjRz766CO36X379m32mUWr1WqxWBwOR/MWb20nTpx46623NUsWq9YPLbVEMw7CVdVms1ksFj93YijnLmF0F8ZzDoIhH/kLKFartYW7RMC+vOhGVdVmDGBYWFizdz8/B+HgwYP79OmTkpLiWSouLrZarampqcePHx8zZswrr7ziPI1kt9tdoei6rbme7OzsnJyciIiIWbNmpaamuoLQG4vF8uGHHx44cMBt+rRp01JTU5t3B2tra81mc1hYWPMWb213TX3waMQVIk4r5q023dtxpwpRX1/fxr70WVdX18buUfPU1dUpisKLuNVqra2tjYiIaPYabDb3p+rzq198J0unz8aHCLHw0TkTJkzQZ3OaVFVtxnMqOjo6PDy8eVv0cxCOHj3aW+ncuXMjR4587rnnEhMT586dO2fOnHfffVcIMWPGjLFjx955550rVqywWCyLFi3ytp7CwsKkpCQhRFJSUlFR0SWbiYyMfPLJJ++7777m3x8PDocjNjY2YIPQbDaLa6eIK2/QqOWu8vPGTE1fwmSKiYmJi4vzcyeGslgsbeweNU9ISEhERET79u2NbsRgVqs1LCysJbuEZ4geOZp/KGKA+Nn4lrXWKOF5L509e1aHDfkQEhKi83NKvw/LDBkyZOfOnc7by5cvHzhwoPP2qlWrbrvttpycnNLS0k2bNvlYg6qqJpPJeUNRlNZuGAACRbcrRf9f6LAd81fZmtNNJtOkSZMyMjJcUyZPnpyRkeE6M2W32xMSEuLi4r799lvnC7VzKberrNnt9tZpvEX0e0P/0KFDrk/KhIeHu/7XExcXl5aWtmHDhmnTpvl+P6FHjx5nzpwRQhQWFiYkJLR2wwAAl7y8PNepO6vVmpeX17C6e/fuyy677MKFC0ePHm043f7fdOu2SfQ4IszLy7vppptqa2vvuuuuvLy8Pn36LF68eOLEic7q2rVrd+3aVVBQMHHixKioqLFjvX6wMCUlJT09fenSpenp6c0+yQcDqXbrjh07iouLNat33nlnZCQ/4QQEqOTk5O3bt48fP14IsXPnzlGjRmVmZrqqGzduvP/++0+cOJGZmTlo0CDj2mwOPYJw9OjRqqomJyc/9dRTKSkplZWVY8eOXbNmjbPaq1evzZs3R0ZGbt26VfPjpi6LFi2aPHlyr169hg0b9vbb2p+NRCCru1C4+sMDYZ8VepYsB7NvuukmghAIWHfcccd7773nDMLs7Oy0tDRXENpsts2bNx86dOi777773e9+96c//cn17mhQaJUgdPs+g/NPk8k0c+bMmTNnus08btw4543u3bt3797dx3o6duy4ZcsW/7cLvag2m+XWRy0DNX62Iip/t/79AGi8cePGzZs3z2azmc3m3NzclStXuko7d+4cPHhwQkJCt27dKisrjx496joobJiIr7766oMPPqh3340QxFeWQVtic4i0e7VPEneKifwwO0v/lgA0FBMTM3z48Ly8vMjIyEGDBjm/Ee60cePGTz75xJV5Dd8dDYrLfhGECAi22sovr5wiouM9CvXhbz6we7f28WKHDh2k/bUaQH/Od0ejo6PT0tJcE61Wa3Z29qlTp5wXKtm2bdsjjzwSXO+OEoQIGFfdJOI8LkNafd4qQifOWug5u/1idWJM2NHD7hdMANBKJkyYsGDBgsjIyMcee8w1cfv27YmJia7Ldd14442nT59u+O5o4CMIEdgcdmGzVMzZoVE6cdC+5RHdGwLkFRcXN2DAAIvF0qVLF9fEjRs3NrwKWLt27caMGRNcnx0lCAEAl+A61ffxxx+7TVy3bp3bzO+//77bUgGOIETQUmwXyirunvKAZvHWm5J/8+B0nTsCWkNcfFzEzrfCTn2uw7bs/zooxGAdNhRQCEIEreLvyqtrssJGaZS+36/u+pQgRNuw9E8Lhw7sp9fWbv/5z3+u17YCBUGIIGaKjFVH3a9VMAnLp7q3A7SKiIiIKVOmGN1FWyb7j4cBACRHEAIApEYQAgCkxjlCtE2VFeVffPGFZqlnz57dunXTuR8AAYsgRFtUdCxv/2c33zvDs2Ip/fFPjz3y+OOP698UgMBEEKItqikT/UdXPviOZyV005P6twMgkBGEABBA2rdvn5mZmZ+fb3Qjxrhw4YL+GyUIASCALFmy5NVXXzW6Cz+orq6OiooKCWnaRzL79u3b8Mql+iAIASCAtG/ffvbs2UZ34QclJSXx8fFms9noRi6Nr08AAKTGESHkotaU/u/Lm99Yv8mzZDaJtaufvf766/XvCoCBCMLg88HWbVN/rX056eqLFsHLuE/qj8fO97nl/LWTPUvRm+eXl5fr3xIAYxGEwSdzU3bFqN+JazReykOWaf0UA9zE9RQ/0bi+vjmqo/69ADAcQRicIjuKTgmek00m/VsBgODGh2UAAFLjiBD4N1Wx7d279+LFi5rVcePGRUdH69wSAB0QhMC/XSw+9eLmyvB9Jz1LlsMfHT1yuG/fvro3BaDVEYTAv6k2i+V/HrP8bLxnKfpkf/37AaAPzhECAKRGEAIApEYQAgCkRhACAKTGh2WAS6u3Wq/qP0DzcgWhoaH1F+v0bgiA/xCEwKUpdVXqoi9F597uBYddnX2ZER0B8BuCEGgcc5gIDXefqHBROyDoEYRAC6iq2q5Dv2HXaBaTenRZ8PijmqXOnTv369evNTsD0FgEIdACDrtaV3H8tmc1Sj98/t3W5QdOPuFZsVZeGHP1kPez1rd6ewAagSAEWqz31RoTy86I9rEVj+zSKB3caL/wQWs3BaCR+PoEAEBqBCEAQGoEIQBAagQhAEBqBCEAQGoEIQBAagQhAEBqBCEAQGp8oR7QnbXun18fHTP+Ds+KSYgZ90+6++679W8KkBZBCOjuzFcl9nZFiZM0Sgc39N+zZ9CgQZ6VdzI3P7tiqeZFvk0mcfiLg1y8FGgeghAwgKljdzFsokbhwPq/vvnOuvc/8azUlhY77lkjhk7wLMWsHKUoit+bBCRBEAKBpOq88ovZ1SkLPCuhCwc6wtqJdjGeJVMIJ/uB5uP5AwCQGkEIAJAab40CQU9V7Dt27CgoKNCspqWl6dwPEFwIQiDo1dXWPPFKdmhUrGep6kBOVVVViNZJxJCQkPbt27d+d0CgIwiBoOew1df98gXRY4BG7Zsu8V26eU5WVbVnr8QT3+W3enNAwCMIgTatrsL2Sr0weRwRnjtufeMuIxoCAg5BGLgOHjxos9k8p58/f1500L8dAGibCMIAlZOTM+k3D4dfluBZqjlTICb+Qv+WAKBNIggD1KlTpxxDJlT+8nnPknnxCP37AYC2iu8RAgCkxhEhIKX66vPlldGdLtMsjh83bv0763TuCDAKQQhIqfqCXTXZnvpao/R1bsm593VvCDAMQQjIyhQiorWOCLWu6w20YZwjBABIjSNCAH5QUlLyl7/+LTRU4yXFZBJz58xOSND4LhAQCAhCAH7wyPyFH51VReIQz1K73S9P+tUvCUIELIIQgJ8MHieuuddzcsRXWfr3AjQe5wgBAFLjiBBAYymKsmDRU4qieJaOHz8urh2ruZSl/uL4u+4NDw/3LJkc1hf/d1VYWJhnKTIy8sYbb2xhw0BjEIQA3FWWl+Xm5npOX79+/Vufn1AH3OJZCim64G1t1qrSc/e8IHr016ituPG+p9aYQsxukx2Wi+0rThWfPdmktoHmIQgB/Lcfv/3qWME9T2pc57b2hy/FqF+LsY97lkL+8XeHj3V26SMSBmlMt1ys+u17Iszj94FLT4W/eHPjWwZagiAE8N/qa0w9f1o580ON0vNjTbq3A7Q2ghBAILJZLenp6ZqlXr163XKLxtuzQPMQhAACT+W56tq62W/t8awoZYUDY6yXXaZ9ufCuXbv26NGjlZtDW0MQAgg8DkWERtROeVWjtOOlf25ZOvruBzwrlorzv516z+rn/rfV20PbQhACCCp1FaJH/8r/t0Oj9PHzDrVY94YQ9PhCPQBAagQhAEBqBCEAQGoEIQBAanxYBgD8pqSkZOzdU0qKCjWrv5123zN/WqRzS7gkghCA1KxWa1lZmWbpyy+/PHDggMmkcTmdmJiYefPmeU7/9ttvT5RdrPnt+xqr+zzjx+KSljWLVkEQAmj7qqurX3rpJc3Sxi3bvz3yT3OYxo9jXKyqEDfPFmHt3AuWmg6H/qYZhEKIkPD2oktfjYIpZNOHubv3jdComMSfFzx+9913+7wTTfDOuxs+/8c/NEtX9P7JnNkP+2tDbQNBCKDt+90j/+/dL86Yev7Us+Q48rX6+82iz7Vai8WI2x8VkR3dp1cWiUPvNrmJwqOVXQeX3/aYZyVi+7MlJX47WCwrK3vggQfsqU9r1C6cGrBvPUHohiAE0HacOnHi/fc13pY8deqUY+gdIlnjejQh+99WW78xJ1N0vLh8uMb0Dp39uyFzeDv7LY9oFL7fL3Yc9u+22gCCEEBbcerwRycO7/nR5lmpOX5I9LxD/46Cy8svv1xdXa1ZGjNmzIgRGm/qtg0EIYC2wlLt+FlK5a80rjUa+sxQ/dsJLm+88cbclX9zDLhVo5a/KywsjCAEAPyHw25xmMaM1zjELLtQYrEo+nfUcna7PbT3iLo7lnqWwrMe1b8fPRGEANBENWWWmsqdiZM0SuUfmO3H9OniySUrNr23yXO6othtNqs+PbQNBCEANJkpNEwdNlGjcOGEKNIpCP/ywvNVU14VUXHuhfJCcVzjY0Gt4c2MzKeX/FmzZFbtedtze/bsqU8nLUEQAoDBlPOnZs1aO2vWLM3qzp07R48erb1kryEitpv7xOh/+dhWRen5BQsXek4/fPiwomj/3LEPa197/eSw6ZpfPmn/UorVGhwHpgQhABjMUVEk7l0tRv/Os9TxxVv8uaWz3xSVVf35a7NG6ciZ0F6dtJdSHYcPH373XY2vTpaWlorBPxG9fuZZMoWGtahVHRGEACARU4cu6vg/ahRKvve2iHLqn+uPVeSc0PheSu3JU37szSgEIQAELmtdzW/mPBodHe1Zqq2r06kJW7392vuqtb6hb37iCp16aE0EIQAELkv5uR9GzNV871F8map7O20TQQgAga3nYHHVjRrTNX4VA81BEBqsrKxM84NVVVVVQuh2BUQAkBdBaKT8/Pxrf/E/FovFs2StqRCjpuneEQBIhyA00pkzZ9QuV1yc9ZFG7a+/0vqAMwDAz0KMbgAAACMRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqfGFer/57LPPcnNzNUsdOnSYN2+ezv0AABqDIPSb0bf8j+XGmSI03L1QX9Pxn68RhAAQmAhCv7rtUdG+g/vEih/FPzcY0Q0A4NI4RwgAkBpBCACQGkEIAJAaQQgAkBoflml9DsViV++aPM2zUnLuR81f5QUA6IYgbH3V5y/WVL4Xcb1Gqeojs71K94YAAP+HINSDKTRCvW6qRqGyWFSc1b0dAMD/4RwhAEBqBCEAQGoEIQBAagQhAEBqBCEAQGoEIQBAakEfhIcPH46KijK6CwBAsAruIDx37tzTTz9dV1dndCMAgGAVxEFYX18/Y8aMF1980ehGAABBLIiD8OGHH547d25iYqLRjQAAglgQB2FmZubo0aNNJpMQwmQy1dfXG90RACD4BPG1RisqKpw3TCaTqqrGNgMACFL+PyJUFKVfv36apfLy8pSUlLi4uAkTJpSXlzd1PU1aHACAxvDzEeHq1aszMjKOHz+uWV2xYkVSUlJWVta8efNWrly5bNmyJq3H2+LeDgdramrmzp27YMECt+lz5syZMmVKE+5VA2VlZRaLJSwsTKPm66jUe8mvh7J+PzD2tb7mbEu/A3dvQ+HrQfJ/d17X2Kz3MHQcvaZvqxWa8+cqfY638btygLyj5a0Nn7urr528tLQ0Ojq6hV01UmxsbERERPOW9XMQDh48uE+fPikpKZrV7OzsnJyciIiIWbNmpaamOpPMbreHhv67DddtzfVoLu5DVFTU/PnzU1NT3abHx8fHxsY27w6qqhobG6sdhCaT9+W8l3ws1HS+WmjeCptZ8+cyzeNtKEw+nrj+787rGk3Neah0HD0dt+WdP3vwOd7N3JX9mF6BMNzCexs+d1dfO3mnTp3i4+Nb2FUjhYQ0/w1OPwfh6NGjfVQLCwuTkpKEEElJSUVFRc6JM2bMGDt27J133rlixQqLxbJo0SJv69Fc3AeTydS1a9crrriiGXfEG/N/+HGdCBAB8mIUpBg9eAoJCQmKV0tdPyyjqqrzfxaqqiqK4py4atWq2267LScnp7S0dNOmTU1dHACAltD16xM9evQ4c+aMEKKwsDAhIcE5MS4uLi0tbcOGDdOmTfP9Dq/m4gAAtIQeQZiXl+e8kZKSkp6erqpqenq669Td2rVrd+3aVVBQsHTp0q1bt/pYj+biAAC0hB5B6Drht2jRoiNHjvTq1evo0aOuD3P26tVr8+bNSUlJW7dujYyM9LEezcUBAGiJVjlH6PZZW9efHTt23LJli9vM48aNc97o3r179+7dfaxHc3EAAFoiiC+xBgBAyxGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkFcRA6HI5p06aNGDGiX79+mZmZRrcDAAhKQRyE27dvt9lsX3zxRWZm5syZM41uBwAQlII4CH/2s58tX768vr7+zJkz3bt3N7odAEBQCjW6gebr1q2bEGL8+PFbtmz5+OOPjW4HABCUgviIUFEUh8ORk5OTlZX10EMPGd0OACAoBXEQvvzyy48//rjZbO7Tp09dXZ3R7QAAgpKfg3DUqFGm//A8SisvL09JSYmLi5swYUJ5ebnvVSmK0q9fPx+LT5069Ycffhg8ePCkSZNeeeUV/94RAIAk/HmOUFXV/Pz8s2fPxsbGCiFCQ91XvmLFiqSkpKysrHnz5q1cuXLZsmXeVrV69eqMjIzjx4/7XnzTpk0++rFYLJs3bz558qTb9BtuuGH48OFNumsutbW1ZrM5LCxMo6Y2a43NWyoQNKdzHe9t0zcV8A+Ffg02Y0ut0Jw/V6n6WlnAP/JGU30Nn/eSqtbV1dXU1LRCRxratWvnGTqN5M8jwuLiYqvVmpqa2r179ylTplRVVbnNkJ2dPWvWrIiIiFmzZrkyzG63u2Zw3R48ePDChQsbs7hvtbW1ZR4uXryotoJmjhoAtFGt8UrbGq/A/jwiPHfu3MiRI5977rnExMS5c+fOmTPn3XffbThDYWFhUlKSECIpKamoqMg5ccaMGWPHjr3zzjtXrFhhsVgWLVokhBg9erTn+jUX9yEiImLy5Mn33Xdfy++ai8ViiYmJ0T4iNDVrjc1bKhA0p3Md723TNxXwD4V+DTZjS63QnD9XafK1soB/5I1m8jV83ksmU1RUVExMTCt05Gf+DMIhQ4bs3LnTeXv58uUDBw50m0FVVeeAqqqqKIpz4qpVq2677bacnJzS0lLfx3maiwMA0BL+fGv00KFD+/fvd94ODw+PiIhwm6FHjx5nzpwRQhQWFiYkJDgnxsXFpaWlbdiwYdq0aZ6LXHJxAABawp9BWFtbm5aWduzYMavVunjx4okTJzqn5+XlOW+kpKSkp6erqpqenp6amuqcuHbt2l27dhUUFCxdunTr1q0+1q+5OAAALeHPIExOTn7qqadSUlISEhLKy8tXrFjhnO464bdo0aIjR4706tXr6NGjCxYscE7s1avX5s2bk5KStm7dGhkZ6WP9mosDANAS/jxHaDKZZs6c6Xn9a9fneTp27Lhlyxa36rhx45w3unfv7nbJULcPAmkuDgBASwTxlWUAAGg5ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACA1ghAAIDWCEAAgNYIQACC1UKMbCDI5H21/4613TCaTZ8lms+nfDwCghQjCpln+7HPfX3W36HalRm3Xdt3bAQC0FEHYdH2vE72vNroJAIB/cI4QACC14A7C8vLylJSUuLi4CRMmlJeXG90OACD4BHcQrlixIikpqaioKDExceXKlUa3AwAIPsF9jjA7OzsnJyciImLWrFmpqanLli1rWFVVtaamprS01G2pyMjIiIiI5m5TFfXVos7L0efFSqEqHhOrhVC1F7HVC4eiXVJswm7RLqkOYa3TLKmqEPU13turEmaPR/xitcUOxI4AABDGSURBVBBCexFrneqtPbtVmLzcKYdDWC966UEVllpfo1cX5T6xrspHe8Lh8DJ6VtVu0x4ihyKs9V5Gz2d79VUaJWu9r/ZUL6NnswhFuz3hsPt83H2MnlZ7TnUVwvNzzpYaoXp5BO31vtqzeRs9r7ulEMLXs6auQoTVa8yvenlwm/WsUR2q9/Z8P6mrhFC12mvek9rq/cH19qwRPp81VSLM49XMx2uOtc7rwNqtakiI92dNk5/UVkUdft0NISFmz1KHqPY/HP9Wa23NFxLS/OM6k6p6PMbBIzo6+vz58+3bt7948WLXrl2rqqoaVtPS0rZt2xYeHu621GOPPTZ9+vTmbXHcxDuPfvONZsliqY+IaOc5XVVVm6U+vF17z5JDsSuKEhaukcqKzaqaTKGhYZ4lu91uMpnMZo3dy6Yo5hBTiEljh7BYLRHh4UK4vyCqqmq1XIxoF6nVnqIodq/tCVNomGZ7NpMpxEt7DrNJe3+1Wq1h4WEmj/aEqlq8tedQ7DZbuNaYK3abqgov7VlNphCz538InKNnMmm3Z7OFhYZqfm3GcrE2or1HfguhOhSb9/YcqhoW5r5nCiHsVoswm0O9tic0X1asNntYqNlLe3UR7TVGT3U4bDarZnsOu11RHdrt2SzCZA4N1WrPbg/xslta7fYws5f26i9GaD01VNVhs1i8P2scYR7Pa/HvZ02IZnu+njV2xWz28qzx/qS2Wuo1O2/uk9r7s8aumEO0d0uL1RoRHua3J7Xdqoqmt+f9SW2xWsPDwjQf96jI9keP/NNzekt07Nix2Uc4wX1EqKqqc5RVVVUU90Ox6Ojov/3tb/fdd58ft5j7webY2NgwrVdYqdTV1dlsttjYWKMbMV5RUVH37t2N7sJ4FRUVERER7dtrZINUrFZrdXV1fHy80Y0Yr6SkJD4+XjM+A01wnyPs0aPHmTNnhBCFhYUJCQlGtwMACD7BHYQpKSnp6emqqqanp6emphrdDgAg+AR3EC5atOjIkSO9evU6evToggULjG4HABB8gvscYceOHbds2WJ0FwCAIBbcR4T6e+GFF06ePGl0F8bbu3fvhg0bjO7CeIqizJ8/3+guAsLbb7998OBBo7swXkFBwcsvv2x0FwFhyZIlZWVlRnfRKARh0+Tm5hYXFxvdhfGOHz/+2WefGd2F8RwOR0ZGhtFdBIRPP/30+++/N7oL4/34448ff/yx0V0EhE2bNlVXVxvdRaMQhAAAqRGEAACpEYQAAKkRhAAAqQX31yd8s9lse/bssdvtflznhQsXtm7d+t133/lxncHoH//4x6lTp9544w2jGzGYoigOh4NxEEKcPHly7969mheWlMo333xTXFzMLiGEqKury8rK6ty5sz6bGzNmTGJiYvOWDe6LbvtWUFCwePFizSvwNltdXV27du1acpnztsFmsymK0q6dxvWIZVNTUxMdHW10F8arr68PDQ3179MtGCmKYrFYIiM1Lngtm9ra2sjISN3+b/Twww8PGzasecu25SAEAOCSZD+yAQBIjiAEAEiNIAQASI0gBABIjSD0pby8PCUlJS4ubsKECeXl5U2qtjGXvLOKovTr10//xvTkexBycnIGDRrUsWPHG264oaCgwJAOdeN7KHJzcwcMGNCxY8cBAwa07QtvNuZF4JtvvomKitK5Mf35HopRo0aZ/uOhhx4ypEMfCEJfVqxYkZSUVFRUlJiYuHLlyiZV2xjfd3b16tXXXXfd8ePHDelNNz4G4fTp01OmTHn11VeLioomTJjwwAMPGNWkPnwMhcPhmDx58po1a8rKyp555pm2PRSXfBGorKycNm1aXV2d/r3pzMdQqKqan59/9uzZ6urq6urqF154wagmvVLh3ZVXXnns2DFVVY8dO3bllVc2qdrG+L6zO3fu/OCDD9r87uRjEHbt2vXggw86b5eUlMTHxxvQn458DIXFYtmyZYvD4aiqqnr//fcHDBhgUI968P28cDgcEydOzMzMbPNPDdXnUBQVFUVHRw8fPjw6Ojo1NbW4uNigHr1q+w9PS0RFRdXV1amqWldXFxMT06RqG9OYO9vmn+2NGQS73f7QQw/NnDlT39b0dsmhcP7+jslk2rdvn+7d6cf3OCxbtmzevHmqBE8N1edQHD58ePTo0YcPHy4tLZ06deo999xjUI9e8daoL6qqOi+LoKqqoihNqrYxUt1Zby45CJ988snIkSNjY2NXr16te3e6uuRQREdH19TULFmyZM6cObp3px8f47Br167c3Nxly5YZ1JrefAzFkCFDdu7cOWTIkLi4uOXLl2/bts2gHr0iCH3p0aPHmTNnhBCFhYUJCQlNqrYxUt1Zb3wMgqqqTzzxxDPPPLN+/frly5e3+SuN+RiKkydPPvroo0KIqKio6dOnHzt2zJgWdeFjHHbs2LF79+7w8HBnPJhMpr179xrTpS58DMWhQ4f279/vvB0eHh4REWFAfz4RhL6kpKSkp6erqpqenp6amuqcmJeX56PaVvkeCkn4GIT9+/dnZ2e///77PXr0qKmpqampMbLR1udjKHr06PH666/v3r1bVdUNGzYMHTrUyEZbmY9xWLJkieudNyGEqqrJyckGttrafAxFbW1tWlrasWPHrFbr4sWLJ06caGSjmnR4+zV4lZeXjx07NiEhISUlpaKiwjnRNWia1bbK91Bo/tn2+BiEJUuWSPXM8r0/5OXlDRs2rFOnTtdee63zAxRtVWOeF5pT2h4fQ+FwOP7yl7/06dPnsssumzp1amVlpaGdauCi2wAAqfHWKABAagQhAEBqBCEAQGoEIQBAagQhAEBqBCEAQGoEIQBAagQhAEBqBCEAQGoEIQBAagQhAEBqBCEClMlkioiIqKiocJteUVERERHh/Gkb52z5+fmt3cygQYNMJtPrr7/u2aRr643spBkNm7xo0kqCi9VqHTlyZElJiWj0nuDi48HyNozFxcUjRoywWq2tdocQ0Nr4r6YhqFmt1k2bNv36179uODE7O1vnF6yjR48ePXo0KioqMzNz+vTpem7aZc2aNT179jRk04ZYs2bNzTff3KVLF+efjd8TfD9Y3oaxa9euv/jFL1566aU//OEP/rsTCB5G//wFoE0IMXTo0DFjxrhNv+WWW5w/cef8s7q6WlGUVu1k4cKFsbGxf/7zn81m84ULF9yadP3MUMPbPjRythYuEtRsNluvXr1OnDjh/LORe4JTIx8sTz/88ENiYqLdbvfPfUBQ4a1RBK577713586dRUVFriklJSU7duyYNGmSa0p0dHRISCvuxqqqbtiw4a677rr33nsVRcnOzm69bcFp27ZtPXv2vPzyy11TGrMniJY9WL179+7evfu2bdta3D6CD0GIwHXrrbd26tRpw4YNrilZWVkdOnS49dZbXVPcztIdPHjwV7/6VVxcXO/evRsu6HA4XnjhhYEDB0ZHRw8fPjwrK6vhGg4ePJiSknL11Vd79vDVV18VFBRMmjTp8ssvHzFiRGZmZmM6V1V17dq1/fv3j4yMHDp06Lp161StH/5s5Gy+ufXvY50HDhwYM2ZMp06devfu/Zvf/Obzzz/XPNWan5/f8MSbjxX6GHBVVV988cUBAwZ06NDh+uuv37t3rxBiwYIFP/nJT1yLnz59OiQkJCcnx+0ebdu27cYbb2w4pTF7gmjug+Vy4403fvzxx01aBG2EMQeiwKUIIY4dO/bb3/525MiRronJycnTp08/duyYa9cV//3m5NVXX7127dr9+/dPnTo1PDy8pqbGWVq1alV0dPTKlSu3bt06f/58s9n8wQcfuJYaMWLE9OnT//73v3u2MX/+/G7dujnfMVu+fLnZbD5//rxbk56333rrrQEDBqxbt+6jjz5ybu6vf/1rk2ZzG4otW7Yc+2/19fWa/Xtb5xdffGE2mydOnJiRkZGZmTlp0qT4+HjNYWw4vJe8L94GfPXq1bGxsS+++GJWVlZaWprZbP7qq6+++eYbIcTnn3/unGfp0qVdu3a1Wq1u93fYsGFZWVlug3zJPaExD5aPYVRVdcOGDcOHD/ccf7R5BCEClPPlLy8vTwhRUFCgqurp06eFEJ988omPIHz++eedt6uqqlwlh8MRHx/vPJRxmj9//vXXX+9a6tFHH9XsweFw9O7de/bs2c4/v//+eyHEK6+84tak5+1hw4Z9//33rtn+8Ic/JCcnN2k2t6Hw9PXXX2v2722dt99++9SpUxuu1vlZEs+u3NLF933xNuDdunXLyMhwlhRFuf32299++21VVQcPHjx37lznPP369dMc+c6dO+/bt6/h3W/MntCYB8vHMKqqumfPni5dunj2gzaPIESAcr78KYqSkJDw9NNPq6q6atWqrl272mw2H0H4xRdfuK1BVdXi4mIhRGlpqau0Z8+e+Ph412x79uzR7OHAgQOeL5233HKL5ybcbkdFRbkt1bVr1ybN5jkUPgaqYf/e1tmlS5ddu3Y1XHDnzp2NCULf90VzwM+fPy+EcPusitOyZcsSEhIURXGO7bfffus5T2ho6JEjR9xWe8k9oUkPlqavvvoqNDTUxwxoq/j6BAJaSEjIpEmTMjIyFi5cuH79+l/+8pehob52Ws9XbW+rVRTF9Wfnzp01Z9uwYUPnzp13797tOme2bt26VatWnT9/3tsiTpGRkZ9++mn79u19t9HI2S6pYTPe1mm1Wt2+cmc2mzXXVldX1/gmNQfcbrd7W/8999zzxBNP7Nu3LzMz89prr+3fv7/nPB07dqypqXGbeMk9odkPlkt1dXWnTp0aMyfaGD4sg0B37733Hj9+fOPGjV9++aXbpwQbqXPnzvHx8Vu3bnVN2bJly6BBg3wv5XA4Nm7cmJqa2r9//37/cf/99zfm44gDBw4sLCx0LnLVVVetWbNm3bp1zZ6tSbyt86c//embb77ZcM6MjIyGf7ry77PPPmthk127du3UqdOOHTucfzocjhEjRixdulQIcfnll1933XXvvPPOu+++6+1LmT179mz4AVEXH3tCSx4sl6KiIqm+rIn/Y/QhKaBNNDjhdNVVV3Xq1CkpKcnhcKj//d6d8PLmpNufK1eujImJefbZZzU/LKP5jtm+ffuEEFu2bGk40eFw9O3b9+abb/a99ZycnJiYmOeffz47O3vq1KlCCM3N+ZhNcyh8D5TvdW7fvl0Icccdd6xfvz4rK2vy5MmxsbGuYUxKShozZkxubu6bb77p9uW8Rt4Xtz+XLl0aFxf30ksv5ebm3n///eHh4a6zcWvWrAkJCYmMjKyqqtK8RzNnznzyySc9V+tjT2jqg6XpiSee+P3vf+9jBrRVBCECVMOXrWeeeUYI8fjjjzv/bEYQKory3HPPOb8DMGTIEM8PJXo2MHv27JiYmIafKnSaN29eSEhISUmJ762//vrr/fr1i4yMHD58+Hvvvedtc95m8zYUjal6W2dubu4111wTHR19+eWXT5ky5dNPP3UN444dO/r16xcdHT1mzJh//etfbv9FbuR9cRvw5cuX9+3bNzIycuTIkdu2bXPNdu7cuZCQkGnTpnm7R9nZ2Q0/NNSYPaGpD5am5OTkzZs3+5gBbZVJbfr3lgC0Afn5+f3799f/FeDEiRO9e/fes2fP9ddfrzmDzWZLSko6cOCAnm9Unj59+pprrjl16lRYWJhuG0WA4BwhAJ3Y7fbq6urFixf369cvOTnZ22xhYWEzZ8587bXX9Ozt9ddf//3vf08Kyun/A4weD+bO6sD4AAAAAElFTkSuQmCC" + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlgAAAH0CAIAAABuMsSDAAAABmJLR0QA/wD/AP+gvaeTAAAgAElEQVR4nO3deXxTdb7/8W/SJd2g0MpaoQwglEVEATdQRNSrYCmI1xkEkRFnUFyBhyOOiAsMUJhRAcfLD6UjOoNAK6UoDIvQCoIbiqAIVBSklJaldG+znZzfH7nmdpKT0CU9Sfp9PR/8Eb6fnHM++eacvJuTzaCqqgAAQFbGQDcAAEAgEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpteQgPHXqVGlpqX/Xabfb+VI6IYTD4VAUJdBdBAWbzRboFoKCoigOhyPQXQSeqqp2uz3QXQSFEHq0bMlB+Pzzz3/00Uf+XWdZWRl7uRDCbDZXVVUFuougcOHChUC3EBQqKystFkuguwg8m81WXl4e6C6CwsWLF0Plb6OWHIQAAFwSQQgAkBpBCACQGkEIAJAaQQgAkBpBCACQGkEIAJAaQQgAkBpBCACQGkEIAJBaeKAbQOCZzWZvX44VHh4eGxurcz8AoCeCUBZnzpzJz8/3HDebzQ89NqPk7BnPkupQBlx19f7P9jR/dwAQMAShLIbflXbOFmEMi3AbV6y1lYUF4g2tn+n48VNL3lw9mgOAwCEIZVFccLLqhYOiVTv3wtl8Me+6QHQEAEGBN8sAAKRGEKIxvv322w5JXYxeLF26NNANAkB9cWoUjfH1118br/2tOuENz5Ip65lQ+VlqABAEIZrEwBkFACGPBzIAgNR4RggAQWThwoXr168PdBd+oChKWFhYIxbs3bv32rVr/d6PDyEfhAcOHBg2bFh1dXWgG2mJqkryj/8U1/Yyz4rdYlYGjde/I6DFKygouOuuu/77v/870I0ERn5+/oIFC3TeaGgHYXFx8csvv1xTUxPoRlqosjP2mETrrO0apX8+3pi/9ADUw+WXX3711VcHuovAMBoD8IJdCL9GaDabp02btmzZskA30pIZjGEi7jKNf+GmQLcGAP4RwkH4xBNPzJgxo2vXroFuBAAQwkL41GhmZubbb7/tvGwwGGpra6OiogLbEoQQQrFt+mjL6aJiz4pBiD8+PPWKK67QvykA8CaEg7CsrMx5wWAw8Anu4GH/+avcVh1zj7fyLEV9/s7tI28lCAEEFf+fGlUUJSUlRbM0dOhQw68eeeSRhq6ntLQ0NTU1ISFhzJgxpaVav5aAINFnhLjrT57/TO27BbozAHDn52eES5cuXbNmzbFjxzxLqqoePXr09OnT8fHxQojwcF+b1lxPenp6cnJyVlbWrFmzFi9evHDhQteaNVdis9l27NjheuLocu2111555ZX1v1F11dbWRkRERES4/5hRkMjds+/NlW9plsxms5831ogn4apqsVha2Lt8a2trW9gtapza2lpFUTg3Y7Vam7hL2O12P/YTilRVbcQEmkymxn1sUfg9CAcMGNCjR4/U1FTP0tmzZ61Wa1pa2rFjx0aOHLly5UrnS3p2u90Viq7LmuvJzs7OyckxmUyPP/54WlqaKwi9URTl5MmTnlPTpUsXb89ZL8lms9lstsYtq4PZL7z0ffvholNvjZryoe7taLDb7cE8gY0Q5LuEbmw2m9FoZCpsv2r0GhwOh9vIa0uX/TMrp2l91ZdRiBeeeWrMmDH6bE6TqqqNmMCIiIhgCcIRI0Z4KxUXFw8ZMuTVV1/t2rXrjBkznnrqqffff18IMW3atFGjRo0fPz49Pd1iscydO9fbegoLC5OTk4UQycnJRUVFl2wmKirqD3/4wwMPPND42+PBZrPFx8cH7TNCY1iY6Hur6HWzRu3dS5yLbjBDIxYxxMbGOk8JtBg1NTUt7BY1jqqqJpMpOjo60I0EmNVqNRqNTdklIiMj3UYOHT76jamvuOruprVWv63nvXH69GkdNuRDEyewEfR7s8zAgQN37drlvLxo0aJ+/fo5Ly9ZsuTOO+/MyckpKSnZsGGDjzWoqmowGJwXFEVp7oYBIFh07CX63KrDdsIOZmuOGwyGCRMmrFmzxjUyceLENWvWuE6G2+32pKSkhISEH374wflA7VzK7VlacJ741e9zhN98882+ffuclyMjI02m//1EdkJCwrhx49atWzdlyhTXoKbOnTsXFBQIIQoLC5OSkpq7YQCAS15ensVicV62Wq15eXl1q5988slll1124cKFw4cP1x23/yfdum0QPZ4R5uXl3XLLLdXV1ffee29eXl6PHj3mzZs3duxYZ3XFihW5ubn5+fljx46NjY0dNWqUt/WkpqZmZGQsWLAgIyMjLS1Nh87hXw6bedU/3tmZm+dZMhrE7Gef5RwjELSGDRu2Y8eOu+++Wwixa9euoUOHZmZmuqrr169/8MEHT5w4kZmZ2b9//8C12Rh6BOGIESNUVR02bNiLL76YmppaXl4+atSo5cuXO6tdunTZuHFjTEzMli1bNN9u6jJ37tyJEyd26dLlmmuuee+993ToHP5VU3xqfZsewqLxMlLk9r9Nf/RRghAIWvfcc88HH3zgDMLs7Oxx48a5gtBms23cuPGbb7758ccfH3300Zdeesl1djQkNEsQur2F2vlfg8Ewffr06dOnu1159OjRzgudOnXq1KmTj/W0adNm8+bN/m8Xehpyn+h3u+dwxKcr9e8FQP2NHj161qxZNpstLCxs69atixcvdpV27do1YMCApKSkjh07lpeXHz582PWksG4ivvXWWw8//LDefddDCH+zDFoSi0MMGHKD5hfPt2kV99OxH/RvCUBdrVq1GjRoUF5eXkxMTP/+/euev1m/fv3HH3/syry6Z0dD4qOlBCGCglJdXjbz3yK+k3vBUl254DrXiXQ3HTp0uO+++5q9OQBCiF/PjsbFxY0bN841aLVas7Ozf/nlF+dPIGzbtu3pp58OrbOjBCGCRnxnkXC5+2B5sU1x/OnDo55Xd1Sc+435F4IQ0M2YMWPmzJkTExPzpz/9yTW4Y8eOrl27un4IaPjw4adOnap7djT4EYQIcqpQhfm+1zUqJ75SNz+tez+AvBISEvr27WuxWNq3b+8aXL9+fd1vAYuKiho5cmRovXeUIAQAXILrpb7t27e7Da5evdrtyps2bXJbKsgRhAhZ5srTxee79blKszhhfNrC+a/o3BHQHBISE0y73o345XMdtmX/+SshBuiwoaBCECJkXSwwOwy/3JehUTr44Y8nTurdD9A8Frz0wtX9Gvk7AQ1317XXXqvXtoIFQYhQFmESXbT+ei34VlhO6t0M0DxMJtOkSZMC3UVLpt93jQIAEIQIQgCA1AhCAIDUeI0QLdMvPx9ftmyZZmno0KGDBg3SuR8AQYsgREv085cHTp777qN8z4rj+GfzamsJQgAuBCFaIsVmSL7act9rnpXwDX/Wvx0AwYwgBIAgEh0dnZmZefSoxvfryuDChQv6b5QgBIAgMn/+/LfeeivQXfhBZWVlbGys5m+r+dCzZ8+631yqD4IQAIJIdHT0k08+Gegu/ODcuXOJiYlhYWGBbuTSCEJIx2az1dTUaJZMJlNIHLcA/IggDD2Kohw/flyzZLFYdG4m5Kjnfp770qaX5y/wLDls1uzsDWPGjNG/KwABRBCGntf+vuKFl+ZFxMZ7lqpKz4vb9O8olKhlZ9SxL9vvfMazFL/yHv37ARBwBGHoOXToO/PoF8zD/+hZCnu2m6J/QwAQyviKNQCA1AhCAIDUODUK/C/FUpO+9O/vrc/2LEUYxet/TW/fvr3+XQFobgQh8L/MxSf2tRstIq7yLEVtfP6Vuc8ThECLRBACdfS5VVx1t+dw+PZF+vcCQB+8RggAkBpBCACQGkEIAJAarxECl6Yq9ry8vPx8jV/6LSsru//++/VvCYC/EITApdWazU+/tjo8Ks6jolYe3EkQAiGNIAQuTbWZqye+Jdr3cC8oNuNjGl/6CiCEEIRAU50+fVpzPDIyko8eAsGPIASawKE4TLG9r7nes6LabXGxsWPHaHwqUQhx45BBU6Y82MzNAagXghBoAtUhaitrlp7XKO3PMv/ribcu/kajdOrA6eJ/E4RAkCAIgeZiiI5XRz6uUfhqvbjwoe7tANDG5wgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFLjc4RAACh2e3l5uWYpKirKZDLp3A8gM4IQ0N2Fkzs/3tE+qatnRbFa5s7589y5cz1LFRUV27dv97bKO+64o3Xr1v5sEpAGQQjorrzI0Hu4dfoHGqXsuVu2f1xSVuFZ2Z736clKR+RlXTxL5u8+/uaLff369fN7p4AMCEIgmPy494vI6C8K2nlWjCcLHPf9zTx4vGep9fyBzd8Z0GIRhECQ6XG9+K+ZnsPGT1c59G8GkADvGgUASI0gBABIjVOjQMiz2mwTpj4aHR3tWYoKE3k7thoMBv27AkIFQQiEPEtFyXe9ZohEjc9jiKWpS5Ys0QzCxMTEhx56qNmbA4IeQQi0CD1vFJ37aowbjHNyz3oOq9UX2xd+RhACgiAEWjiHYrtngTB4vBug+Jj4x72BaAgIOgRhkCorK3v4iZnm2lrP0sGDh8T1V+rfEgC0SARhkHr33Xc3HSy0Xf+AZ8lY8aX+/QBAS0UQBi9jxyvEtb/1HDdsW6J/MwDQUhGEgKSs5tr33ntPs9S1a9fhw4fr3A8QKAQhIKXS0yVlFdPf2uZZsZcUDP1NAkEIeRCEgJTsNmGKrZr8D43SgRzHyX/p3hAQMHzFGgBAagQhAEBqBCEAP1jxj/diYmIMXnz99deBbhDwitcIAfjB7j2fiqnviOvv9yzFp1+nfz9A/fGMEAAgNZ4RAmgAVVVVVdWseF1Ese/evbuwsFCzevfddxuN/EWOQCIIAdTXnj17brllhEN1aNRMsWLiaM2lqivKnl/5QWSrtp6lih8+7ZPSx6AVhL+5vNOHGzKb1i9QLwQhgP9UW3Hg+x/6DRnqWblQdFq9c5YYN9+zFP7iALuX9ak2c+29f6tNvkajNi3mh1teEuGR7uPlZ4s3PduwtoHGIggB/Keio5Xh8T+MXKBRWv+MQfu8aBP0uF5ERLsPlvzi560A3hGEANwZYtqInjdqFKLjde8FaHYEIYDgY60pr7UmdOqiWbxj5K1r/7la547QghGEAIJPdalis5bO/kqj9P2/d+554+b/StVc7sH7xk2d+lDz9oYWhyAEEJQMRpFwucZ4+dmLNuOeXlM1St/mXPX94ebuCy0PQQggxBjiEsRVWh/VOJsvxFnd20HI43OsAACpEYQAAKkRhAAAqfEaIQD4044dO6qqqjRL/fr1u+GGG3TuB5dEEAKANrvdbrFYNEtGozE62uMLcYT47LPPpj49W/S5VWNthUcmjxxEEAYhghCAFGw2m+b4x3l7Vry9SgiDZ2nv51+Wnyv0LKiqGh8ff+FskeciFoslvF23ion/T2NLu948eWLb2rVrNdsYPHhwz549fd2AhqioqPD2rDQyMvKyyy7z14ZaBoIQQMu3YtXq6X+cajBopJ3DFCdGPi7aa4XQmSzxtwIR08Z9vLzItqjhvzZ88uvcn499sTzbs2I78fVfn5/hryCsqam58tphFy5c8Cw57LY+vXt988Vev2yoxSAIAbR8ebt3qw+8qQ77vWfJ+Eyyo9/toofWGcvV0/zZhLVGpNxaOelNz0rUuif9uB2z2Xz2zGnLq1ofqTy+z7Lzz37cVstAEAJoKSrO/SN37foPNmpUqqrEmIY/h5OM1Wr18qvLIjw8PCwsTOd+dEMQAmgpio/W9r6terTGM56wN9L0bye0fPXVVzffervNUutZUhVlyZLFM2fO1L8rfRCEAFqQ6HjRrrvnsCHMz491DsWel5fnOX7o0CGHQ/HvtvTx7bffGoeMV+5f4VmKzHpG/370RBACQAPVlFVbbOOeeNGzYr5w2hadoE8XP/744xdffOE5XlVVFaJhHCgEIQA0kN0qhCh76mON0vbXwvZn6tPFiLvSyi7rY4iIchtXzVU2m12fHs6fP5+Tk6NZqqysfOihh+LjQ+DHnAlCAAgw1Wb+1/oPvv7uB89SmEH8aebTV1xxhWepsuxi9aOvi/iO7oXzP4tD13jblmK3FxVpfAKyvLzcyxtlfJn4h8f2nCg3JiR5lmxfrE9LSyMIAQCXZj/57Wften5W2duzFJ23/P777tUMwsYoO3P8+PGe/TVi0lpdLgaNb+j6KioqzCOeEP3/y7MU84PWM+agRBACQBDodZMY8ajnsOm7Df7cStVFtW1SzdyvNUoZv/eWB46Lp2fPefv5F1/xLFlFuBjgzwYDgiAEgOCl2m27du0qLi72LNns2l8a5/8eSk7Z73zWdssjnqWweYP16aFZEYQAELyqS4qXfLA7MvdHz1JtrVm/PiKiNL5qTgjN72gNOQQhAAQv1W6xjJpjSblFo7ZfM5nQYPwwLwBAajwjDKSKioppTz9bUHDKs1RUcFJJuln/lgBANgRhIH3xxRcffvJZ9eiXNWon08Ma8aEeAEADEYQBFt7qMjFglEZh72rdewEAGfEaIQBAagQhAEBqBCEAQGoEIQBAagQhAEBqBCEAQGoEIQBAanyO0G9KSko0vyFeCBEREdGrVy+d+wEA1AdB6DdDb7/7zIUyg9H9Sbaq2CNtVReKCwPSFQDAN4LQb07mH7YsPCGiW7sXys60WXJjIDoCAFwarxECAKRGEAIApEYQAgCkRhACAKTGm2X0oNhtH3zwgef4wYMHFcWufz8AABeCsPlVlVTVWh5a8i/PiuVMvt0Up39HAAAXgrD5KTZhCKt4eJ1G6d9Lwg5t0r0hAMD/4TVCAIDUCEIAgNQIQgCA1AhCAIDUCEIAgNQIQgCA1EI+CA8cOBAbGxvoLgAAoSq0g7C4uPjll1+uqakJdCMAgFAVwkFoNpunTZu2bNmyQDcCAAhhIRyETzzxxIwZM7p27RroRgAAISyEgzAzM3PEiBEGg0EIYTAYzGZzoDsCAISeEP6u0bKyMucFg8GgqmpgmwEAhCj/PyNUFCUlJUWzVFpampqampCQMGbMmNLS0oaup0GLAwBQH35+Rrh06dI1a9YcO3ZMs5qenp6cnJyVlTVr1qzFixcvXLiwQevxtri3p4O1tbVvvPHGRx995DY+fvz422+/vQG3qo6ysjKHwxEREaFRa9yz0tB9KtuYzvW7td7uDR8dNENz/l2ljrOnyyJ6rtLn0Rncu3IQ8HnKzXtJVcvLy3V70hIXF6f9yFwPfg7CAQMG9OjRIzU1VbOanZ2dk5NjMpkef/zxtLQ0Z5LZ7fbw8P9tw3VZcz2ai/sQERExYMCAG264wW38iiuuiI6ObtwNjI6Ojo6O9jLdhsassVEL4ZIMXibWoOtDGPduSGjM3aTvjhRgBm+H06VERUU1+sG2oYzGxp/g9HMQjhgxwke1sLAwOTlZCJGcnFxUVOQcnDZt2qhRo8aPH5+enm6xWObOnettPZqL+xAeHn7zzTc/8MADjbgh3phMpqioKO0glO1Br5GPHsErqJsTQs8GG7GlZmjOn6ts7CM5Lsn7zBoMzgdMHZtpJF3fLKOqqvMvC1VVFUVxDi5ZsuTOO+/MyckpKSnZsGFDQxcHAKApdP34ROfOnQsKCoQQhYWFSUlJzsGEhIRx48atW7duypQpJpOpoYsDANAUegRhXl6e80JqampGRoaqqhkZGWlpac7BFStW5Obm5ufnL1iwYMuWLT7Wo7k4AABNoUcQul7wmzt37qFDh7p06XL48OE5c+Y4B7t06bJx48bk5OQtW7bExMT4WI/m4gAANEWzvEbo9l5b13/btGmzefNmtyuPHj3aeaFTp06dOnXysR7NxQEAaIoQ/oo1AACajiAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASC2Eg9DhcEyZMmXw4MEpKSmZmZmBbgcAEJJCOAh37Nhhs9n279+fmZk5ffr0QLcDAAhJIRyEV1111aJFi8xmc0FBQadOnQLdDgAgJIUHuoHG69ixoxDi7rvv3rx58/bt2wPdDgAgJIXwM0JFURwOR05OTlZW1iOPPBLodgAAISmEg/DNN9989tlnw8LCevToUVNTE+h2AAAhyc9BOHToUMOvPJ+llZaWpqamJiQkjBkzprS01PeqFEVJSUnxsfjkyZN/+umnAQMGTJgwYeXKlf69IQAASfjzNUJVVY8ePXr69On4+HghRHi4+8rT09OTk5OzsrJmzZq1ePHihQsXelvV0qVL16xZc+zYMd+Lb9iwwUc/VVVVM2bMeP75593Gn3766UmTJjXstv3q4sWLFoslIiJCo6aq3pfzXvKxUMP5aqFxK2xkzZ/LNI63qfB1J/m/O69rVBuzMR1nr+Hbaobm/LlKn/Md+F1Zv7vWJ29t+Nxdfe3kJSUlcXFxTeyqnuLj400mU+OW9WcQnj171mq1pqWlHTt2bOTIkStXroyKiqp7hezs7JycHJPJ9Pjjj6elpTmD0G63uyLTdXnAgAE9evRITU295OI+xMbGzp49e+zYsW7jCQkJzqhuBFVV4+PjtYPQYPC+nPeSj4UazlcLjVthI2v+XKZxvE2FwceB6//uvK7R0Ji7SsfZ03Fb3vmzB5/z3chd2Y/pFQzTLby34XN39bWTt23bNjExsYld1ZPR2PgTnP4MwuLi4iFDhrz66qtdu3adMWPGU0899f7779e9QmFhYXJyshAiOTm5qKjIOTht2rRRo0aNHz8+PT3dYrHMnTtXCDFixAjP9Wsu7oPBYOjQoUPPnj2bftNcwn7lx3UiSATJg1GIYvbgyWg0hsSjpT+DcODAgbt27XJeXrRoUb9+/dyuoKqq8y8LVVUVRXEOLlmy5M4778zJySkpKfF9qlNzcQAAmsKfb5b55ptv9u3b57wcGRnpebq2c+fOBQUFQojCwsKkpCTnYEJCwrhx49atWzdlyhTfZ3g1FwcAoCn8GYTV1dXjxo07cuSI1WqdN2+e68W5vLw854XU1NSMjAxVVTMyMtLS0pyDK1asyM3Nzc/PX7BgwZYtW3ysX3NxAACawp9BOGzYsBdffDE1NTUpKam0tDQ9Pd057nrBb+7cuYcOHerSpcvhw4fnzJnjHOzSpcvGjRuTk5O3bNkSExPjY/2aiwMA0BT+fI3QYDBMnz7d8/uvXW+9bdOmzebNm92qo0ePdl7o1KmT21eGur1nV3NxAACaIoS/WQYAgKYjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSCw90AyFm45btGe/+02gweJZsNpv+/QAAmoggbJj0v712POU+0aGXRi3vY93bAQA0FUHYcD1uEN2vC3QTAAD/4DVCAIDUQjsIS0tLU1NTExISxowZU1paGuh2AAChJ7SDMD09PTk5uaioqGvXrosXLw50OwCA0BParxFmZ2fn5OSYTKbHH388LS1t4cKFdauqqpaUlJw6dcptqfj4+Li4uEZuUlVF5QVRdka7Wl4sLBWAbDcAABFESURBVFXug5XnhXBoL2KuFHabdslWKyzV2iWHImrKNEuqKkRVidf2Ks4KxeOtrRXnhapqL1Jbripe2rPWCDVSu6TYRW25dkl1iOqLXtpTRcVZYfT4y6zinBBe2qspE4rdW3uqrVZ7ihSrqK30Mns+2hOi4qyIjPbYkFkIob1I9UWhKtolS7VqM2uX7BZhqfJ6v/tq75yIaaNdKisSnu9zri7x3l6lard4b097t1Qdiqgp9dpe5XmvpbIiERHlPlh1Xqg+jhqrl6PG6+yp3o8aoTp8HTXlxcJa4z5Y6eOoqRDejhpbrbCavLTn46hRvbanClFxVqgO9/GKs16Pmtpyr+1Za1WjwctRY/N21Pg4qK12x4BB1xoMGk+3WsXGFJz8SWNtTWA0Gg1a7+evD4Oqqv7tRk9xcXHnz5+Pjo6ura3t0KFDRUVF3eq4ceNyc3Ojo90fvGbOnPnggw82bovj7pvw/eEfhNZsV1VUxLVq7VlSHY6aqqrY1q09F1FsVqvVFh0b61mymWsdwmiKMnmWLLW1xrDwiMgIz1JtTW1kZGRYeJhnqbqyMiYuznNHUVVHTWVlbOt4jfbsNqvF6q09RTVERXs8fglhqa0xhkVotmeuqY3w1l5VVUxMrMHo2Z5aXVEeF6/xEK/YbRazJUbrbxqbxexwCJNWe9baGhEWHhkZqdFebW14RGS4dnvVMTExnu0JIarKSuPatPUcdyh2c61Zsz271WJXHFEee6YQwlpTLcIiIk1e2guPCI/Q+OO1promKirKGKbxiFNVVhoX39Zzt3QoirmmJqZVK+327EpUTIxGe7U1whjurb2w8PCICI373Vd75eVxreM12nMotdXVsa00jhq7zWq32bXbM9eoIkzzqPHRXm1NTaQpKkyrveqKilivB7WXo8Zqtdp0PKhj47SOGp8HtdkaHafVnsXsUIUpSvOoqTV4ac9cY46IjPDSXlVMrMZBLYSIi4468NUXnuNNER8fbzJpzG19hPYzQlVVnY/sqqoqiuJWjYuLW758+QMPPODHLeZkrYuPj9c8nKRSU1Njs9ni4zWONNkUFRV16tQp0F0EXllZmclk8vy7UzZWq7WysjIxMTHQjQTeuXPnEhMTw8I0MjLYhPZrhJ07dy4oKBBCFBYWJiUlBbodAEDoCe0gTE1NzcjIUFU1IyMjLS0t0O0AAEJPaAfh3LlzDx061KVLl8OHD8+ZMyfQ7QAAQk9ov0bYpk2bzZs3B7oLAEAIC+1nhPr7y1/+cvz48UB3EXi5ubnvvPNOoLsIPEVRHn300UB3ERRWrly5d+/eQHcReIcPH16yZEmguwgKzz777IULFwLdRb0QhA2zZ8+ekpKSQHcReD///PO3334b6C4Cz+FwcE7Caf/+/Z6f2ZXQ+fPn+YPAaceOHdXV1YHuol4IQgCA1AhCAIDUCEIAgNQIQgCA1EL74xO+Wa3WTZs2Ob96xl+Ki4vffffd3bt3+3Gdoeizzz47c+bMggULAt1IgCmK4nA4mAchxNGjRxVFKS4uDnQjAfbjjz8WFBSwSwghKisr33zzzYSEBH02d++99/bq1atxy4b2l277dv78+VdffbXR30euyeFwGD1/IUE+zt3Gv3MbotglnNglXNglnHSeh0mTJvXt27dxy7bkIAQA4JL4swUAIDWCEAAgNYIQACA1ghAAIDWC0JfS0tLU1NSEhIQxY8aUlpY2qNrCXPLGKoqSkpKif2N68j0JOTk5/fv3b9Omzc0335yfnx+QDnXjeyq2bt3at2/fNm3a9O3bd/v27QHpUB/1eRD4/vvvY2NjdW5Mf76nYujQoYZfPfLIIwHp0AeC0Jf09PTk5OSioqKuXbsuXry4QdUWxveNXbp06Y033njs2LGA9KYbH5Nw6tSpSZMmvfXWW0VFRWPGjPn9738fqCb14WMqHA7HxIkTly9ffvHixVdeeaVlT8UlHwTKy8unTJlSU1Ojf2868zEVqqoePXr09OnTlZWVlZWVr7/+eqCa9EqFd7169Tpy5IiqqkeOHOnVq1eDqi2M7xu7a9euDz/8sMXvTj4mITc39+GHH3ZePnfuXGJiYgD605GPqbBYLJs3b3Y4HBUVFZs2berbt2+AetSD7+PC4XCMHTs2MzOzxR8aqs+pKCoqiouLGzRoUFxcXFpa2tmzZwPUo1ct/+5pitjY2JqaGlVVa2pqWrVq1aBqC1OfG9vij/b6TILdbn/kkUemT5+ub2t6u+RUVFZWCiEMBsPevXt1704/vudh4cKFs2bNUiU4NFSfU3HgwIERI0YcOHCgpKRk8uTJv/vd7wLUo1ecGvVFVVXnN2WoqqooSoOqLYxUN9abS07Cxx9/PGTIkPj4+KVLl+rena4uORVxcXFVVVXz589/6qmndO9OPz7mITc3d+vWrQsXLgxQa3rzMRUDBw7ctWvXwIEDExISFi1atG3btgD16BVB6Evnzp2dX1VaWFiYlJTUoGoLI9WN9cbHJKiq+txzz73yyitr165dtGhReHhL/hZf4XMqTp48+cwzzwghYmNjp06deuTIkcC0qAsf87Bz585PPvkkMjLSGQ8Gg+HTTz8NTJe68DEV33zzzb59+5yXIyMjTSZTAPrziSD0JTU1NSMjQ1XVjIyMtLQ052BeXp6Pakvleyok4WMS9u3bl52dvWnTps6dO1dVVVVVVQWy0ebnYyo6d+68atWqTz75RFXVdevWXX311YFstJn5mIf58+e7zrwJIVRVHTZsWABbbW4+pqK6unrcuHFHjhyxWq3z5s0bO3ZsIBvVpMPp19BVWlo6atSopKSk1NTUsrIy56Br0jSrLZXvqdD8b8vjYxLmz58v1ZHle3/Iy8u75ppr2rZte8MNNzjfQNFS1ee40BxpeXxMhcPh+Pvf/96jR4/LLrts8uTJ5eXlAe1UA1+6DQCQGqdGAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgRpAwGg8lkKisrcxsvKyszmUzOHzt1Xu3o0aPN3Uz//v0NBsOqVas8m3RtvZ6dNKJhgxcNWklosVqtQ4YMOXfunKj3nuDi487yNo1nz54dPHiw1WptthuEoNbCf0cbIc1qtW7YsOGhhx6qO5idna3zA9bhw4cPHz4cGxubmZk5depUPTftsnz58ssvvzwgmw6I5cuX33bbbe3bt3f+t/57gu87y9s0dujQ4dZbb33jjTdmzpzpvxuB0BHoH0QEtAkhrr766pEjR7qN33777c4fPXf+t7KyUlGUZu3khRdeiI+P/8tf/hIWFnbhwgW3Jl0/PFv3sg/1vFoTFwlpNputS5cuJ06ccP63nnuCUz3vLE8//fRT165d7Xa7f24DQgqnRhG87r///l27dhUVFblGzp07t3PnzgkTJrhG4uLijMZm3I1VVV23bt299957//33K4qSnZ3dfNuC07Zt2y6//PJu3bq5RuqzJ4im3Vndu3fv1KnTtm3bmtw+Qg9BiOB1xx13tG3bdt26da6RrKys1q1b33HHHa4Rt1fpvvrqq9/+9rcJCQndu3evu6DD4Xj99df79esXFxc3aNCgrKysumv46quvUlNTr7vuOs8eDh48mJ+fP2HChG7dug0ePDgzM7M+nauqumLFij59+sTExFx99dWrV69WVbXRV/PNrX8f6/zyyy9HjhzZtm3b7t27/+EPf/j88881X2o9evRo3RfefKzQx4Srqrps2bK+ffu2bt36pptu+vTTT4UQc+bM+c1vfuNa/NSpU0ajMScnx+0Wbdu2bfjw4XVH6rMniMbeWS7Dhw/fvn17gxZBCxGYJ6LApQghjhw58sc//nHIkCGuwWHDhk2dOvXIkSOuXVf858nJ6667bsWKFfv27Zs8eXJkZGRVVZWztGTJkri4uMWLF2/ZsmX27NlhYWEffviha6nBgwdPnTr1X//6l2cbs2fP7tixo/OM2aJFi8LCws6fP+/WpOfld999t2/fvqtXr/73v//t3Nz//M//NOhqblOxefPmI//JbDZr9u9tnfv37w8LCxs7duyaNWsyMzMnTJiQmJioOY11p/eSt8XbhC9dujQ+Pn7ZsmVZWVnjxo0LCws7ePDg999/L4T4/PPPnddZsGBBhw4drFar2+295pprsrKy3Cb5kntCfe4sH9Ooquq6desGDRrkOf9o8QhCBCnnw19eXp4QIj8/X1XVU6dOCSE+/vhjH0H42muvOS9XVFS4Sg6HIzEx0flUxmn27Nk33XSTa6lnnnlGsweHw9G9e/cnn3zS+d/jx48LIVauXOnWpOfla6655vjx466rzZw5c9iwYQ26mttUePruu+80+/e2zrvuumvy5Ml1V+t8L4lnV27p4vu2eJvwjh07rlmzxllSFOWuu+567733VFUdMGDAjBkznNdJSUnRnPl27drt3bu37s2vz55QnzvLxzSqqrp79+727dt79oMWjyBEkHI+/CmKkpSU9PLLL6uqumTJkg4dOthsNh9BuH//frc1qKp69uxZIURJSYmrtHv37sTERNfVdu/erdnDl19+6fnQefvtt3tuwu1ybGys21IdOnRo0NU8p8LHRNXt39s627dvn5ubW3fBXbt21ScIfd8WzQk/f/68EMLtvSpOCxcuTEpKUhTFObc//PCD53XCw8MPHTrkttpL7gkNurM0HTx4MDw83McV0FLx8QkENaPROGHChDVr1rzwwgtr16697777wsN97bSej9reVqsoiuu/7dq107zaunXr2rVr98knn7heM1u9evWSJUvOnz/vbRGnmJiYPXv2REdH+26jnle7pLrNeFun1Wp1+8hdWFiY5tpqamrq36TmhNvtdm/r/93vfvfcc8/t3bs3MzPzhhtu6NOnj+d12rRpU1VV5TZ4yT2h0XeWS2VlZdu2betzTbQwvFkGwe7+++8/duzY+vXrv/76a7d3CdZTu3btEhMTt2zZ4hrZvHlz//79fS/lcDjWr1+flpbWp0+flF89+OCD9Xk7Yr9+/QoLC52L9O7de/ny5atXr2701RrE2zqvvPLKd955p+4116xZU/e/rvz77LPPmthkhw4d2rZtu3PnTud/HQ7H4MGDFyxYIITo1q3bjTfe+M9//vP999/39qHMyy+/vO4bRF187AlNubNcioqKpPqwJv5PoJ+SAtpEnRecevfu3bZt2+TkZIfDof7nuTvh5eSk238XL17cqlWrv/71r5pvltE8Y7Z3714hxObNm+sOOhyOnj173nbbbb63npOT06pVq9deey07O3vy5MlCCM3N+bia5lT4nijf69yxY4cQ4p577lm7dm1WVtbEiRPj4+Nd05icnDxy5MitW7e+8847bh/Oq+dtcfvvggULEhIS3njjja1btz744IORkZGuV+OWL19uNBpjYmIqKio0b9H06dP//Oc/e67Wx57Q0DtL03PPPffYY4/5uAJaKoIQQaruw9Yrr7wihHj22Wed/21EECqK8uqrrzo/AzBw4EDPNyV6NvDkk0+2atWq7rsKnWbNmmU0Gs+dO+d766tWrUpJSYmJiRk0aNAHH3zgbXPeruZtKupT9bbOrVu3Xn/99XFxcd26dZs0adKePXtc07hz586UlJS4uLiRI0f+/PPPbn8i1/O2uE34okWLevbsGRMTM2TIkG3btrmuVlxcbDQap0yZ4u0WZWdn133TUH32hIbeWZqGDRu2ceNGH1dAS2VQG/65JQAtwNGjR/v06aP/I8CJEye6d+++e/fum266SfMKNpstOTn5yy+/1PNE5alTp66//vpffvklIiJCt40iSPAaIQCd2O32ysrKefPmpaSkDBs2zNvVIiIipk+f/vbbb+vZ26pVqx577DFSUE7/Hz0kdvMCJ2OgAAAAAElFTkSuQmCC" }, - "execution_count": 10, + "execution_count": 46, "metadata": {}, "output_type": "execute_result" } @@ -369,7 +358,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 47, "metadata": {}, "outputs": [ { @@ -378,7 +367,7 @@ "0.0013128198764010824" ] }, - "execution_count": 11, + "execution_count": 47, "metadata": {}, "output_type": "execute_result" } @@ -390,7 +379,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 48, "metadata": {}, "outputs": [ { @@ -399,7 +388,7 @@ "0.07228069619249913" ] }, - "execution_count": 12, + "execution_count": 48, "metadata": {}, "output_type": "execute_result" } @@ -420,14 +409,14 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 49, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "609.049840 seconds (2.89 M allocations: 493.832 MiB, 0.36% gc time)\n" + "657.345223 seconds (176.95 k allocations: 361.775 MiB, 0.19% gc time)\n" ] }, { @@ -462,7 +451,7 @@ " -0.00277687 0.00341591 -0.000337529 0.000481492 0.499799 " ] }, - "execution_count": 13, + "execution_count": 49, "metadata": {}, "output_type": "execute_result" } @@ -490,17 +479,9 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 63, "metadata": {}, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "┌ Info: Recompiling stale cache file /Users/juhyun-kim/.julia/compiled/v0.7/CSV/HHBkp.ji for CSV [336ed68f-0bac-5ca0-87d4-7b16caf5d00b]\n", - "└ @ Base loading.jl:1185\n" - ] - }, { "data": { "text/html": [ @@ -574,31 +555,27 @@ "│ 6670 │ -1.24394 │ 0.213697 │ 2.74965 │ -1.80285 │ 1.11401 │" ] }, - "execution_count": 15, + "execution_count": 63, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "# Pkg.add(\"DataFrames\")\n", - "# using DataFrames\n", - "using CSV \n", + "#using Pkg\n", + "#pkg\"add CSV DataFrames\"\n", + "#using CSV, DataFrames\n", "\n", - "cg10k_trait = readtable(\n", - " \"cg10k_traits.txt\"; \n", - " separator = ' ',\n", - " names = [:FID; :IID; :Trait1; :Trait2; :Trait3; :Trait4; :Trait5; :Trait6; \n", - " :Trait7; :Trait8; :Trait9; :Trait10; :Trait11; :Trait12; :Trait13], \n", - " eltypes = [String; String; Float64; Float64; Float64; Float64; Float64; \n", - " Float64; Float64; Float64; Float64; Float64; Float64; Float64; Float64]\n", - " )\n", + "cg10k_trait = CSV.File(\"cg10k_traits.txt\"; \n", + " delim = ' ') |> DataFrame\n", + "names!(cg10k_trait, [:FID; :IID; :Trait1; :Trait2; :Trait3; :Trait4; :Trait5; :Trait6; \n", + " :Trait7; :Trait8; :Trait9; :Trait10; :Trait11; :Trait12; :Trait13])\n", "# do not display FID and IID for privacy\n", "cg10k_trait[:, 3:end]" ] }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 64, "metadata": {}, "outputs": [ { @@ -647,7 +624,7 @@ "│ 13 │ Trait13 │ -0.000151238 │ -5.5921 │ -0.141022 │ 24.1744 │ │" ] }, - "execution_count": 16, + "execution_count": 64, "metadata": {}, "output_type": "execute_result" } @@ -658,14 +635,14 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 65, "metadata": {}, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlgAAAH0CAIAAABuMsSDAAAABmJLR0QA/wD/AP+gvaeTAAAgAElEQVR4nOydeWAURfb4X/fck8mdyQUBBULCIQIiqyEgh6IiEF1RUVjQVVYEdgFRyQIGD+SUa/2xX5dL1FXDIRhcBbkRiHiAHAmQcEguyDn3Pd1dvz8axjgzmcz0TKZnJvX5AyZvuqte95uq19VV9R6BEAIMBoPBYNorJN8KYDAYDAbDJ9gRYjAYDKZdgx0hBoPBYNo12BFiMBgMpl2DHSEGg8Fg2jXYEWIwGAymXYMdIQaDwWDaNdgRYjAYDKZdgx0hBhOuUBQ1bdo0pVI5aNCgmpoaVqhWq8eMGZOQkDB27Fi1Wu1BiMFgWLg7QtwIMRh+WbNmjU6nq6ioyMnJWbhwIStctmxZ586db9682alTp+XLl3sQYjAYFoJziLX333//zJkz69evX7hwoVqt3rhxIwDk5+cbDIaVK1fOmTMnOjp6yZIlLQkdlJeXZ2RkyGQyDjrQNE0QBElyd+eRUQK/lJeXp6eny+VyXy+BoiiBQEAQRMieZbfbRSKRT6cEk/79+3/00Ud33323Xq8vLy+/5557ACArK6uoqCg7O/vSpUt5eXllZWUtCR0cO3bs+vXr48ePb7VGhmEQQgKBwEsNfT0eIUTTtFAoDP7xAoEgrNtgRkaGVCr16W644uv9DM0SOMDdEQaqEfbv33/jxo39+/fnoINGo5FIJNycKItWqxWJRHK5nHMJOp1OIBBERUX5U4JYLJZKpZxL4Jf+/fuvXr164MCBvhpCrVbLZDJfL7ypqSk6OlosFvt0VkNDQ3x8vK+tq66uTqlUhmz/mJiYOGXKlPXr13fp0uWjjz666667AEChUDQ0NMhkMrPZnJKSotPpWhI6yMrKSk9P37lzZ6s1ms1mu90eExPjpYYWi8VqtcbGxnp5vNVqNZlM8fHxXh5vs9kMBkNCQoKXx1MUpdFokpKSXL9SKBSh/NDjGbYX7d27t16vT0xM5FyO3W7XarVu70/QSqAoSq1WK5VKziVwgLvXraio+OKLL4YNG8Y2QlZYU1PTuXNnAGDfw3gQOtDpdCNGjLh06RIHHbRarUQi8ceF6HQ6kUgkk8nOnTtntVqjoqJ69uzpUwl6vV4gEPjjSvV6vVKpDF9HGGR0Ol1JSYlIJLr//vt9HeFFGDqdDiFUWlq6bt26KVOmnDx5EgAQQuxtYZ+s2SPdCh1ERUWtXLnSG/cjkUjsdruXjs1qtZ45cyY6OvqOO+7w8oqsVqtYLPbJEQqFQu+PpygKALw/HuMnGo3mwoULANClS5fU1FS+1WkR7o4wUI0wOjr6ww8/5Ob/RSKRnyNCsVgsEom+//77p154hYyKTZOTl86d9qkEiUTi54hQIpFIJBLOp7c3Xn71nz/8WmKsvGCxWHwdF0YYSqVy1qxZaWlpM2bMWLNmDStMT0+vqqrKzMysqanp0KGDB2Fb8+aiZatXr35wxPA9RV8Gp0ZMqPHMi9OKT5+ndI3vv7Ng+vTpfKvTItzf+TRvhCUlJayQbW8A4NoInYQOCIJISEggeaWmpgZ6PWiYuIlCBC8KcLZCO6Sy4rph/DqCFGzdunXv3r18q8MnDz/88JYtW6xW6/r16wcMGMAKx4wZs3nzZoTQ5s2b8/LyPAjbmqtXr1J3/onGed7aMRXXrhqe+xD6P863Iq3AvQsO8UaIiWwYgXjq+59Of3Uu34rwyZIlSw4dOpSSknLw4EF2tRoAFBQUnDt3LiMjo7S0dMGCBR6EGAyGhfur0SVLlvzlL39ZtmzZ3XffvWnTJlZYUFAwYcKEjIyM/v37f/rppx6EGIxf2C2mMW/TX07lWw8+SU1N3b9/v5MwLi7um2++8UaIwWBYuDtC3AgxQeZ86YXJL8+oqKzkWxEMBhNR4NmpZlj0Nxua0u7s3rlrd4Zh+NYG48y/P1z/qyjLRjmvt8JgMBh/wI6wGZobNoaofemrqutXOW+vDCMGDRpE3Gbq1FvvGEM9NlBKNyC83Z2NwWAw3oAd4R8RiiElk28lggFC6NKlS9XV1Xq9Xq/XOxbf4wBdGAwm4NA0bbPZXLfPhQjYEbZT6urqbDZbXl5eWlraxIkTHaFGdu3aNWPGDIlEMmPGDEeoEbfCYFJZWdl74OBPv9ga/KoxflJ3o2b58uV/f2N+5+w+//73v/lWB8MDdP31WbNnS2XyyS9O4VsX9wQ1nlsIYjab//Of/5w6dYqmuYeGCUdqa2vvvffeVatWderUafbs2TNnzvziiy+AU2wgs9n873//u0ePHmzYDu8xGAwURdnt9laP3L9//290jEXmHLcJIUav13tZl1Ao9DXEmsFgkEql3mz0lMlkQY6OCACDBg0qLi5mP7/88ssffvghAKjV6kmTJp04cSI3N/fjjz9mo6i4FbYper3eaDKCvr6kpi7/rfeAFKK07BsuPx5Me4DR1qLxq0Eks9M/8K2Le7iPCMNyhsmFuW++PX/Lnh3HztE+duLhTt++fQ8dOtS3b9+EhISlS5d+9913rJxDbCCCIBQKBdGWAAApiyFEf4y/Y9LcqK1PSkmb+eprbVq7lxoGmRB/uT1s9JP79h8AhiaUXRBlI0QSkEYHoV4MhgMcn2EdjZCNOuh4Fmbb244dO+bMmbN8+XI20YRbIe/877sD6zd9dPr0r9T9M+BqcXt7R3z69GmLxZKTkwMAYrHYEeONQ4AuqVQ6adIkhULha6w7u93uZdBtqVTqxtnoGxmxnHpsvtl+TqFQeC7BbDYrFApfB21Go1GhUIRm6B/Hy+2ysrIRI0asX7+evZO7du0qKipi32Pn5eWxzc2t0IHNZjt48KA3UXYtFovdbvcmImDZ+V9JRRINAPD7ujOKoiwWi4ezrFar1Wr1fExzbDabT8dTFNXS8SKRyPssGZgIg2MLD+AME8MwNTU1dNCZ9dobX9uzbuqcmsStRnv16tUrV66YTKbgKBP8RapGo/GJJ564ePGizWZ79913H3/8VgykcIsNRIKwnYZpZV9ub9y4saKiIjY2dubMmaycw8ttu92+e/dusxdYLBaLxeLNkQiguQtkP9vtlDfn8gLeMdWe4TgiDOAMk8FgePnll8+cOcNBDX+yT9A0A70fJk7dCgf8hyYrie4/5CGrqu7r3V+1mh+KzT7h/TOp2xKSkpL8CR3Ogdzc3IULF44ZM0ar1Y4aNeqDDz5g5Tg2ULjAvtxmPy9durRXr17sZ27ZJ9auXRvY7BMEQQCwg/jf/5XJpJ5raW/ZJyiK+sc//rF9+/bu3btv27aNfd0SIrO87QqOjjCAjTAmJmbbtm3Jyckc1GDf6XFzIbdeg9x+3/aH/y0645ra2FVD4uPjW1VMKpX6mX1CKpUGP4sCQRDTpk2bNm2akzwcYwPdrKn+8ssv+/fvf+edd/KtS/AI4MvtgHP48GHKbm/3S/FaZ82aNTqdrqKiYuHChQsXLmQDxobRBFPEwPHV6OnTpx3L1VwbIfiSfSJkYWyW/6xfX1BQ8NNPP/GtC6ZlKn89fubixOmv7du3j29VgkrIvtwuKSkZM36yxWr943sWjBs+//zz119/XS6XFxQUvPLKK6wwNLcwRTYcn9mMRuO4ceOOHDnStWtX10a4ePFi10boJAx9jKr6j66QwqN7U1JSBg4cyLc6mBawGqH7YFIa1BfLoUDIvty2Wq3CmCTQNLVpLZFBoNKb6/X6p59++vvvvzcYDP4sOLDb7V7uR/KyBPYVoEMhq83a2NjouQSKorRaLYeV2DExMZxfrXF0hCHbCANM7vPCn/7LtxIYjBsi6eV2uyVQ6c3lcnl+fn5sbCxJkt7M4LYEu6k3ICVcvXq1vLzcbDbD7xNPIBKJWi2coiiGYTjo4M+iX4KXoJqPPPJIXV0d+7mmpiYpKYlbinaGYTjv4qqqqaHkSWBoBGk02K0AAGIZmDQQ3wEar0PSHaCqgphkwmpIkItjYmI86AAAjhX2WVlZhYWFPmmi0+nEYjG3JT984WTBxMREiUTiqyG8NB9N00ajUWW0IMoOEgWYtRCdDNqbEK0EkxqEUgAgSCIxShId3eJONZqmSZL0RsPmFqyrq1MqlaG5fcJPuLVBhBBCyPMNsVqttQ2NDGUHUgAECQQJdiuQJAjFcXKJ51Ue3pTf6vEe2iBFUWq1WqlUell+W5Oenn7q1Km0tLTa2tpu3boZDAYAyMzM/PbbbzMzMy9fvvzYY4+Vl5e7FbpaUCwW+3T3XEEIMQzjj0dxlFBRWQUCEWO3QmwKGJpAGg1ARIElubWbjxDq0qVLkF//8jOdXVJSsmnTJm4LZEKZ8vLyxYsX861FMMAWDHewBUMBNr35q6++6prevNUJpgi24LvvvhvkSnlb19WrV6+OHTvyVXsbEZFDh5bAFgx3sAV5x8/05tiCgQIvcG7vlJSU/OlPfzIajeyfeA8TBhM0cHrzECGcnp4wAUer1T7//PMmk8khCZFIlRgMBhM0sCNsvyCEnn/++fz8/OZCvIcJABjf4VHbkpKS5vEcwi7wPQbDO/46QtwIw5dly5Z17dp13LhxzYUc9jBFGDRNN9ymqampwTusVisv2uIxPQbjP345QtwIw5fDhw/v3bvXNVAThz1MEYZAIEi5jVKpTPEObvt//ASP6d2CELK2DJuwwhUcdLs949eOE9wIXUEIWXwn+N7l4MGDR48eFYvFrIcjCOL48eMQWUHyIh48pncLQsjoO76mlcZEEtxXjQa2ET7+1Piq2gbOyviEmISvv9yalOSc7jwgIITYYAreY7FYgpx6AgAWLVq0aNEi9jNB/B5XgXOQvIixYLjAjuld1xxyHtNHjAVJkkxISHD7FUVRBEG09G24EzEWDD4cHWHAG+HRY8c1ectBkchNH5+QbproT9Ykz5Ak6evuAoFAIBKJ2kgfX+EcJC9iLBguOMb07J8EQRw7diw3N5dz9glswXAHW5AzHB1hwBshAEC3HIgPxms3UtxOU7m2RPMwe37tYcIWDCIBH9MDYAuGP9iCnOA4R7ho0SJ0GwBACOXm5kIIpIAJODRNZ2dn860FhiNFRUW9e/eOi4sbMmQIG7Mx4ikoKDh37lxGRkZpaemCBQs8CMOCvXv39uzZMy4urmfPnu0t01Yk4bS/INQIcGSZCMs+sXbt2s8//7ysrIxvRTBcqKysnDhx4r59+/r27btu3boXXnjhxIkTfCvVJgRsTB9iMAwzYcKEbdu2DRs2bOfOnS+88EJNTQ3fSmF8xnV/QagRgA31ro2wurp69+7djjwaboWhxpQpU1avXs1+fvHFF1etWgUAffr0efPNN3nVC+Mtrha8du3a+PHj77//fplMNnnyZPxAE+K4WpCiqE8//XT48OFGo1EikcTFxfGrIcYzbntRt/sLQg0cWeYWTz755K5duwDAarUWFRWNHz8eAIYNGzZ69Gi+VcN4hasFhw4dumHDBgCgabqgoOCZZ57hW0eMJ1wtKBaLR40aZTQaY2Ji8vLyWGtiQha3vajb/QWhBnaEtxg+fPiFCxfq6ur27dvXr1+/9PR0vjXC+EZLFjxw4MC9994bGxu7du1afjXEeKYlCyoUCoPBsGjRopkzZ/KrIcYzrhZsKXBHqIEd4S3EYvFjjz22e/fuwsLCiRMn8q1OMHC7DCF8g+S5WhAh9M9//vOdd94pLCxcunSpUIhzrYQ0rha8fv3666+/DgBRUVEvvvjixYsX+daxTYiYQJWuFmwpcEeogR3h74wbN+6zzz7bv3//E088wbcubQ67DOGDDz5QqVTvvPPOCy+8wMrDOkiekwWLi4t37dq1e/fu9PR0g8HApv/GhDJOFkxPT9+0adPRo0cRQlu3bu3Xrx/fCgaeCAtU6WTBlvYXhBrcHWGEjScA4KGHHjp16tTw4cNjYmL41qXNaWkZQlgHyXOy4JEjR8rKyuLj46Nvw7eCASbi26BYLN61a9err76amJhYWFgYeXOEkReoMkx7UY6OMCLHE1KptGfPnq7vRZsvi40YWlqGENaRKp0sOH/+fPRHvCmEYRjtbXQ6ndY77HZ7W16Zez3bQxt84IEHTp06pVKpiouLI29Hb+RFi22pFwVfOlKGYep8x58MMBxnTRzjCYPB4DSeKCoqYh9b8vLy2DlSt0InYuIS7DtmE2I55yvxHqvezROx3W4/f/58VVXVyJEjg6BDiMAuQ1i7du3MmTN//vln8CNIXsRYkCAIR8Q7kUgkEolIsvXnRW+OCSy4DbYEm33C7VcURbHZJ1y/8tLQASTggSojxoIkSSqVSg5nca6RoyNkxxMGgyEmJqb5/CfnZ5lD3+z64YcfuCnjMy8/6hp19+uvv37llVc+/PBDR9w4bjAMo9frfTqFvY3+VMqB69evr1u3bsWKFewyhMWLF7NyzkHyIsaCBEHI5be6EplMJpfLg+/kvAG3wZZgs0+4/YqiKKPR6DbGvUKh8LNeXwl4oMqIsSAE/8kS+YfBYHjvvfcGDBjA/imXy81mM/tDlMvlLQk7dOhQVVXlZ9UhyJkzZ+666y69j9y4cUOv1wdZVavVGh8ff+TIEYZh1q5dm5uby8pnz56dn5/PMEx+fv5rr73WkjCCLdinTx/Hn7W1tTRN86iPN+A22BwnCzpht9vr6+uDqY+XNO+KcRvs1atXkCvl6HVbWtaMs9kRBKHwkaioqOCv7G9pGUKERaqMYHAbjFRwGww+HB1hS8uaIy/odgTjdhlC+AbJa2/gNhhJoIgIVBm+cHSEeDyBwfALboMYTKDg/kaOHU84CSMp8j0GE+LgNojBBIRQXA6HwWAwGEzQwI4Qg8FgMO0afsIQy2SygoIChUIBADabTSQSsXtFA4jdTjEMDQBACgExAAAkCQwNAjFQVhBKgLKCQAQMBYSghQNsIBDePgABMAAEAcjDFpnGxsbAXkXI4mRBoVAYqH0/FEXRiACGBoEQaDsIhEDTQJAACAABIXD5iv3lsF9RIBABbZdIJNxqb7cW9L8NWq1WAAIcZSD0+5/sZ4Df/yRIoYAUCAT+1OgWbMHgY6coBhFe9KK3D6BpEIqBsrptp7xYkEB8xA+7cOHCgQMH2M8rVqx49tlnO3bsyKEcs9ksFAodcUCa8/bi5ap+48mTnzOZg6H+CikUMspuxIUD6NG5sHUOjF9FfFWAHvgb+VMh03UQNP5GEsCk9iBK9qDH5sHWOfD0CmL32yj3r+SpHcydfwJVNYloJnt4wunPFs6f27wii8VCkqTDOyqVymeffdanq9DpdGKxWCqVcrgDfOFkwSeeeKJz585uDeEBk8nEhm5pLtz99dcH68Xk1WJm4Hg4uhFyJhKndqEOvUmrDkwaptM9xKVDaNBkOPj/4L4JxPk9KOlOkgDQ3GC6DiJK9qAhU+DIhx07dsxISx3/9O+RqwwGg5db45tbsK6uTqlUhuaGej/h1gZtNhtN064b0i0Wy7w3CxiBBAnFIFYQdiMyakl5NCOQkGIp01RFiGVIKAaJgrCZkFEtSu36zLAB9957r1M5bOQXR0CDVqFp2mKxNE/dAB7bIEVRarWaQ9SSEMTVgmlpaa53wyfc3k8PfL7jqx+PHQaBGPrleduLdriLOPs1Gj2f2Pb6yvdXuD4MMQwjk8lefvllzlfBhSDvW3SlX79+p06d4nauWq02mUxOwsOHD7/zzjtxaZ2h4GdBRm+Y9CEMHC8YPBn+uplM7Q7/sQAAbLCRMUpYcFLQqQ9MXAf3TxDmTIQpn5Ip3WCDDQgC/mMmY1Ng3nFB577w3FrImSy871lYeKpz1l1O1Wm1WoPBwE1/RwnsZucg89VXX/Xq1Ss2Nnbw4MFlZWWsUKVSjR49Oj4+fsyYMSqVyoPQQb9+/Y4cOeJqiFZRqVSuF/763Hx44l1BYga8ugeEEnilkIhLh6FThfc8LrjzHnjmfTIuDeYeAYKElz4hlV3g/omCnAnCTn1g4joyRgnzjgNBwHNrs/r9qXmx9fX1drvdVw3DYkO9/3jfBo1Go0ajcZVrtVqxPJqMS4OUbtDlPjI1E8RyQXIXyOgj6J4LIikZlwbJ3aDrfWRqdxDLZHeP/OSTT1zLsVgsTU1N3mtutVobGxu9Pz5kN9T7CWtBX++GKzabraGhwfvju/buDwtOCp170a6eetGphaTyTthgI4Uiq9XqWiYvNuL+qFtUVNS7d++4uLghQ4aUl5ezQt4j39M0PXzEgwt/smh1ujatKNyprKycOHHihg0bbt68OXbs2MgI2fw7nfvzrUEwCM02iMGEHRwdYch2o4gNVpT3FkEGfvohkrh27dr48ePvv/9+mUw2efLksrIyVs4hBQzDMDdv3qQDBMME6l09CpRK3oCCPsUQsm3QO5Ber8cZIjEhAsfFMo5uFAAmT568dOlSVs458j0myAwdOnTo0KEAQNN0QUHBM888w8o5hGw2GAyzZs3KycnxdZpTo9HIZDKnCXOLxQyggFtuBTX71y2I/cf1CJqmm5qaHH+q1WqGYXwNZadWq0mS9GaOMDY2lvMKHW4EsA3SNP3LL7/06tWr1UqtVqvdbnfN3uAxA44b81Gaxhn/mLn/8PeF//3YqRyWVjVhsdlsLWWTcEtIZZ/AhA4cHWFgu9EXXnjh8OHDHNTQarUSicRsNrN/njhxoqamxn2/2VJfilr4382BCABohlGpVM3ler1eIBD4kwpLr9fHx8fzsljmwIEDb7zxxsiRIxctWsRKkO8pYGJiYjZt2qRUKt0G9feASCSSyWSOCy8vL79/2EOahjoYveD24kOi2b9uubUW0fUIgUCYnJz8+3EEER8f76sjRAiF7GKZALZBs9m8bt06p6x4bjGbzXa73ek23rx585kXptopO+Ep5cAfWhZj1qIeI0x2yilNhNVqNZlM3j9S2Gw2NrO0l8eHVPYJzC+//CISiXr06MGufeURv7ZPBKQblclk06dP57bSyW63SyQS9md97ty5Z1+aTnS9z/07qpb6UqKV750OJQnCSVWapgUCgZ8rtYIfdBshNG/evBMnThQWFnbv3t0h55wCxn9KSkqojn2RrL6Nyo9IAtIGFQrFxo0bXbPquGIymex2e/NYl7/8enbSCy9e0QFCqIVG1OLTjFgkdqrUarVKJBJvNGFhtw14fzxFUQRBeH88pu1gRPJHJ//dcqP88P7vcnJy+FWGY/8bwG5UIBAMGDCA22slyW2AdSfxadpJm+Dkl9wuCgA8OMRbrZkgnFSVSCQCgcCf12ISiST4Y47i4uJdu3adPHlSKBSyUzXsQxkbnXnx4sWuIZudhG0CQQJPe6HCjhB5lFn6/uqLNU2CRC7bnzDhS0lJyfnz5w0G33Kv/gGLXjfn+9h/PRg4pbjDsf9lu9Hdu3enp6cbDAbHpDeOfB8uHDlypKysLD4+Pvo2rByHbA4XQqgNynAyBI6E77rf+wYPfXldUX1DhIQv4OgI22k3ajM1aPW9B+YOuD+Xoii+tfGL+fPnO+2kYeU4BUy40E7bYAQRvut+EUJGTZP++U8JAT+xyQIOx8uYP3/+/PnzXeURHvleXWO2WEuHLyLff4hhGL61iSiqq6srKioQvqte007bYAQRwHW/CCGtVsvchrNKrZZgNBqnvvrPK5fLOVfhDAKnGjlfhT8TTOHtz2vr6lb/6/8JBMIRQwd3SE8PQo2ESIq6DcLzWIGlqamp/6BhOp2WzugXmBJNmuuV1fGpHadN+et7774TmDIxmIASwHW/Op0uLy/vwoULBoPBH0dot9v1er2HTbFnz5796tu9pqfXwNH9DiFy+uBydrMDnDdEIYQ0Gk1DQ4NDQlGUVqvloHxcXBzntRrh7Qjnvvn27goGKKvNbn9p8kS+1cFwxGQymcwW658mCW+cD0yJ6hq7SG7903MNf9zrggkpbtZUffbZZxMmTOBbET4JyLrf2NjY7du3Jycny2SyxMREzsrY7XapVJqUlNRcaDQaN2/eDADV1dUMw4BYDj0ean4A4fTBZZjQ7ADnJcQESSQkJKSkpDgkFEWJxeIgx4MNb0cIANDrIVBVF+39/PDRY2YzDwHEMSGKQASyWADsCEMVXf2vNXV/mzaj3TrCEFn364EPNmz5cP2Gmzdv6mI6gbYWgCQT0mk/9kyHLKG4U9hnqs+r4rOu3fEw5fKghAl9Rj4xvlOnTiaziW9FMD6z7+ChM2fPcjyZoYnU7nwkvwkVQmjdbwusWr3mwl0v6mgBfe+zdMd+0HWgfeAET4GewpaIcIQAREwKKLvyrUX4QdN0dnZ2c0nwl27/eOwI/H1XRLauiGfc089eq6nlW4twJZTX/TY1NV2+fJmi7JDRB0ThlCGOG345wlDoRjGcWbt2bU5OjiPcNkvoL93GNIf3NkiKucdUAoTspPi+B0ZcvHgxcBqFDaG5hen8+fP79u3rec999wwbVV1T03YVhRTcHSG/3ahKpTp58qRarQlIae2TPn36vPnmm05CDtknQpn6utrjx483X5MWSYT9owxDU0ZtyW83uK0SxAScCxcu5D37/NOvL62vvKZ/6wIp8C3VdvjCfbFMnz59unbtOmbMmObCoGWfeHzii2cuXTU21EDGk/6X5isIiH/OXyAUCKZPeyUuLi74CgSEYcOGuQo5LN02Go0FBQWFhYUWi8UnBTQaDZuZ0yFpYd01l+wTAADV578pObb3wOE1S98dMWIEAPga01Wr1QqFQm/2JykUCpEo2L1GoNogTdPnzp276667Wq3RfhvnLzy92fb41pskhfIYiqLYMlss31d9WoCtyO3xAoEgNKOrBxO9Xk/EpmhfKYJp4dqtcYO7IwxUN6rRaO67776Kigqfar9WXqaf/JlgfWvrzRydbIvZJ5y2vjgf53YHDKKpVVeiJCc25Q7KycrKEggEcrncB+3/iF6vT05O5iX7hCsclm6LRKLBgwdLpVJfs09oNJrS0lLa07Ynb7NPuMekQf3yRHa9WCyWyWQymcxXR8helDf9Iy99aKDaoNlsXr58+WsWOJUAACAASURBVOjRo1utkc0+4fgTIdSKn/udlvO6IGQwGNhBIZtNQiDwNpmozWYzGAzem5WiKL1e7zbLRExMDM4+EXwYu23Hjh2nTp164IEH+vTpw5caAd4+waEbjYuLKywsTEtL86kitqkQrW5sdxzQYvYJp60vzse1uAPm4dekZXsTExNTU1P9zD4RFRUVOi2Qw9JtsVj80EMPSaVSX335gsXvf/XtPpPp90Q8nOMUeDqRIEQikUQikUqlvjpC9qzwGihwzj7htHvMLc2zT5SUlDAM08qzyO/ftRzOniDi4uLY2q1Wq1Qq9Sn7hEQi8X7nHEVRAoHAmyvFBAeTqv7/HbtOfHlgBUI8OsIAt3C2xwQA127USdic4CchwrREMJdu/3btmunPK3CYnsDCrQ36ypUrV3JHjjaZTV6PCDEYNyCGtj84i8gawq8aAXaEobMDBsONUFi6jfGH4LRBvV4P8jg/xvC/Y9Zrc3IGvf322/4XheFMY2PjY+Oe+8fr/7RYfZvpjwwCPBQrKCiYMGFCRkZG//79P/30Uw9CTIjgFFcwaCGbV61adePmDegZ2FJbpLa29vjx4xKJZNSoUUGqkifCrg1SJi3qhaPY88ae/YemzXxVp27UJvWilXeR1FG+NeIBfx1hkLtRhNDs+W8dO3a8tp7/POaIoa9cuQIAHTt29GeOsB2ybNmy+Rt2oca6YFTGUGfPnl2wZGUTKOiKs2ajH6lEQxJeHmWuXLnicZUTJmzYtOWT65l5cKVYkNgJMu6G8vboCMNpFQAA2Gy2D1YsOT3wDbud/3SAZq1q6ty3ho8au23bNr51CRsoilq0dMWOnV/RPR4kxL6tMuWG9cblD7btrbz+m/HPK/B0VkD48ssvn5v8V5PJHMAyjx8//tFHHwWwQIwPJHZq59mVw8wRArtStMfw1teLtj20xWR+fgsMfIZvRcKJ/fv3L163+ZfrjRCsKJPIZrY99BpBersiH9MqZWVldIfWNx36QMPVQ2evTn1tfkp6R40GR8kIEqd/PfPa3Pzz5wOU8sU/dDpdbW0tX6EVws8RYsKXL7Zt/8+GTRCXBvEd+dYFE0oggLg0Sqyw2Gk8WRg0nhj/l5WnzVdq+J9momuvvPXe0s7dsv827e+8KBA2jrC+vr5f7oj0O7vTdMi1E6vVqtfrfY2r0t5obGycNPn5onKdzeZtHJAAgxhKIBk5dtyChXiBInfWf/LFijUftEnRpADvpQkCCKFF768dOXZcQ2Mj5D5PymL41giQvoF+/F3bM6vtPPXuYeMIL1++fK1Oo3q5KNT2LTH1vxW89XZCUvLkF//Gty6hjlAig245vFVvt9Am7X7o+e3B9rgcwH8oinr2penzCxZqGHFbtUKEqqqqHAmJMG2BzWZbmP/a/g5PWSIxsyA3guEIAxX5nhBJIDEjgIoFBEZbx4xbRk1YV3Lh4mtz81etWs23RoHHHws2NDQsXLhw6KN5A4aMsNlsbaShtxAC6J4LAGq1+s/PTX7gkTGDHxr152eea2xs5FmxNiYgbbC8vPzbnVtpIEEoCax6DmwgGPnIY2P+/NTHH3/SRlWEKYHqRauqqsrKyoAg4Z4/E0TIDYRomjIYDNevX7906dKlS5dQsFYSBONG+Bn5vr6+/qG8p6bNfiOk3z1e/eFSo2Xlz7p5S1Z36Jp9Z2Z2SGvrI/5Y8C9/m/7Ot6VHjxyu6PtiSEz/6BtLL1xMSe/4VWnj8V/OHEfd9ny332g0tn5iOOOPBWmaLr98ZVTen/vnPEDRdJv2nshiYOLSjpRc//Kb7w4fPrxy5craWpzvECBA+UPKy8vvvm9I7qhxDBOSOcwbr3/zv/8pU9PvH/7IwJGP9+zZ02wO5MpkDwQjtlmrke8vXrzIRr5nGObIkSMajcZoNHbt2rWhSbVz9zdlF0vPNCFGoSTNl6CmFACg6iwCBLXliLJBUyWYtYhAoKoGuwWqzjc/AOw2UFeDSQN2MaiqgLJCzTn2AEAI6i4juxXUNWBUI5KEpkqgrFB1FgCg+hwgBHVXkN0KmhtgUiHGDo0ViLLdPuA8IAbqroDdApobYGwi4tIgJdP2s+nGn5eQm55ftmxZbW1tbGwsQig2NjY1NbWxsTE9PX3voaMMg4Qk/PtfayQSCQDY7fYQDzLXqgXLysr69esnFAoZhtmxY8elS5dMJlNWVlZpaenVq1ch6ymidD8SCAAh0NaC1QiEAPT1wNDQ+BsggMbrwNCgb0BmPVhNoK0Fhob6awAIGiuAtoOhEZm1yGYBzQ1ADNRfBQBQVSDaBiY1ALplR4Sg7jIA3PoxmLVIIADKBqpKQAjqrwIguP4zJY1l4jJIRSJBCiG+A0MKps6cI2CoJ5/IO3fuXEJCQmpqalpamsFgqKmpGTBgQG1tLRuiMzs7GyF0uPinLZ9+JhXA4X17HHHGQzx3ga9t8Ny5c3FxcXK5/NT5i198uau2qgJEUkFiZzCcR4gByopsRoKyAgCiKbCbwUICQoAYsNvAbgLKBoAQQ4HdgkjBra8oG9hMQFkBsWdZwCYAhIBhgLKC1QR2KwAAQ4Esbv/BQ//btYMUisxm8x133KFSqUpKSrp169bU1JSRkTFo0CBWVZ1Od/XqVYTQ3XffbTAYGhoaDAZD//79GYZBCJEkSRAEQohhGIIgSJKkKEqr1Uokkps3byKExo4d62h6EWDBrKwsu91utVqPHz+u1+uvXbuWnp5+s7Z+/9HjNGX79VwpY7fqQUZN2gDLh7ddL4rsNo69aMNV6Hof1ftRYs9y/Qv/JVeMePb5KQatuusdnUQiUUbHDtnZ2Xa7vUOHDrW1tayxHnnkEUeIdn8sSARh7KlQKBoaGmQymdlsTklJ0el0zb/NzMwEgB9++AEArFbrgyMfphFhNhrkUrGNRoxQYtRpaCClErFer09ITKq7UZ3aIaOxvi4uIdGg04qlMqDtFAMyqUSn0ycqk2pr2APqY+MTTAadSMIegGQyqU6rTVQm11ZXpnbs1NRQHxMXb9TrRBIZwVB2BsllUp1Gm5icXFtTldIhQ9VQHxMXbzbohRIpwVA2CkXJZRqNOik5pbamKiU9Q93UoIiJMxt0ArFUAIyNoqOi5Bq1Oik5pa6mKiW9o6qxITo2zmLUS8QikiAsFmtMbGx9Y1NSSqqYhEPffct2o3q9PjExMSaG/ynrlmjVggzDHD16VCqVWq3WoQ89YjSZjXqtTCq1A0nZ7VJ5lF6jjktMVDc2xMbFm0xmkYAAgdhmtcgVCp1aFZ+YpFE1RcfEWKxWAQGEUGK3mOXRMZqmhoQkpUatUiiirDaKBCQQS80mQ0xsnKqhLjEpWavVyOUyO42AoUUSqdloiImLb6q7mahM1un0MomYAgJRdrFMbtBp4xISVA318QmJBqNRIhQwpJC22aRRt3TT1NcmJibpjOb4xES7XiWTK0iRWK9VK+Qys42SRyloqzkuLtZqtZrsNE0Im1sQQj53ga9t8GZtbWxcvM1kICRRao1aLJIQBIEArGajUCwRkgQiBbSdomlKLJEQBDCIsFlMIrFEQBCIJGmKoilKLJEAAIPAbjWLxBKSIAhSQFF2mqLEYjFBEAwQNotZKBILSQBSSFF2irKLRWKSFDAAdotZKCATExPMeq1IHtPUUCeTR4lkUXIB6t+vr81mq6ioMBqNNpphKKprlzsrKiqEEpndZr37rt6VlZU0Td9xxx0SiUStVtfX18fHxycnJ9tstt9++81isdgZkMuk+/buiSQLHj161GAwREVFPTTyYQagpqYmOTlFo9UJpHLGbm2pFzXqtGKZHFE2igGpRKzT65OUSp96UZNBL5JIgabsDJJLJVqNNiklxade1GLUC8RSEtE2ipbL5Rq1SpmSWnejOjmtg6qpMTom1mrUS8WimJhou8UsFArNVps8KooE+C5QFkRtj1wuN5vNCCGj0SiXy52+7dev36lTp7iV/NFHH504ccIf3f773/8ePXrUnxIKCwsPHDjgTwk7duzYv3+/PyW0Na1a8K233iouLva12HXr1p05c8bXs95///3Lly/7etbbb79948YNX8+aO3euTqfz9awQJFBt8MiRI5999pn39f70008bN270/vjz589/8MEH3h9/5cqVFStWeH/8jRs33n77be+PDx28sWBZWdmqVav8qeXatWvLli3zp4SKioolS5b4U0JNTc0777zjTwkcCMYbOc9JfNLT0//xj39wy+d39erVmJgYpVLJWbdr165FRUWlpKRwLuH69etSqTQ1NZVzCRUVFU8//fSDDz7IuYS2plULbty48auvvvLVEGVlZUql0vucOyylpaU7duyIjo726ayzZ8/u37/f14yJp0+fPnnypDePmW+99VZODn8LYlsjUG2wrq7OZDJt2bLFy3obGxvVavXWrVu9PF6tVtfV1e3evdvL4/V6fWVl5b59+7w83mw2l5eXHz9+3PWrCLAgRVHV1dV79uzhXIvBYKioqDhw4ADnEoxG47Vr1w4dOsS5BJPJdOXKlWPHjvl6oj8WDMar0VdffVUikSxevHjevHkURa1YsaL5tw0NDb/++mtb6xDi9O3bNzk5mW8tWgRbsFWwBcMdbMFwxx8LBsMRajSaCRMmnD17lo18z2b1xIQR2ILhDrZguIMt2KYEwxFiMBgMBhOyhO5yYQwGg8FgggB2hBgMBoNp12BHiMFgMJh2Tdg7wqKiot69e8fFxQ0ZMqS8vNync/0P3+dP7Q5KSkoiI8E9hwvx9QZyMxlnM0WMafzB890bNGgQcZsXXnjBg3Vcbed9yVOnTnUr8Vw+4YLn8iMGf363EdCjciHI+xYDS0VFhUKhKC4uNplMK1asyMnJ8en0uXPnTp8+3WKxTJ8+PT8/P8i1s2g0mnvuuSfcDYE4XQiHG8jBZJzNFDGm8QfPd49hmISEhOrqar1er9fr58yZ48E6TrbzqWSz2ewq8Vw+QkjfjDfffHPu3Lkeyg/kXeMVf363EdCjciO8G/nhw4dfeukl9nN9fX1iYqJPp3fv3v3ixYsIoYsXL3bv3j3ItSOEGIZ5/PHHt2/fHu69LbcL4XADOZiMm5kixjR+4vnu3bx5U6FQ3HPPPQqFIi8vr2vXrh6s42Q7n0quq6tzlXguv/lX586dGzFihN1u91A+1zsUWvj5uw33HpUzEdLIKYqaOnXqtGnTfDorKirKZDIhhEwmU3R0dJBrRwgtWbJkzpw5CKFw7239vBDvb6A/JvPJTBFjmkDh9u79+uuvw4YN+/XXX5uamiZNmiQUCj1YpyXbeVPy+PHjXSVelm+1WgcOHFhaWuq5fM53JqQI1O82THtUzoRfI8/KynJ6qbt///5+/frNnTu3+ROfN3gO3+clnGs/dOjQAw88YLPZUBj2ts2t4NOF+Gk+zibzqZawNo3/cLPRjRs3CILwYB23tvOy5Pj4eM+SlspHCC1evHjGjBkertdtaeEC55botgSWMO1R/SG8GznDMPn5+YMHDy4rK+Nwerdu3crLyxFC5eXlmZmZQa59/vz5TvO1x44d41AO73C+EA43kIPJONQSMabxH89379SpU46o942NjQKBwIN1nGznU8mpqamuEs/ls0KKojp16uQapb3V0sIR/3+3Yd2j+kN4O8Ljx49nZWWp1WrHlLhPp8+ePTs/P5+9+6+99lqQa29OxAw7fLoQDjeQg8n8NFPEmIYbLd29w4cPI4S+//775OTkCxcuWK3WmTNn9unTx9U67JHIxXY+lTx16lRXiefyWeG+ffsGDRrU/IpaKr8t7h6PcPvdRkyP6ivh3cgXLVrkzyJYtVo9atSoDh06jBkzRqPRBLn25kRMb+vThXC4gRxM5qeZIsY03Gjp7rEfGIZZt25d165dk5KSJk2aVFlZ6WodxylOtvOpZK1W6yrxXD4rfO6555zyLrVUfpvdQn7g9ruNmB7VV3CsUQwGg8G0a8J+Qz0Gg8FgMP6AHSEGg8Fg2jXYEWIwGAymXYMdIQaDwWDaNdgRYjAYDKZdgx0hBoPBYNo12BFiMBgMpl2DHSEGg8Fg2jXYEWIwGAymXYMdIQaDwWDaNdgRYjAYDKZdgx0hBoPBYNo12BFiMBgMpl2DHSEGg8Fg2jXYEWIwGAymXYMdIQaDwWDaNdgRYjAYDKZdgx0hBoPBYNo12BFiMBgMpl2DHSEGg8Fg2jXYEWIwGAymXYMdIQaDwWDaNdwd4aBBg4jbTJ06lRWq1eoxY8YkJCSMHTtWrVZ7EDooLy83m81OQoqiEEKcdYuYEsICJwva7Xb/y6RpmmEYPwthGIamaf+VCcgVhTJu2yCLn79h/03g580PyA8p9PFgQfDdiL5aDSFEUZT3xwdBJV/h6AgRQpcuXaqurtbr9Xq9fs2aNax82bJlnTt3vnnzZqdOnZYvX+5B6GD8+PEXL150EqrVaj8vW6PR+GqbgJeg1Wojvg+FP1oQIdTU1OR/mTqdzmaz+VmIzWbT6XT+K9PU1BTZDzRu2yCLn79hq9Wq1+s5nw4AKpXKH09mMplMJpM/CoQFHiwIvndlvt40mqZdRzieaWuVfIWjI6yrq7PZbHl5eWlpaRMnTnR0N7t27ZoxY4ZEIpkxY8bOnTs9CB0ghHQ6HRNKVFZWHj9+/McffwyaYpzt5w+BGtOHIGazubi4+OTJk1euXOFbF4xf1NfXFxcXFxcXGwwGvnXBtMi1a9eKi4s9jEpDHCG302pra++9995Vq1Z16tRp9uzZM2fO/OKLLwCgpqamc+fOAMAOAdmD3Qod6HS6MWPGOHVYTU1NFEUJhRzVAwCVSmWz2UQiEYdz7x083CJLMleWfLn1i4EDB3LWQa1WW61Wb3SIi4uTSCScK+KAY0wfGxsLAI5bzQ7fd+zYMWfOnOXLly9ZsqQlYSjzxsJFmz/fDgRxX+/uB/d8zbc6GO7kjZ/URAkt1ZeOHT7oT2PEtB01NTWPPPGMUVV/7szp7OxsvtXhAkdP07dv30OHDrGfly5d2qtXL/YzQoggCPaD492mW6GD2NjY7du3p6SkNBeSJBkfH++PIxQIBLGxsb46wnPnzjU0NDRU/YY+LIl5f3BCQoJSqayrqwMAuVzO+gzvEQqF0dHRYrHYp7OCg2NMX1ZWNmLEiPXr10ulUgDYtWtXUVERO3zPy8tjfZ5bYShz9epV02MLQSCmrn3Cty4Yv6itum5+91Lsv0fzrQimRUwmE0WKpMmd+FaEOxw9zenTpy0WS05ODgCIxWLHaCY9Pb2qqiozM7OmpqZDhw4ehCHI5cuXhzwyllB2QfD7hND6zR/PnDULgBj96COrVy4Xi8Wpqak8KhkoAjim12g09957b3V1NQAghBoaGvyfUdNqtVKplPMo2WqxAABoak4cO0qQJAFQWVkpEAi4lVZfX88wDPsw1xLx8fHskwQG037QarWlpaURMAvL0REajcZx48YdOXKka9eu77777uOPP87Kx4wZs3nz5sWLF2/evDkvL8+DMAQxGAwgj9P8/Tv42+/970+//GLLexfUNUV71u85dLRDivLyhXM8KhkoAjimj4uL2759e1paGnsASZL+PytIpVKZTMbZtUjYE1XV0PV++PtumBqVmprK+QUDQRApKSmeHSEG0w75y9SZe4p2IsqGFIlRimi+1eEOx8Uyubm5CxcuHDNmTIcOHdRq9bJly1h5QUHBuXPnMjIySktLFyxY4EEYZmjriLseNf/9G3ukrMQ+ffp0cXEx+9l1TA8ArmN6J2FzQsdJzJq3sM99Q04Un/xdRODNsmGP2aAbNGSoRBa1bds2vnUJJBRFTZs2TalUDho0qKamhhWG0YK1365dpeI6Ml3uC/dl1Rz7CIIgpk2bduXKlYaGho8//jgmJoaVx8XFffPNN9XV1bt373bMqLkVhj4Wg3bw4CEffbSFb0XaBKPR+MQTT1y8eNFms7mO6RFCrmN6J2Fo8q+l754fstBIu3HMBoNh586dO3fuPHnypOu34UgEr/t1gjJqqX/8T9Tn4ePHj2/btu3ChQt8axQY1qxZo9PpKioqcnJyFi5cyAo5bELjmZB5DuYMflhuEcqkQ3OPCNK6861ImxDJY/rMXELoZoHSzPw3J85d8pd/Ll/w7tLgKxVwAriXNzwQSW11V//zTfGkOW9t376db20Cw+eff/7666/L5fKCgoJXXnmFFXLYhEbT9JkzZ+xB548DQURRVPB1cODPPjTuyzLbBWTEPiiwY/pp06Y5ydnhuzfCcEIaff+IR69evWZ+9C2wmYq/fTchLUNEQtVvV0NzTa83RPa6XwBQqVQ6na55V4tsVvtjr0FdOY9aBZaKioovvvhi2LBhXbp0+eijj1ghhwVrZrN51apVY8eOdVsLG9PA+yX0RqMRIeSNX2GarRhACOn1eq1W600VbaFSTEwM5+aMHaFvIIZhLR1G73gxyKz9pd9MwcW/AwA0XLMl3Gl+8RPBguywntgI4LpfvV4/bty4n376ybUWlUpltVo59y8Wi8VqtXKI0GQyme4b/rBOb7JYLQ4havZtY2OjN+UYDAaCICwWS6tH+tONcob19KWlpevWrZsyZQr70p7DgjWFQrFx48akpKSWKvJpL5lUKkUIRUe3vvhF0GwBGkGQ8fHxHnQIjkrcwI7QFwyNNTdvJnfoZDPq/OkdMDzQcwQhvLUgiBBJIaEjv+r4TwDX/crl8nnz5rl9tqMoyp+9sGKx2Gq1cnhqpGlaVV9nXVUH02IcQsdMlEQi8bJMkiQJglAoFK0eyXl3jT8olcpZs2alpaXNmDHD8XI7rDehhSkR++rPV06cOPHzzz+3EuDUqEayWNuaRlLIJWANpk0pKirasmVLGA/xfCSA634FAkFmZqYoxPBw7QKBIODVkXzMgzz88MNbtmyxWq3r168fMGAAKwz3BWvhCHaEAAD/+9//Rj45cfZ7a03m1l+hYEKQPXv2PDdtzt8/Ogzh/LbTJyJ13W+7YsmSJYcOHUpJSTl48ODGjRtZYZguWLNYLL3v6rNy5Uq+FeECfjUKAHDz5k3oOdzUbRi5+22+dcFwwWg0CjvdrfvLRvi+vcRUc6z71Wq1o0aN+uCDD1h5QUHBhAkTMjIy+vfv/+mnn3oQYngnNTV1//79TsIwXbBGmXREr4f8zNjDF/6OCEtKSqKiohx/RuQeJgwmBIngvbwXLlw4efIkQi0sEbQaP9n21UNjnly99l/B1QvTGmEbvMIvvbVa7fPPP9880FzE7mHCYDBBoaysLOfBx8bPfNNmayEPYsWpa1GZB8zpR3/4ObiqYSIW7o4QIfT888/n5+c3F3LYChqOIGlMl55339G955kzZ/jWxV/wmB4TUmi1WohO0v/1c08HpWVDXIfv9u2PSUwZ8chjwVINE7Fwd4TLli3r2rXruHHjmgs57GEyGo2zZs1S/xGNRqP2Dy9LuH79+szZrxZu20ZT7HrRFpdaNE9JgYzqmgn/baJEdXV1/uugVqvtPCWyx2N6TLjScI3qPEA/ccONBhXfqnCn/QTJC3E4OsLDhw/v3bvXNT4Fhz1MIpFo9OjRMp6Y99aifx29eqi8sZWNEwDNdjEBAEBatkAil0gkAVGDl6XbARzT22y2YGgcYNCZM2d+/fVX3K2EKyIpyOP5VoI74R4k78CBA40NDXxrERgIbsE1FixY8N577zWXHDt2LDc3NzMz89tvv83MzLx8+fJjjz1WXl4OAK7CRx55hM12CwA1NTVJSUlOmedommZ3wnK9Lm9LaGhsNCAx2K0AAGIZmDQQ3wEar0PSHaCqgphkMDSBNBrsVgAE4igwqSC+IzReh8TOpK42JSnBKVVQVlZWYWEh+7mpqSlkE/MCwNKlSxsbG99//32C+P1noFAoGhoaZDKZ2WxOSUnR6XRuhc0tWFVVRdN0RkYG+ydN0/7vTWbz/3n/AzCZTCq9iYlOvmU7dQ3I44CyAW2DmBT3wqZKUiBENJWQEN/85XCXLl3+7//+r3nh9fX1SqUywvIRttoGWXw1hBMIITYzl/enWK3W2oYmRqEETQ0kdobGCkjMAHUNxCSDvhHkcWDRg0gKDA2IAWmMyKzq2CG9eQnN26Ber/dyQ33wqa2tzczMzMrKcgTJS05OBoCsrKyioqLs7OxLly6xIfTcCr20IPjenbKRzDxbjWGYispKIEggSCAFQNsBIRBJExRSb1ZjeVapuQVZDAZDKEaWWbRo0aJFi9jPzbtRL/MRlpSUbNq0ibV6JFFeXr548WK+tfAKdkzvunTbyzF9ZFuQza3oICLzEUa2BfnWwiv8DJIXwRZctGiRSvWHN95srFHPU0gKhSJUYo16v4epV69eHTuGfZgrJ3h5w8mNgwcPHj161PG7IQiCHdN7H94JWzBEKCkp+dOf/mQ0Gtk/1Wr1pEmTTpw4kZub+/HHH8fHx7ckxBbkF/+D5EWqBQmCaP6SBm6/WnASOsE58zYEJLJM85erEbCHqZ2waNEidBsAQAjl5uYCjksSbuDlTuFLYJNjRxIEQUh8x59noLB5esIEhzAN79Q+ac9bmDxD+07wU5HgIHmhAw6xhnEzpnc6IPTDO7VPArWFKcKgabqpqYn9zKZhslqtrZ4VGxvb0mKTNgIHyQsdsCPEYMISP5c7RTACgcCxhEQmk4XsqtH2lRw7tMGvRjGYsMSx3In1cARBHD9+HPAMEwbjO9gRYjBhCV7uhMEEilB5Nfr4U+OraoMUpEBMwtdfbk1KSgpOde0EbMEQgfMME7ZguIMtyJlQcYRHjx3X5C0HRWIQ6pJummix4AS8AQZbkEcCstwJWzDcwRbkTKg4QgCAbjkQH4zZC1Ic1LVh7QhswXAHW5APOIdEcAO2ICe4zxHu3bu3Z8+ecXFxPXv23LdvHyuMsLjpbmPDY8IIiqKmTZumVCoHDRpUU1PDtzoYnyFc4FujABPxIRGOHj3at2/f6OjoDyNpVgAAIABJREFUvn37fv/993yr4x6OjpBhmAkTJnzwwQcqleqdd9554YUXWHkkmbCl2PARQ3t4lFmzZo1Op6uoqMjJyVm4cCHf6mB8Rt+MN998c+7cuXxrFEjaQ0iEiRMnzp8/X6VSzZs3b+LEid6cwoYV9RU2Vjg3ODpCiqI+/fTT4cOHG41GiUQSFxfHysPXhFOmTFm9ejX7+cUXX1y1alVdXZ3NZsvLy0tLS5s4cSKbh6FVEEKW21itVot3BH9rV+Q9yrhaEAA+//zz119/XS6XFxQUvPLKK7wqiGkFtxZU3Oa3334rLi52xPqPDCIsJIJbC8bExGi1WoPBoNfrvdzQyTCM9o/odDqdTqf1CEVRnDXn6AjFYvGoUaOMRmNMTExeXt6GDRtYefia8Mknn9y1axcAWK3WoqKi8ePHs7HhN27cWFFRERsbO3PmTG/KQQiZfcefZxluRN6jjKsFAaCiouKLL75ISEgYNmyY95HpGd9pwwtrN7i1IIvNZnvppZf+9a9/+RNYOdQIYFbXEMGtBT/++OMpU6YkJCS89NJLn3zyiTflCASCpD+SmJiYmJiY5BF/Et759atSKBQGg2Ht2rUzZ878+eefIZxNOHz48IkTJ9bV1f3000/9+vVLT09PT093GxveMyRJOuaxGYYJ2XyE7KOMwWCIiYlx7MWGcH6UcbUgAOh0OoRQaWnpunXrpkyZcvLkyVbLoSiq4Y/pRpuamlpN5xYXFxfkAF2Rh1sLsqxcuXLgwIE9e/bkUb2A438GmFDDrQXnzp37xhtvzJo1a/Xq1fn5+QcOHOBbTTdwHBFev3799ddfB4CoqKgXX3zx4sWLrDx8o1qIxeLHHnts9+7dhYWF7IvslmLDRxLso8yiRYsc493wfZRxtSAAKJXKWbNmpaWlzZgxo6SkxJtyBAJByh9RKpUprcHLzyPCZnndWhAAaJr+8MMPvXwlE0ZEXkgEtxb88ccfZ8+enZaWNnfu3B9//JFfDVuCoyNMT0/ftGnT0aNHEUJbt27t168fKw9fEwLAuHHjPvvss/379z/xxBPQcmz4yCDyHmXAxYIA8PDDD2/ZssVqta5fv37AgAH8qhdYIm+WF9xZEAAOHTqUkZHRrVs3HhULJmGdAcbVgn369Nm0aZPBYPjkk0/uvvtuftVrCe5zhLt27Xr11VcTExMLCwsdc4RhbcKHHnro1KlTw4cPj4mJgWax4Tt06KBWq5ctW8a3goEkIh9lnCwIAEuWLDl06FBKSsrBgwc3btzIr3qBJfJmecGdBQFgy5YtI0eO9L4QhJDpNmaz2eQd/L7qiJisrq4W3Lx587fffpuWlrZjx46QbYPc5wgfeOCBU6dOOQnDOm66VCrt2bOnY0TfUmz4yMDxKPPbb79lZ2dv3ryZlYd1ChgnCwJAamqqa36GyCDyZnnBnQUB4LPPPvOpEHbxPfvZbrcTBOH40wMikUggEPhUEcYVVwtmZ2efOHGCR5W8IVSWYMXEJdh3zCbE8iDUZdW7mSOx2+3nz5+vqqry6dkzrAnsowy2IC8EcMFaxFiQJEnHmIld5RSaaZgCTsRYMPiEiiM89M2uH374IUiVvfxoQkKCk+zrr79+5ZVXPvzww9Bc5Bn6YAsGmevXr69bt27FihXsLO/ixYtZOec1h9iC4Q62IHcQH3To0KGqqoqXqtuUM2fO9OnTx/FnY2Oj1WrlUZ+2I4IteNdddzkJb968yTAML/p4wGq1xsfHHzlyhGGYtWvX5ubmsvLZs2fn5+czDJOfn//aa6+1JIxgCzZvgzqdTq/X86iPZ/bs2dOjR4/Y2NgePXp89913rFClUo0ePTo+Pn7MmDEqlaolYTuxIIter2e3QrUROB8hBhOWROSCtXZFRK77DVNC5dUoBoPxlchbsNaucKz7NRgMTut+i4qK2CW+eXl5bOgZt0JMoMCOMMDU1NzgWwUMBhMGROS634DAMIxKpWouMRqNqNliYLcoFArOc5PYEQYY1PohGAwGc4tIClQZKAiCiIqKai5hZ/KchE74E4eW+xxhUVFR79694+LihgwZUl5ezgrDN7wTBoPBBJOIjO4UEAiCkPgOSXJ3ZxzPrKysnDhx4oYNG27evDl27Fg8zRuO4EcZDIZHIjK6U5hCIMTlZd6RI0c+++wzdqFaQ0NDjx49GhsbASArK6uoqCg7O/vSpUt5eXllZWVuhZmZmYMHD2Z3udpsNpFIFITE0wghm90OAjFQVhBKgLKCQAQMBYQAEAMAQJLA0M0OsIFA6HwATYPQ7QEIADWpNYcO7Fc1NbI1NjU1hWz2icrKyl69eu3bt69v377r1q3btWsXG/0hPz/fYDCsXLlyzpw50dHR7Jy8q9CDBR3vcAKO3W5nCBIQAoRAILplBdoGpCAQQgYE4qa6G+Xl5efOnWteb21tbUpKSoTlRuelDTbHZrMhUggMBaQAGBoIEhAAICBIQDQIREDZbrVToRgo+6221trBer2+tLT07NmzbC16vT6UN9QfPXq0eXSn7OxsANBoNBMmTDh79iwbyIkNDuAq5MWCVqsVgAC2HoSafQYABA4FEOIchr6xsbG5BVkMBgNCKDo6mqvircDRETqgaXrGjBkkSa5btw4AFApFQ0ODTCYzm80pKSlsMltX4YULFxzJON555x2DwfDmm282L9ZoNMpkMn+GuiaTSSKRNI+ZVFNT89H2ry0jZsPWOTB+FfFVAXrgb+RPhUzXQdD4G0kAk9qDKNmDHpsHW+fA0yuI3W+j3L8Sp3agO/8EqmoSUUyHPsTZ3Wj0Atg6B55aTnz9Lhr0PHl6J3PHvaCtJe1mptfDcb98cun8r2yNarVaoVCIRKJWtQ1+Eh8/H2WaW3D58uWjR49m2zAA6PV6/3+vZrNZJBI5vfT/cNOWsth+0PgbaWhkBr8Iha/C+FXkt0uYux6FpkpSV8cMmQKFr8L4leS3y5jeD4OqmtTeZB74G2tQcs9yptdI0N4kVdXM0Jdh62vw1FLyu5VMjxGgbyAbf2OGTSO3z9244T+TJk1qXq83jrDVPE1tQVFR0fz586urq/v06bNx48bu3bsDgFqtnjRp0okTJ3Jzcz/++GM2KZirsLkFV6xY8eyzz3bs2NG1CqPRKJVKOcces9vtFEXJZDLXrxavWF2XPZb8eSvTYwRc+wmSu4LFQFp1THov4tJhNPgl2LMCnloK2+dC3kJizzKU+yL54+dMr5Fw9QdI7gZmHWk3MSlZxOXvUc5k2L8Gnlwi2DnvvXfe6tSp07PPPsvWEuKO0B+8tCD4bkSr1QoAbnukWbNmEYoERiAlJXKmsYKUyBihlJBFg64eaAqkCiSOIiirwKx++sknMjMzXTfde6OSUql0WJClrR2hXxvq9+/f369fv7lz59rtdlYil8vNZjNCyGg0yuVyD0IHffv2PXz4MP1HamtrrVYr7Qd1dXVs5ncHP/74Y2xmf/iPBQBgg42MUcKCk8JOfWDiOrh/gjBnIkz5lEzpChtsQBDwHzMZmwLzjgs694Xn1kLOZOF9z8LUQlJ5J2ywAUHC/xnIuDTIPyq4oz+MXwVDXhLeOw7ePtuhaw9HjfX19Waz2Rtt/bGCn1AUNXXq1GnTprF/RkVFmUwmhJDJZIqOjvYgdNC1a1elUllXV1dXV1dbW3v+/Pk6vykrK6usrHQSPvDIGJhaCA/+Q3D3KNhgY+0oSO0GL38OI2cL+zxyS7jeIkjrDn/7Lzw8R9B75G2DmgRpWfDSJ/DoG4JeI2CDDUgB/J9ekJ4Nf/0IHpsn6DEUNtgEYmlFRYVTvefPn6+trfWssMViCaLREEKooqJCoVAUFxebTKYVK1bk5OSw8rlz506fPt1isUyfPj0/P9+D0EG/fv1OnTrlthY/g0KYTCa1Wu32q+z+90H+98IOPeCvH0HWAzBmAQydKrzncXh+A5meDYsvgVgOG2wgksKaWjI2BeadELIWzBoCY9+EIVOE9z4Jf/k/smMveOccSBWwwSaOitFoNM1rCfEN9YHCgwURQg0NDTabzfvSPOxeJ0hSkNwFMu4W9BgGYrkgPh3SehDZQ4m4NCIqnkjtDt1yyIy7CHmcOD5106ZNQVApIHBcZoMQmjdv3okTJwoLC9nnUBYO4Z0IgoiJiXEa/JG34aae2xK8K83vh3ri94r8v4q25sCBA2+88cbIkSMXLVrESpDvK9ZiYmK2bduWnJwMt5+r2M/+IBKJZDKZVCptLhSLfk9h6hC6HYcRbj79/pEAN6cTt/9VKpVOIxiGYZKTk0Pt1ei1a9fGjx9///33A8DkyZOXLl3KykN/F9rFixd/+OEHrUbDow6YNgVRVjJ7EN9a+ABHR1hcXLxr166TJ08KhUKDwQAA7MsHdkZ38eLFrtO8TkIMvwTwUQbDC0OHDh06dCgA0DRdUFDwzDPPsHIOu9BMJlN+fv7WrVtda9FoNBRFcZ7ntlgs7ICyuXDIyFGmTveZ6+sB2Hkmb3E+lD33D1Kk0WgYhnH8bTAYvMw+4eUsBsZLaF3Ty9Nm7N6z76vthXzr0jocHeGRI0fKysrY6QcW9rcemkl8VCrVC9Nm3aypslgswa89NMGPMm5hxFEDH3hIICCXvTX/4Ycf5lud1gnImF4oFObm5rqdyZPJZDKZzJ8FXyRJOpVs1utMT60iL/T3dXmCy5DcdYxOSKXS5tVRFEUQhNtLc9XTR3UwnmAMjXTWMI3BzLciXsHREc6fP3/+/Pmu8tAM77Rt27a9F2ptyb1IOw77covwepQJGoxJWzJ4gezg6traWr51aYUAjunFYvHo0aOdXkSzSCQSqVTK2RGyDti55OYvmf154Uw0+xcAAGih5NEnn5WKBB+v/3dmZibczkfo9tJCAX+WO/GtuxcQYfNsETaK+olAeQd0vodvLUKI+fPnO00Xs/Kwzo4dGLrlCGJT+Faiddgx/e7du9PT0w0GAzush/a9C4026n7tO+Pc5etarZZvXVoH78YOHdqLI8RgIgzHmD76Nqy8vWefyBoilMfwrYRXOJY7yWSyyZMns1uVAGDXrl0zZsxgVzbt3LnTg9ABTdOlpaX2tqS8vDztzu7I+zlddgIXMW2qVXOazw37Co41igkDho564ucTR62IhAlPBaM+hj537ty3336bnJw8YMCAYNToO+E1PYFxJbDLnd577/+zd+bxURTZA3/dcx+5EwIJoBASWFQOxQsQAY9V5FRUdmURXV1ZZEUOFQUBEcP1U0FlZZVl3XW9UEBAVuRGLkUDBIKBADnIncncV/f0Ub8/GsY4SSbTPZM5MvX9I5/Jm66q1/2m+nXX8d4b999/f4sN2e12AAh8KZAQ4drHr/z888/O5B5QcbGpsE2vyLJci2/nIVHJh8TERBx0G9Nh4Xn+4M6t8HatfFmYFmTTNSXvbTz1j20Hb+qReXAX9h+YdiQky50SEhLWr1+fnp7eWitJSUmBex21Wo2a7V5PTEwkZb7+wt8ELwEAoFAoWtMqeJVCCHaEIYbn2NLSUgDo0aNHpHXpWOhSg1pYIQZEu5h753j0aVzxB+FpEROH4C1M0QOeIwwpdkO9wdR/6N29evVyu2Nj3TAGg4kI8bDcyWxsbD6jGYVgRxhSKDtSah1LzsuUUbpcG4OJIBUVFT/88APf0dPpBUjHX+7kMJ69UPbQQw9FWo+2CcoRchznDbUsEG1JfIrOFt8z5qE1f/+A49hwtosJFe+9995rr70WaS0wIcDpdA66Y+R9U2e62zmuBeOhX1y45OHHHv/xxx/btaEg6fhbmDiGyPpdpJUICOmOcM2aNYMHD/Yu+RWIth0wb6x6c48j4xyl5YJYWduBifJHmc8//3zuyveXnOBEBeLCRCcMw9jNRuvcw+0910uZDfsVA7756VxZWVm7NoTpMEh3hP369fPJnQSSdsC0O90HQkq3cDcaC8TEo4yy2/UwpoVNAhiBKH+U8bJjx44PP/yw+XLH9uKGUYqMa8PUFib2ke4IR4wYMXr0aB+hhB0wbrd7xYoV9t/icDjswSHUwLQVbLfZi0arbx6oha/8vaYIs98BasuyERi5DdWjDM/zFRUVweTMao1gdsheIbg3SWGpeiCI2GgcOmLiUQYACgsLH5n6zKvbzrB4dhATlYR4+4SEHTAkSXbu3Nknxw1xFcmaBFs+FAR/Fu3KiBEjmgslPMo4nc6ZM2cOGTIEABBCZrM5+Cj+FotFo9EI2TibiK98bips0QO16JTQrx9aKI5aOpZlWaPRaDab5XK5fzsmJSWFObUyAPTr1y8nJ2fMmDFNhVGYholhGEVKpmv0Qvjxq3C2GxNwHHfdddedO3fOK4nCWKMURR09ejR8L/RhJ8SOUMIOGJVK9fjjj/ukkHa73Xq93idBuSgoitLr9XKFwv87QQDx7L1ftBDq3k/Ner1eJpPp9fpgIveHH2mbeb/44ovQ5iOsrql96533ykovcWxT73I1d2C75SP0OVahUHTq1Ck68xFC6B5lbDbbXXfd1fRe7MVsNrvdbslPNkIaJpPJhPiW+6GoN+lA0jA1+ZK3Wq0NDQ0EQTidzjYrT05ODv+jzJo1az799NMW3+m/+uqrOXPmrFy5UnhqaVEYNua/9saad9cR3ftJK37ixImcnJxoXuMTYkcYPUl8Xnpt2Sf/+pBFJEy8OTwtNgXJFH96ahrw/OsLX+nbt2/4FZBMlGzmfX7egu/tqWCwyfWxEGU/mpD2KLNu3bqMjIzmtZEkmZCQIPlhzu120zRtsVgIsuXHCFGPFy0/thItH0IQZGJiYnp6OkEQPs/ZLRKRNEyx8k5fX1/Pdeol0VuoEwbfOXLjJ/8ZO3ZsaLUKISF2hNGTxGfP7l30jO2yT54NT3M+cLRrs+xWzcHVz9bVxZYjjJ5HGeg/GkoOgulymJprCavZuHnzZovFMmXKlGDGJ8KJhEcZgiBSU1Nb9ATkVaQpc+7cuTvv/r3TYpR1yWv7aLG05UMJgiBJUvgb+tZDQQinJ5577rnt27e32IqQrFhsYM+mCxc8Hg+IS6J8Neg2AFB21aAHHA5H01VawavUnGBSKwfbt31GNqIr4K9KF7agXC1w+2RFwacRa10q0fMoE3lqfim+VPHEyo8dP21/5JFHAnmriAai6FEG4OzZs+TvRsLv7kf/Wx62RmMdCe/0SqVy/PjxreUfFrIrB+4khMVf3trKy8sF1yv5ZopYz65du5KSku66666QqNQiwTzuxMZDLqb9iLZHmS++2nzg4PcXLl2CnuFpsHU8LuicZ3tqo/xU9M5tNAc/ysQ6Et7pFQrFyJEjW8s/LGRXDtzrsCyLEPLW9vvxj1TV1EB6T3GesMm4tbuu7D/VFz0gf+CBB0KiUsiJ0hEDTHzCsuwfHn14naFbvTEGEqtGCS0+ykRDXJKzZ89WV1fzOBiCSKIt1qjZUE/oUoOqgudQVt9o/h10QEfo8XgcDgffyio1TNRDwN3PEUp/YyCY6GfTpk03DR25eNn/0TQdEQV42v2PDf9+9vk5u3fvjogCkomeWKNms3nEmIlOlzMEN1PKvnnz5m49coKvqT3oIEOjVquV53m5XK7T6frffuel4jNMFJwaQ7uefWF+UmLiS89NmzBhQqTVwWDCgd1u/+STT7799lvPkCcRZZdHaLkTXXP+gPp24vjla7NPRnnvi7bpCS8lJSU/F53jeRSCFyaXhet+k/HyT2+//fakSZOC32ocWjrCG+GhQ4cyu16Tmd29U48+KZnZ504eZ94xk/LIX2ja3HAuZ0KBXVNeXh5pXTDS4RWaO+8be+vwe3bt2hVpXWKAGXPmzXz3ix2HCiI/JnrdPXDNjS6Xq7Gx0ZvkCCMKmUobusqQm/K8snRVFN4PO4IjLCwsJG77I/Pw/3mUetvcQ5FW57f0uJlMuybSSkQ1BoNh1apV+fn5ox969LEnnw42Klo7wLvtJwY+d9pMVFdXR1qXGMDhcHhu/RPXuXekFQEAgPqLb7+7Nqt7j6f/OiPSqsQYCKEDBw6EMvojQiCTqTK62e12qp0zkIgl8uOHoYSUQ1JmpJXAiGPy08/uraRR42UgCH7YU4D+E2mNWqLPcNmZLZFWInopLy8/deoUAAwbNizSuvwGZK1Bo+bxKV0ZOsoekaObS5cuvbbk9W27DxCJIb6jum3m+0c9kJXT5/cjhq1ftza0lUumYzlCTExRWVlZVVXV2GjgbpsBxXvlthp+8J/gv3+LtF4YEZw/f76srOzxac+5U3tSpSdze+XU1dfDPXdGWi+MRHieX//Rx4veWG53OFCXvqTTGNr6Wbed79Sr2qPauWdfaGsOhnAMjbZHCpiampotW7YsXPJG9943LH5jRUSyNwQKQg0NDefPn28eDyJWCJUF7Xb73r179+7d++mnn+7atavfLUPuf3LWqcIzIVQV0yLt0QfNZvOlS5duGTZy0itvN9RW2Sev5wjZLzkPWj1RN7jdAQhbIq2ysrJ58xfYbTakT2u/VkCf1mhzXpvbZ9GiRSdOnNi0adPq1asjtcAYwuMIQ5ICxu12z3jhlfsffHTMhIlFRUX3jH9kyqJ3899cU9nnQWNq74gkwQkQvq7k/977YOCQEVOfeqaoqOjChQuR1kg0wVjwH//4x4oVK9566y0A+Oucl8c9NeuBPzz5+Iw5j8x9w9JQY51ziNTFxnZ11lTzl+kz1PrE0RMmRloX0YSkDyKEFi5a9Mr8BY9PnTrjb891733DDTcPsZmM1unf/Bp1pMfNhCL6tr44TXsOHup53cBbh9zxyvwFy5bHXpib9k6kRVFUY2Pj/v37+w+6lSGUBCkLeRO/wUN5PMzlOsPStf8acd/oR5/+29wXXyorK9u0adPx48fbt+mWCMfQqP9wsRzHFRcX33DDDQDA8/zevXuNRmN9ff3AgQNTUlJ4nr98+fKcBUvsFpPZSfO3PQaH/vnN15tAkwiPfyBrEPbTEMhpAlMVMBRUngEAqCxEgKCuBBgPmKvAZQFGCaZKYGmoPi0cAAhB/QXE0GCuBqcZkSQYLwNLQ2UhAEDVaUAI6i8ihgZLDbhMiGegsQKxnqsHnAHEQ/1FYCiw1IDTiGgHNJYBx/zmgIZLvLmKHzKVddv27Pnkhhu2J3TuPujGgUoSvvz0Y2+gBJlMFrXhEEGMBTmO27t3L0Jo87d7vv7sYwAEiZlEdl9Uefrtdf80GM30vS/AxSMyl9n6wGIoGg6VhYhjobEM7I3IbfOaDzgWjOVXhVdMhjgWGsvBYUBu65WLXFmIWBaMFWBrQJT9qvA04hgwVoDdAO5mQlv9r0dWnUGsB0yXwVaPaOevQk4QNgDluCKsLmINFdyIGaxMceCHT7vlXacioeDHo1rtlTV1HcaCPM/v37/fbDaXlJRce+21J04XlZSWO13uisuX3TaLgdcCzxEeB0rpRvIk/8RH8M64K5eo5hfhB3/l6lEOsNWDx4Xc1itdr7YYEA+VhYAAqs8C4qHh4pXrTzvAVg9OM3JbwVwFDA21564cDE0OFixIO8BaD04jooSDKag7f/VgBNVFV7r2lYOdYK0HS41dlWZN6l1+avtxe6K67LCHpgFg7ty53njiHcmC33//vcFgyMjIEELIms1mnU4nnGBFRYWQpZUkSZvNZnV7Vr/zXmZaaklZhdVmA5eF6N4fTFWAELAMeJzAUAAAHAMeNwIeAIBngaEQ5QDEI8QDSwPlAI4FhID1AO0Cxg0IEMsC40YUCQCI58BzpQgIRTxOAACeQ4TMRqpItQyxzO8G3Ezw7LU5uT26ZWsTk+oNjXcPufXhhydWVlbabDaE0PXXXy+coNPpNBgMADBq1ChvEOBgLEiE4V1Kr9cbDAaNRuN2uzMzM202W9Nvc3NzAeDYsWMAQNP03ffexyJw2a06jbpLly5Go9FqtRKaJIah7XZ7alp6fU1V5+xujQ31yalpTptVqdEi1sPyoFGrbDZ7WkZ6XXVV5+xuhvr65NRUl8OuUGmAY1geaTRqm9WaltGprupy567djYaGxOQUl8OuUKmBYxkeaTVqm8Wa1qlTXXVlZnY3Y0N9YnIK5XTIVWqCZz0s0mk1Fos5vVNmXXVlZlY3s9GgT0ymnHaZUk0izsNyOp3WYjand8qsr6nq1CXbZDDok5I9LgepVJPA0R5Wr9dbTMb0zM5KEvZ99z9v6LzExMRoztYUuAUpihpx1z0gkxsMjWp9As94PCyXmJTUUFOVmd3NbGzU6BMQ6/F4uKTkpLqaqs7Z3cwmo0arQxxDe9jklGTBfGZjo1qrJ3iW8jDJKSmCySxGo1qnA+63QpNRrdUBz1K0rxBxLEXTKalpgr1sZqNSoyMQ56bolNTUuurKzKyuNrNJqdGSwLvcdEpqan11Vaes7KtC5HK5U9LSfnvkFWFHtaDQBzkE1VWVnbtkGRqN2sRkymlXqLU8QyFSriAJysPodFqbxZqant5QV5OZ1bWhtiY9s7PFZNQnJrsdNqVGy3koRMrlBFAeRq/X2azWlLR0Q11tZla2cLDZaExIunow7Qa5UkYg2sPqtFqbzZqSlt5YV9spK7u+pjqjcxezsTExOcVltyk1Oo52CQd7GE6rUdts9pTU1MaGuk5dhIOzzEZDYnKK025Ta3WM2wVypZxAHpbTaNR2uz0l5crBHdqCv6comgQ+PT09Ozvb7XZXVVUhhHr27Hm66KxcqbSbTdqEJNrtVGn1ToqmPSztdiqUahkJiJAxNEWQpEIuA0LGsizHMkqViiAIHiHa7VKptSQJCAiP2y1TKuUkAYTMQ1NAEEq5HGQylmU5hlEqVQRJ8Ahol0Ol0ZIEgQjSQ7lkcqWcJBBJsh4PAhCKcCzHeGiVWk0QBAKgXE6VWqPVJ6iBuenGG0+dPkPKFS6bZdCgQSRJVlVVmUwmHki1WrVr57ehsSBqf7RardvtRgg5nU6tVuvz7cCBAwsKCnyEr732Wk1NTTCN5ufitb6BAAAgAElEQVTnV1RUBFPDihUrSktLg6nhzTffLCkpCaaGKCFwC9I0PWvWrOBbXLduXfNfhVgKCgrWrVsXvDKzZs2iaTr4eiKIhD4oEORv+Pjx4+vXr5dcHCH0wgsvCPmZpfH111/v2LEjGAWiBMkWRAi98cYbly9fDrytHTt2fP3114EfX1tbu3jx4sCPD4NKYgnHG2Fubu7//ve/3NzcCxcuPPDAAyUlJU2/HT16tMVi8Q4xCRQWFubl5fmPNe6fM2fO9OzZU6fTSa6hqKjo2muvDSbnwC+//NK1a9fExMQ2j1y8ePHgwYMlN9TeBG5BnudPnDgxaNCgIFssKSlJS0tLSwtqut5oNBqNxry8YBMA/fzzzzfeeKP/UZcOY0EfAv8Nt0hjY6PZbBbeV6RRUFDQv39/yTmwKisrCYLo2rVrm0d2VAuC+Juh8PrYrVu3AI93u90lJSX9+/cP8Ph2UikYC4bDEc6ePVulUuXn57/yyissy65atarptwaD4eTJk+2tQ5QzYMCA4LO6tx/Ygm2CLRjrYAvGOsFYMByO0GKxPPbYY4WFhUIKmLAFv8eECmzBWAdbMNbBFmxXwuEIMRgMBoOJWqJ3uTAGg8FgMGEAO0IMBoPBxDXYEWIwGAwmrolqR1hUVBTM/oetW7def/31ycnJw4YN81lt3CbBR/YLpvUOQJC2g6AvYKhiM8a5HQWCsWYEu2E8227IkCHEVaZNm+bzbfML6/9aNa3tiSee8GOUFk0WeOWCqmKVDwHtt0UxSCwWy0033SRZw4qKCr1ef/ToUZfLtWrVqsGDB4sq/tJLLz377LMURT377LPz5s0Lc+uxTpC2Q6G4gEFaMFRqdACCsWYEu2E8247n+dTU1KqqKrvdbrfbhZ34TfG5sP6vlU9tc+bM8WOU5iYTVbnb7RarfEiuWJQ6Qp7nx48f/+WXX0q+me7fv/+pp54SPjc0NKSlpYkqnpeXV1xcjBAqLi7Oy8sLc+sxTfC2Q6G4gEFaMFRqxDpBWjOC3TCebVdbW6vX62+66Sa9Xj9u3Lj6+nqfA3wurP9r5VNbTk6OH6M0N5moyuvr68UqL/0yNSFKHeGyZcvmzJmDEAr+nZVl2WnTpk2fPl1UKZ1O53K5EEIulyshISHMrcc0IbQdCuIChsqCQaoR64TKmhHshnFou5MnT44YMeLkyZNGo3HKlCmTJk3yOaC1C9vitfKpTS6X+zGKH5MFUvmkSZMkKx8M0eIIe/fu7R2q3bdv35133unxeJDI7te0EoHdu3cPHDjwpZdeYhhGlD7+I/sFiOTWY4uQ2A6F1HwoRBYMXo2YI3hrRlU3jB/bNb/sAjU1NSkpKT5CrVbrDT3ovbCBXKuamhqCIPwYpTWTBVi5j6qtKR+qru0lWhxhU+bPn+8zkXno0CGxlfA8P2/evDvuuOP8+fMSdOjVq5cQa7ikpCQ3NzfMrccuIbEdCsUFDNKCoVIjpgnemhHshvFsu4KCgiNHjgifGxsbO3fu7HOAz4X1f618apPJZH6M0txkoirv3LmzWOUDuyRtEI2OsCmSB2QOHz7cu3dvs9lsv4qo4rNmzZo3b55gwrlz54a59Y5BMINpwV/AIC0YKjU6DNKsGcFuGM+2+/777zt16vTLL7/QND1z5sxp06YJ8v379wsffC5sa9dKON6ntn79+jU3Sms1o9YN0WLl06ZNE6t8SK5Yh3WES5cuDWZ9rNlsHjVqVHZ29pgxYywWS5hb7xgEc9bBX8AgLRgqNToM0s49gt0wnm3H8/zatWtzcnLS09OnTJlitVoFufci+FzY1q6V8MGntsuXLzc3Sms1o9YN0WLlVqtVrPIhuWI41igGg8Fg4pqo3lCPwWAwGEx7gx0hBoPBYOIa7AgxGAwGE9dgR4jBYDCYuAY7QgwGg8HENdgRYjAYDCauwY4Qg8FgMHENdoQYDAaDiWuwI8RgMBhMXIMdIQaDwWDiGuwIMRgMBhPXYEeIwWAwmLgGO0IMBoPBxDXYEWIwGAwmrsGOEIPBYDBxDXaEGAwGg4lrsCPEYDAYTFyDHSEGg8Fg4hrsCDEYDAYT12BHiMFgMJi4BjtCDAaDwcQ12BFiMBgMJq6JvCMsKSlxu91iS3Ecx/O8hOYQQizLSigIAAzDSCvIsixCSFrZ6CdAC0q+CJILSraX5IIxirQ+GExX4nme4zhpZSX3fei4li0pKbHb7WK7CcdxYq0gzegSunAwvxAJSHeELMtOnz49IyNjyJAh1dXVgtBsNo8ZMyY1NXXs2LFms9mP0MukSZOKi4vFtu50OiV0XQDweDw2m01CQQBobGyUVtBms3k8Hmllo58ALSj5IpjNZmldwmQySbhjIoSMRqOE5sJPZPsgy7IWi0Wa5m632+l0SisbTG+S9pOIfiZNmvTjjz+KPTWXy+VyuUQV4XneZDKJKgIAVqtV7CMIRVEOh0NsQ5KR7ghXr15ts9kqKioGDx68aNEiQbhixYprrrmmtra2e/fuK1eu9COMRWpqao4ePfrTTz9FWpG4oLGx8ejRoyUlJZFWJHqJoT544sSJo0ePXrp0KfxNY0JIYWEhRVGR1qIdQFIZOHDgqVOnEEI2m+3nn38WhHl5ecXFxQih4uLivLw8P0Iv/fv337x5MysSk8lktVrFlmJZ1ul0NjQ0SCjIsmxa9jWJeYMSUtLEFmxoaHA6nX4O4HlesiEizsCBAwsKCto8zGg0UhQVeLU3D/+9OqPbmIcebWhoYBhGgmJ1dXUcx4ktxfN8bW2thObCT6j6YIAW9MHj8RgMhkCO3L17ty6zu6ZLz8em/lmQOBwOq9UqtkUBk8nkdrullZX2k4h+Bg4cuHv3bpZlRZWy2Ww2my3w48vKyjQ6/cGDB0VqhxobG2maFlXE6XRaLBaxDUlGLtmDVlRUfPbZZyNGjOjZs+e//vUvQVhdXX3NNdcAgPD46Ufoxel0zpw5c8iQIaJat9vtMplMwrOJx+NxuVxI/JwTQshYXQFvH9Muvl7s0JnFYqFpWqlUtnZAUlKSSqUSq1LH5nLZJarf2J3ffZrVtetzM2YMGzZs3LhxkVYqughVH7TZbKNGjTp9+rSo1hmGsdvtgQzHnTt3DnKHunsOc7oPNTQ0AIDL5eI4Ttq7hcVi0Wg00vpLY2MjQogkWxgJw32wTex2u9sZvuHKcCLdEQqPEmfPnl27du3TTz/9ww8/AABCiCAI4YN3XqdFoZeEhIQvvviiU6dOolpXq9UymUyn04lVm6Zpp9OZmpoqtqDXdxIEIVZbuVyu0+lwNxONtQ7l3M6XHl+97VhhSRl2hD6Eqg/q9fo1a9akpaWJap1hGLlcHkgpvV5PECQAqFRK4Xi1Ws1xXGJioqgWBUiS1Gg0arVaQlmWZdPS0lp0hC0KMU0J5+qVMCPd9hkZGc8//3yXLl1mzJhRVFQkCLOysiorKwGguro6OzvbjzCmkbxUJ6pgQ7TUon0hZQjx/HX3dtxVt9IJVR8kSbJLly6ydsPrYwiCbL9WgkR4UMD44cnpz0VahfZCuiP8/e9//9FHH9E0/cEHHwwaNEgQjhkzZsOGDQihDRs2eJ/fWxTGKh63m0XJaRl9ru8faVWCJYaWWmBaJE77YIejqKio6eBW1D2MXqXs4sWItBsGpDvCZcuW7du3LzMzc+/evevXrxeECxcuPH36dLdu3c6ePbtgwQI/wliFcfMMzb76syf212B/+umnL7zwglarXbhw4V//+ldBuGXLlhkzZqhUqhkzZmzevNmPsCkURfGhplkjKORNhJB2tFPrxGkf7FhYrdapU6c23cYQtQ+jNOUGTaJ3NrojIX2OsHPnzrt37/YRJicn79ixIxAhJuKEaqmF1WodOXJkRUWF/+YsFovb7fazYqgpW7ZscTmdkPHr1KzHwxgMhoBPDgBAWNMkdu4HIWQ0GmUymahSycnJ4Z8Dxn0w1kEITZ06dd68eQ8//LBXuGXLlq1btwrPnePGjVu2bFlrQi8sy/7www+DBw8W9bulaRoAAuySAMAjHgBoDyMUDByPxyOsGhWlG8OIa0ihUEie6JXuCDGxTqiWWiQlJX355ZeZmZn+m1MoFAGuGPrqq69mv7GGcjoBwDtzo1Qq22yiORkZGRIcIQBIaAuDEcuKFStycnImTpzYVCjhYZSiqC1btjzzzDOiHKHwGiq2gwib0EQVcTqdBEGI2lPvdruF1ViBF9Hr9YE7dR+wI5QE7TTZXZndc24ZdOP2zV9GWhuJNF1qsXr1akEorKrIzc1tvtTCR9h+uFwuWY9BYPqmvRvChBkPTRsMBo1Gg1emAMD+/ft37tzZ/J1e2rrfFStWpKaminKECoUCABISEgI8ngACEL9j53elpaXeOelAQAglJCSI8lIul4thmKSkpMCLBANeMSwJu4FhmIYH8i9WVEVaFengpRaYsGK4tHXr1uxres6a+2KkVYkK9u7de/DgQaVSKXg4giAOHz4M0b72HrlslkhNircf2BFKhZRDclaklQgKvNQCE1YstTBwLPNgPtvR7qISWbp0qTeyCQAghIYOHQpR/jDa4VyggHRHOGTIEOIq06ZNE4RRu/AX0xxhqYXFYjl48GCvXr0EobCqoqqqatu2bd5xiRaFGIwUEDp16uSCVxd99913kVYlSonqh1FeYr6RKEeiI0QInTt3rqqqym632+127wxT1C78xWA6HjH5MHrh8GkDs3pv8c69+yOpRpTRdEUlfhgNPxIdYX19vcfjGTduXJcuXSZPnuxNbCRtFxoGgxFL7D6MEt37w+9GRlYHDKYpEleN1tXV3XzzzW+99Vb37t1nzZo1c+bMzz77DKQG3X7xxRe//FLc2ksh6LaEtGQ0TbtcLrGL1hwOR7PnaAQAPMcF8nxtsVg8Ho+fnQN6vV5YwYXBBIj3YfT8+fN33XXXBx98IITflLALjabp//znP3l5eaIUYBjG6XQGEvOToigECOA3ne6XM2dWrlx544033nbbbaLadTqdHMexknICO51OjUbT4oYBtVotarE+piMh0fADBgzYt2+f8Hn58uXXXXed8FnCwl+FQjF8+HCNRiNKAYZhZDKZ2FIAQJIkQkhsweEPTLh44cJvZQQAECQRSFUURfmPl48D/mLEEsKHUf5q5ilRCjRd6CGaS8eO11l/Wr912n01t956a9jaDUpnTMdFoiM8ceIERVGDBw8GAKVS6b3FS9iFplQqR40aJTaWvMfjkclkEiLQEwTBcZzYgheKCt1LimFWl2a1kYFUpVKp1Gp1tGWfGDJkyNGjR4XPzzzzzLp16wDAbDZPmTLlyJEjQ4cO/fe//52SktKaMMzwPO92u9VqNd6CJhDCh1GNRvPSSy8Fvp9MgGEYnucDKaVWq4nfvg4CxxJ5Q7mUbgqlWWy7LMtKzj7hcrkSEhLwc6cEGhoaaE9HTMkLAJLnCJ1O54QJE4qLiz0ez+uvvz5+/HhBHtULf0MO4s1G499mzVm//p+RVkU0MTbDZDccOnhAp9OdOnUqAq1HJULOd+Fz84dRiNJdaJhYpbS0FHXMrRMAAITkEYb333//rbfeslqto0aNevfdd4XUYhaL5bHHHissLLzxxhs//vhjYXVTc+F9991XX18vVFVdXZ2eni72bUnY0SnhyU4YGBFbsLyiAqV0BeNlSMkGSw0kdAJHI2iTwWUBpUZN8F06X4nI1bt3788//9ynuMlkirZ8hHV1dbm5ub179/bOMAlJFnv37r1169Y+ffqcO3dOmH9avnz5m2++mZmZqVAoGIapr6/v2rVr06qqqqq0Wm2br4kcx5EkGcj7nMPhMDooRLtAqQGEgKFAqQHEkzzXuVN64PEpAm+xKbm5uStXrhQbYk2lUokNTxokhw4dmjhx4oEDB3Jycl588UWapt9//30AmD17tkqlys/Pf+WVV1iWXbVq1fLly4WvUlNTTSaT8KFpVdL6IEKI5/lAztrhcDTa3VfvNAgIEngOZDIg5YlyPk1kclCe54WFsq0d0GIfFKivr5cQdS/iLF++/IsvvvBzQHV1dWpqqtgZH1F3UZqma2qEQXWUlZUl6tfSpsl86N2794YNG8IZWUaiIwySrl27/vOf/xSb3jb6KSkpyc/PLyws9JFHoSM8derU7NmzvTNMHo9HmGHS6/VCECy3252ZmWmz2Z599tnExMRHHnkk0iqHg5KSkqVLl27atEmsI9RqtWFe7hT4w2i8WbDFPigQo44wDi147NixcDrCiK2Suu6663xeLDoAMdTBRM0wde3adeDAgRHRM8yQJEmSZGJiYvRv1SIIYvr06dOnT/eRt5hoIq4sGGkV2gVswfZtNPxNYqIBUTNMGAwG04HBjjBOEbXcCYPBtAc7d+7s27dvcnJy3759d+3aJQijKAxQ3IAdYZwydOjQRYsWjRkzJjs722w2r1ixQpBHS0hDDKajw/P8Y4899u6775pMpiVLljzxxBOCPPKLtOMPHEkhThE1w4TBYEIOy7Iff/zxyJEjHQ6HSqVKTk4W5P7DAGHag2DfCIuKinQ6nfffaA/4i8F0OALpg5goRIgl4nQ6ExMTx40b9+GHHwpy/2GA4gGe5x0Oh8PhsItBWtQ9gaAcodVqnTp1qsvl8kqidDs2BtNBCbAPYqIWvV7vcDiWLl06c+ZMQeI/DFA84M1ULIpgWpQ+NIoQmjp16rx58x5++GGvUELAX4HxD0+qrDNIVkYUShK2b/oiPT29nepv/tvlrtJaEQn7viPI22ve+e9XW8PTFgnw6gszx44dG57mYovA+6AP2ILRQHl5+dq1a1etWqXT6f785z/n5+cLcv8xKQU6tgUJgtDr9QzD6PX68LQo3RGuWLEiJydn4sSJTYUSAv4KHDx02DJuJejTJOsTOOp/Tqao9gqax3Gc0Wj0EVosFpqm/YRESUpKitR2+6KioltvvdXpdAr/BhJW9Ezx+ROqvtB/dBjUUx54T9jOgWlO4H3Qh45tQY7jWjtxg8HAsmyLO9VSUlKkxS+VTFZW1j//+c/Ro0cPGzbsiy++8G4TFFZu5+fn+1m53bEtGH4kOsL9+/fv3Llz9+7dPnIJAX9/pddgSAnHxjVS2Y4uRyaTNY+YI5fLoy2yjEBrA2tfffXVnDlzVq5c2epEfWZueFLKkae/DkMrsYioPtgCHdeCMpmsSxff+PhXlCHJ6Ikso1Qqt2zZMnv27LKysj59+mzYsEGQL1y48LHHHuvWrZsQG6jV8h3XguFH4g9i7969Bw8eVCqV3sHcw4cPQ8cK+NvQ0DB58uQuXbp07dr1L3/5i91uj7RGIcY7sNZUGNNZlDmO69OnT1NJB145IqoPxgrNLdiasGNw5513FhQUmEymo0ePes8xppPRNzfW1q1br7/++uTk5GHDhpWUlERKMf9IdIRLly5tmtkLITR06FDoWNknnnzyyR49elRUVFy8eDElJWXx4sWBlLJYLO2sV8iQPLAWnaxZs2bw4MHnz59vKpSwcoTneYvFYhKJhBzRQSKqD8YELVqwRSEmOmlurMuXL0+ePPnDDz+sra0dO3asd69ktBHiIYIWt2PHxB7tp59++u233xY+//nPf37rrbcOHjw4d+5cpVKpVqvnzZu3adOmQOoRArpHP8LAWvORz5hYsdbcWADQr1+/V1991edICS+4BEFotVqdSKInuXmMdjdoxYItCjERJ0ALlpaWTpo06fbbb9doNI8//njUPtCEwBE2zV/R4kt9TLzpP/TQQ1u2bAEAmqa3bt06adKkQYMGLV++3GKxNDQ0LF68OLZej9okpgfWmhsLAEaMGDF6tO/aAQkvuARBCMFXRRHZaac2+2C0EbgFWxRiIk6AFhw+fLiwP5LjuIULFz766KMR0bZNomLSOBoYOXLkL7/8Ul9fv2vXroEDB2ZlZX300UfCk/Xtt9/es2fPVJFZ06KcmB5Ya26s1o6MiRfcOCRwC2KiE1EW3LNnz80335yUlLRmzZqwaSiKaBnPiThKpfKBBx7Ytm3bgQMHJk+eDAA6nW7Lli3CnoeDBw/27t070jqGg0BXrEWU5sZqjUC2ZGHCT+AWxEQnAVoQIfTKK68cOXLk888/z8vLC6eGosBvhL8yceLETz75ZPfu3RMmTACAF1988ZlnnrHZbLW1tfPmzXvuuecirWB7EXMDa9DMWK0REy+48UmAFsRELYFY8OjRo1u2bNm2bVtWVpYQNS2cGgYOdoS/cs899xQUFIwcOVLI9L1q1SqTydStW7e77rrrySeffPDBByOtIOZXfIzVGjGxciQ+CdCCmKglEAseOHDg/PnzKSkpCVcJp4aBI90RBp5JK1b2cqnV6r59+3pf89PS0rZu3Wq1Wn/55Zenn346srphfPAxlpemb7cQOy+40ojpbHYBWtCPEBNZArHg/Pnz0W8Jr46BInGOUMiktXHjxhEjRmzevPmJJ56orq6GVuKSBBKsJDE5lflqFqHUBnMyAULbW7g7MAxz5syZysrKe++9Nww6xDSpqamqPR8rLv8YhrbY0p9g/PU+QmwsENkHfcAWjHWwBUOLREcoKpNWIFGA9+3YcuzYMcmnIY5n7m++BHT79u1//etf161b5yciKEbgjUULBvQN29Kh+2+55RYfETYWBJfNDlsw1sEWDC0SHaGQScvhcCQmJnq3oEEQQbdzcnJycnKkKRMSHnzwwZDMAiIEzcejLBaLx+PxE2tUr9crFIrgWxfFzp07Z8+eXVNTk5WVtXr1auHJLpCg2yqVKrIr/UJlrJhGVB/0oWNbkOf51gIiOhwOtVrd4qZPjUYT/qgIW7dunT9/flVVVb9+/davXy+sq8R9MPwEtVgmwExa8bWXiwCNeMK/HVsYWHv33XdNJtOSJUu8oY9wNrvYAmeza47YPHbBZ7OTRmvhx3AfDD8S77/l5eUvvPACAAiZtIqLiwV5Rwq6LQ0CQN0MlUrVXNgUmUwWZj29A2tOp9NnYC12g26HBMF/iCX8qwBE9cG4Qshm1yI6na61r8L/Otha+DHcB8OPRNuLyqQVSHotTJgJZmCtY8PzvNlsFjtSHf6MksFks8NEA8OHDx8+fDg0Cz+G+6DQBxmGETWkodfrJU9YSp8jDDyTVkwEK4lPhIG1NWvWzJw586effgI8sAYgk8nS09ObJ5WMNoLNZoeJDvbs2fPiiy/ee++9S5cuFSS4DwqB7xmG0el0gZcK5p1eekkhk5aPUNi2FYiwo8Jy3O7du3v16tWjR49I6+KP8vLytWvXrlq1ShhYy8/PF+Q4JlkMEXgfxEQhrYUfw32QIAghkH3YRllwZJkQ46bosX/6y3//+99IK9IGwsDawYMHEULNB9ZwTDIMpr1pLfwY7oPhJzJBtzUazcKFC/V6PQB4PB6FQhGRVVvNYRiGJ0jgOJDJgWNBJgeOAbkSWM9v/soUwLFAygBxQJCAeAASABmNRqTQ0Df/MdLn0TaBD6xpNJqvvvqqqKhIGLFpPnkWEgvStAeEGhAPBOn3AwIAhVze5lJb7xBT4DQ2NorXPdoRLOgnFVxo+yDP8wzLAUEAQgAEgPevlyZCgrhi1l//Ct1KBoDkJCF2HVkMWdAbfswrEZZcSeuD8gB6hGQQQs1TT7fT61pELEhEJObNL7/8smfPHuHzqlWr/vCHP3Tt2lVUDRRFkSQpYWqUZVmPx6PV+oawcTqd2//37fkLl0w9RpBnvuUHPUQc/xwNfZI4+AG6dxZ8uxLGLoItr8IDL8N3b8LQJ4gfP0fX/5648D3q2o80lPL6DJJx8xxLpnRB19782i3qpgkqTSaTTqcL82KKUOF2u4WMYq0RoAVdLpdSqWxtHP+lV16lkruTSjWqOoMy80hShurOo855JM8hQynq3Jtk3MhUhTJ7k5RVhTx/nTxRWFDgB7vdrtfrxd7f09PTR44c2blzZ1GloplQWdAHjuMoimo+i3P27NmPvz3s1mWStlpen0G6zLwqgWTcPCknCeA5llSoedpJapN4h5FIzETmGkjrDoYyyMwl6s6h7OvIqjN89wEKl2ncDZ3vuOMOn/pdLpdCofCzmikjI+MPf/hDi1/V19dnZGRENnmkBAKx4NixY3Nzc0WdGk3TEJg/KykpWbvuA0DoyrMLkARwq996K8CGnE6nqLXxGRkZ48aNYxgmbGERI/NG2Ldv3759+wqfP/roo0mTJt14442iarDZbDKZTNRUqgBN006ns3lkmWkzZh5zpZNMGVx3D3HxMAwYQ5zahm5+GI58BLf/Cb57E4Y9BVtehdsfgz3vwICxcGo79BlB1J5FPW4mOZrPyCXdFp6hibRuURpNTyoajcZ/5o0ALej/aeDV15dRGT0IbSKqOw+dckiZnGssh069SM7DWWogM5d0W3iHEXXOI201St71yCOP3Hrrrf5blHbXQwjV19eLKhLlhMqCPjAMY7Va09PTfeTffPPNFwVV7vTfkTVn+E65pOkyn5BJui28XE0QCBgPoUkAu5FIyQJDGZHVBxEkXDsIGBp63gKUBXJuJ+wNkDtUZiq7886+M2bM8KnfbDZrNBq1Wi1K25gmEAtOmDBhxIgRol6ghbADgQTC3rt379oP/vmrIyRkBILAE/IYjcaEhARR7y0ulyvwg4MH5yNsQt5Qsq6Yj7QWGEyMYrVa//y3uUWFJyg+OdK6YDAikD5EsHXr1uuvvz45OXnYsGElJSWCMKazT2AiAkLojfxlNE1FWpHYI9r64I8//rjzaMF5TV5oFv0jZLFYqqurw/xygIlDJDpCUcGBcMQgjB/ef//9xRu20LTvVLwfWA81b/HS5Sv/r/20in6isw/KE9KgU2iCBnMNlxbnr+jRKy/612BjYh2JjlBUcCD/EYM4jjt//jwTaXhe0phoK/OBHMeFo/XgiJ73CcW1N4pa0uI2NRxokG/+366QaxJDhLAPIoSMRiMfNKFdecfbGrnxSxW3/wEhFFQwvx4AACAASURBVLxubRJCzTExh8Q5QlHBgfxHDHK5XAsWLLjnnntEKWC322UyGcuyYjX3eDxOp7PplHJhYeGERx+jaBpGvyqmH/96bNPuL3ykadpqtXqFNpuN4zg/c8WJiYlhzmYivE/s2rVrwIABa9eufeKJJ44cOQJBZJQMN936g+VopJWIJCHsg3a7/dFHH/VGKw0QhmHsdntT52exWIR/BdnVvwi8vaVZT/GlSZGrAmS32w0Gg1ditVqF+L2itBUwGo0A0OL6qeTk5Bhd1x0hUGFhYf/+/SOtRmgIarFMgMGB/EcMSkhIWL9+ffO1Z/5RKpWSV42q1eqmq0YPHz5sv+mPsnP7AEDMQnvi1z/Eb6QIQKvVNj0jkiSjbfuE930CAB5//PHly5cLcgkZJSmK+vDDD1etWuW/RafTiRBiGMZHTtM0avXV+le8d9MrN0mGvlBWcfvwu9eszPeuQG7eooTMHgghp9Pp3d0cIGq1OvxRmyFEfTAxMXHjxo2ZmZmimmYYRq1WN/2dp6SkCK0QTfrFFYlwxG97Sgs0KXJVQCQmJjbVTalUBrNqNBa3T0Sc5uNASKFbnL9yyxefRESfkCOx64oKDhQDEYNUOiBDeRf7/vvv8/Pz77vvPrFL0sNGCN8nBM/R5rAYukpzuZQTMFfZWNmZ8jqz2dxaDa21KE3PaKOj9UFMFPPvTz73FRHARXsXEYHEu78QHOiHH36Qy+XCs7MQJgZnnwAAVHJoL6CD679KTk6OWkcoEJL3CY1G8/zzz7e5G4lhmBZfi9VqNdEs9EhzCAB09T1DOJLQJitUpFarba1pl8uVkJAg4Y1QKCiqVPjBfRATNpwdfeGuREcYeHCg1oQdGYRQ37vkjrpI6+EP/D4R68RLH0TIbDZXVVWlpqY2DwiFwYQEiWPl8+fPR79FkAuR76uqqrZt2+aNjtOiEBNZRAX8xVGAo5Co6oMGg+HWkff/cepT7lC/OjD1pQuWvNEzt/fGjRtDW3OUwHFcnz59mkqicDe2xWLxkSAETqe4efRoBk8axyne94mEqwjyhQsXnj59ulu3bmfPnl2wYIEfIQbj5ezZs8VVjcbrH+T4EE8cIYeRfWiF6paJoa02SlizZs3gwYN94qFH4W7sM2eKfEU8t+/b7WFWo/3AjjBOiar3iWBgGEbCLhpMyJGptKAXt/Yb069fv6bR+QUk7ARtbzjeZ2UAAl9JbBPXsUbLyspGPzK5tqIUbnsq0rpgpOC2Gu++595bBg/94dCBSOuCwYhmxIgRzYUSVm5brdbRo0eXlpaKCrotTIgEE8EuwPD0ZrPZ7Xb7SRjSHLfbzTAMRYmIvBjMTtC4doQ7duy4KMv26OhIKxK/FBcXHzt2jOc10oqzThsas4BqOBBSpTCYSCJtJ+iaNWsyMjJEOUJhL2YwC6QzMjICOYwkSQnZJxiRaZiC2R4a1NBoTEzz+odMSAeVvj1q5h2m11at6Xvjrd7Mi5jm3Pn70Z9/e4ANSYzmuCQa+uChQ4cOHjzIcniAOjQIi7QBoPnKbR+hF4Ig1Go12T7s2Lmrtb1Nf//739upUQkEc82lF46Vad7W8Hg8zaOchBC29kJD7zGXuQQhqlN0EvHbqNNmYbP7hbDCuCIa+uCpU6dGPfKnZf/a7Ha3Y/4Qzm6at2T57wbe8t1337VfK1FCtK3c/uBf/2l5l68mcd2/PwunJu2HdEcYK9O8LVJYWJiamT1n7tx29YWQfg2p980AHD1Ew200JJgbG5YtW1ZRURFpRcJNNPRBl8slS+lC3zYlhHU2h2korb/+4csoJZofK0NFzKzcjvroS4EjfY4wVNO8Tqfz+eef37p1q6jWhaDbHo+I3D0CNE27XK7jx48T192FqKaDOWKNin7903ooYSH8mMVi8Xg8fiZy9Xq9qJnkkNCvX7+cnJwxY8Y0FUqINRphjBVVBvOS1f+45ZZbhJ9Z/BCqPuhyuV5++eXPP28WRssvDMPYbDafuNsC6NfO4fNFix+bHfWboNtXSb+WNF0QehNN09KWRVitVrlc3uIwWkT6oIDPBRQWafsc06IwbLRmLA9NeScvY5oQL5aRMM2rUChGjx6t0YhbLsEwjEwmE1sKAEiSRAgplcqgTUe08KlZKGHiaoxgjUbjp+sGOcAtjVDdRu12+8SJE48fP+6/ObPZTFGU74T51VtA8+wEVwXo1wNbfOxAiEjOUiYkWq3WxsbGpmVNJhNBEGKvLULIZDKJjaAd/vwhrSGhD8rlciGdk6iG5HI5wzAqlYpoFkC71c7VZtBt4Sui5c7F2RrnLX5Du+LNd1Ytu//++0VpK6BWq1uLwx6RPhgTWC3mlj0hz10oOklRlIT7cLQRYkcoIUCXUqm8++67xcaS93g8MplMQgR6giA4jlMoFC35LPEQbRUlCIVCoVKpJCeOCTMSbqNarfbll19uc30Xx3FNY40aDIap02fSPu/0za7krzdZ4tdcH6jZ4Sztem3l2yOO/LBq+RveshRFJSUlSXCEQkFRpUSt1mtXpPXBsWPHiu1NMpmMpmmlUtnC7//X1CyhhGmscg+ZRpfsoShKWvYJoSdinycKyt3K/gqEQK7++OOP//KXv4RXo9AT4h9EtE3zNsdqtT4xY/ai/JUShlXjAQkr1mQyWV5enkIkGzduPHDZ7XWrbd40/R9AG2tOQ9ddBw+LVSNURM+9Nfr7YFAkZ5OaxEgrgbkK20H2noW490bzNK/H45m3aOm4hyd9e7jgorxrs1gJGIDw3kbJjJ6hqgoAICOnztA4asLDp06dCmW1sUbY+qDRaLxn/KOjxk90OsObmoDnysrKCgoKmua+xrQfjYZGP9/OnvuCyWQKmzLtRLBDo9E/zevl9OnT7/xjvbvrIJmagtTuYKsJT7scx7EsG/357QRiOHeBtdbqpA4WXqitrR0wYECktQkfkeqD5eXlF6vqHUOnkz99Edqa/UMZLi9ZfWrJspUbP/nP2LFjw9l0HLJt27YyP+uxZQqnw7Fs5f+tWp4fRqVCT7xEltm+ffvOnTtBnQDd+0NRmFwgAHgaqyZP/hMA+uabb0aNGhW2dgMnhh5l2kaTpEjpDACFhYUGgyE7OzvAyBcYaZAKJYR9gxCiHPRDK5MK/hvmduMTl8sFiG/1a5IEQvXZV18vf+P16Jkml0C0TGy0K2VlZQ9Pnrru6GWaDve8IGdvRE9/nHBDC+sz45kjR458++23PN96B5MK4tg9e/YMuXf0mIl/aHMhKwaD8U9RUbO8Ez4QUF1d9fe//z0s6rQXceEIEUIKXRJ/44ORVgRzhbvuvW/XqdL2cITuxup3//WpW6ZV9hx0+PDhb775JuRNYKxW6/DRD419+A8ulztSOiDWs3379jVr1sRhIIWwQVHU8jYDaCAEPHfu3LmysrKwKNUudPyh0bKysp9//pnH0SyjhtraWoQQdMppj8p52s31G0OeP+iuL/vou+rd3x+76aab0tLSpK22xzTnwoULP/zww08nC123PU0e/Xek1HDVlv7rJ7n8650///xzbm7ulClTrr322kgp01Gpra3lCEUAwUbQp1//r6Ki4sMPP+zSpUs4NAs1HfmN0Gg0njx58qYhw//80utuMek82gOWZT0eT6wsmWk/vvvuux55fcOwdwWxDNvnrqqa2mt79sLvhcHj8Xhmz1805K77brj1jr++/DrF8JAc0Vsez3F3PMUSsv9eYJet3XDp0qVIKtNB2blzZ4BHWs3GHTt21NSEb/lFaAmHIwxP9gmbzXb8+PEjR45UV1e/tPiN/rffmd2r7x0PTDQ3Njgmrm6nRgOEsjSMf/BBtVp98uTJyGoijVBZ8IkZsx/545+YIU9AeGIyMW6kSVb1GTp/6coHxk7YuHHj5cuXw9Fu9BG8BSsrK99//x9HNQM9hMI55vWQaygRBHDbH1Wdrjl//vxPP/0kJNjrkIQ/h8/OnTuffX5uQIciHnEcAKxbt06nT4jF3RThcITtF7KZZdnS0tLS0lKXyzX5mefuevjxO+4f37P3dSvz3zh900zaanDODPSJpl3hnFb03PaEnjdUV1eXlpbGXEb14C3odrtnvfjy5//6wJaaKz6sa1DQDeUlZNa3+w4+PmPOyy+/vGDBgq+++urUqVMWiyWcakSWYCzIsuzLS/KnPjODI+TQ87Z20jAYXKb6OUvfuv2eB9Iys7QJifPnz3/vvfdOnDhRVlbWvlH1w0jYAt+bTCabzbZq1aqnp01HgXdVjgWldv1/PnFxkNGtlzIxZcGCBXa7vf30DC3hmCP0H7KZ47ji4uIbbrgBAHie37JlS3l5Ocdxw4cPP3nypMPhUKvVAwYMaGhosNvtxcXFarVaqVQqlKpPNm2z2h1V50+DSp+UlOSkPeyDy4mNcz2Pr4f1U0CdAABQcw4QgoZLwLFgqgSWBmsdol1gb0RuG4AMEQhoB9jqgaHAXIM4BkyXgefAUAYIQf1FQAC15wAAas4CANSeA4SgsQw4BsxVwFBgrUeUA5xGRDuA9SB7I3icYK1HLA3masSxYKxAPA+GUpfVMnbsWNAmpyUnqRVk0akTOp1OuAgymSx6QpM0R5QFv//++/Lycoqizp4vcTicRpMZsR7gue9+qQOEgOeRzQAIgLIjRyMAgNsOCiUCALcNEAcIwGVDchkAApcV8QwgBE4rou2AeHBZEOUAhMBpBtqJEA9OM/K4gefBYQKGAp4DhwlYGgEChxFxHgRyUKhBrmRI9WffHQLnNkKm5J0meVInGUd375nbJS3p4YcmHD52vLSqlkRcnx7dSJK8/fbbKYoym819+/YVwvIJBsrJyTEYDGfPnuU4bvr06YqrYZo7kgUPHDhgNptLSkpyc3MLCgoUCsWq1e9y/ccRHgoay4FjoLECsR4wVSKWBlPVlb7jcYOtDmjnlR5hNyC3/UpHI+WIdoDMDYCAZYFnwG1FCiVQdmQzAOUAWz14nGCrB9oN1jrkcYOlFhgKLDVX+hHrAeNlxDFgrEAcC43liOegsUzoqpzbzoxeQG5f4hm3hPx25YqPNvMOI7IZQKYYPW6CwWylKYqjXX+b9nRRUVF6enppaWlOTo5cLn/uuec6jAVLSkqGDh3K8zzP85s2bUpLS+vcuTP8dn+UcIJCxESEUEFBQVVVVV5e3u9+9zsAKCoqeunl+WabAyHkpmgATlyYPJ4DkgCe43mO58hlb65Z897fc3JyLC5P7+6drTa71enO7pw5buzoCyUl6enpW3d8+/qiV7Va7alTp0aOHOnVk+d5giAoimJZ1ps0+PTp0zqdrkePHl4buVyuM2fO/OlPf/KGBQ7GgkQYZq30er3BYNBoNG63OzMz02azNf02NzcXAI4dOwYANE2PuOc+hgeedirlMheLlCoNwVI8y5BKNeVyUxwvA0QSBMexSKHmPLRSo6VcTo0uweWw6ROTbRZTcmqaydCQntm5oba6U5dsQ11NeqdMU2NjcmqqzWrV63VOp1ujUdEeViknWR5IxIFcyTMemVLN0m6lRke5HFp9gsNmTUxOsZiMKWkZRkNdRmZWfXVlZnZ2Y31danons8mYnJxss9n1eq3LTamVSg/LyUnggCR5VqhQrlJ73G61Tuey23WJiVcqNDamZnTSKGT7vvufN1ht9IRsbhFRFrz73t+bTCZCJrdYbTqdDuRKh9mYkJjIETKX3arX6xEhdzmsep2eI0jK6dDptBwiKLdTq9VxCDxul1arZXnkoSmtRstwHOOhtRotw7Ecw6o1ag/D8Byn0ahpmuERr1apaNoDBKiUKoqmSIJQqlSU203KZHK5gqbccplcoVK6nU6FQiFXqlwOu1KpvPJBpZIrlC6rSafTIYXKYbVqtWqZXGk3NyYlJhIyudVs6n5ND4fVxAORmJTitJlvu+22H374QaFPVpDE3o5rQYZDNdWVXbK61tbVpWdkNjYaUtPTTY2NKWlpFpMpOSXVYjEnJiXZrNbEpCS73Z6g1zscTp1O63K5tVq1y01p1GqKptVKJeVhVAq5h+UUcpLhkIIElgcSEAeEHBBPyAjEIlJOcAzIlYj1EHIVYmhSqeI8tFylZmhKoVIzlFup0XrcLqF7qrV6yuXQ6BLcTrtWn+B02HX6RIfdmpCYZLdaEpNTbGZzYkqq1WxMTk0zGxtT0jLMDbVdunSuratPz+xsqKvr1LmLSk52JAtyHHfs2DEhAOyddw7PyemZmZkJAAghb6xgb8RgwREeO/6z3W6/pmt2nz69EULHfy64ePEiAhKAB4K4Gt73qoNAcPXz1QDAV/8DAECo2XwHAoKUkzK5Si0H3uFyA0GqFWR2dle7w84zjMPlvvfukWazubS0fMSIO+FqIGjhA8/zHMcplUpB1cPHfuickd6zZ0/viTQ0NFy4VHpg397QWBC1P1qt1u12I4ScTqdWq/X5duDAgQUFBWLr/PLLL3ft2iVBmaKionfeeUdCQYTQjBkzpBVcs2bN2bNnpZWNBkJiwXfeeaeoqEhC60uWLKmurpZQcO7cuU6nU2wpmqZnzZoloblopj36YHl5+bJly6Tps2fPni+++EJa2X/84x8///yztLIvv/yyxWKRVjaytGnBp59+uqGhQVSdW7du3bp1q6giDQ0NCxcuFFUEIbRq1aqLFy+KKnLgwIFPPvlEbEOSCcfQqP/I91lZWc8995xWqxVVZ0VFhVKplLBU12Kx1NXVbd++XWxBADh+/LhPGtsAKS4u/uSTT/wkNFi8ePHgwYMl1BweQmLBc+fO/fe//xWb1QEATp8+/d1334n9hQBAQUHBiRMnxAa84Hn+xIkTbe8j/i3xYEEfHA5HeXn5vn37JOhTW1tL0/T69esllC0pKUlLS0tLS5NQ9sSJE0ePHm3xvSHWLbhp06Zz586J2iYkRNJ/7733Ai9CUdS5c+eEwYPAOXv27KZNm7zjnIFQX1/vcrk++uijwIsEY8FwDI3Onj1bpVLl5+e/8sorLMuuWrWq6bcGgyFG11KGkAEDBnTq1CnSWrQKtmCbYAvGOtiCsU4wFgyHI7RYLI899lhhYaEQslnCOwEmsmALxjrYgrEOtmC7Eg5HiMFgMBhM1BK9y4UxGAwGgwkD2BFiMBgMJq7BjhCDwWAwcU1sO8KioiJvcJbA2bp16/XXX5+cnDxs2LCSkpJAigQT6E9Ccx0G/+c+ZMgQ4irTpk0ThP4vtZ9vJbTlRx6IMhhpv+0I9iZpd4woh+O4Pn36NJUEfoX9//59kGY4UU0Ecy5BEbYdiyHHYrHcdNNNYk+hoqJCr9cfPXrU5XKtWrVq8ODBgZR66aWXnn32WYqinn322Xnz5rV3cx0D/+fO83xqampVVZXdbrfb7cJmYdTWpW7tW2lttSZvszkMCuK3HaneJO2OEeWsXr36lltu8TmpAK9wm79/HyQYTlQTwZxLkMTqb4Ln+fHjx3/55Zdif9b79+9/6qmnhM8NDQ1paWmBlMrLyysuLkYIFRcX5+XltXdzHQP/515bW6vX62+66Sa9Xj9u3Lj6+npB7v9St/attLZak7fZHAYF8duOSG+SfMeIcvbt2yeEB2kqDPAKt/n790GC4UQ1Ecy5BEms/iaWLVs2Z84chJDknzXLstOmTZs+fXogB+t0OpfLhRByuVwJCQnt3VwHo8VzP3ny5IgRI06ePGk0GqdMmTJp0iRB7v9St2kIUW21Jg+8OQwS/9uOSG8K/o4RzficVIBXuM3fvw8SDCe2CcnnEiQx85vo3bu3dyx33759d955pzfPraiyArt37x44cOBLL73EMEwgrfsP9NcmYpuLXaRd6pqampSUFOGz/0vt/1uxbbUpD9LuHY/guxKKRG8Se8eIZpqbADU7KT9XuMXiqPV+EWC1gRBIE0jMuYSQmPxNzJ8/32em89ChQwGW5Xl+3rx5d9xxx/nz5wNvsVevXiUlJQghITdN4AWlNdcx8H/uBQUFR44cET43NjZ27txZ+Oz/Urf2rbS2WpO32RwGBfHbDn9vCuaOERP4OI8Ar3Cbv38fJBhObBNI6rkESUyuGl26dKnPVRs6dGiAZY8ePbply5Zt27ZlZWU5HI4AU1qPGTNmw4YNCKENGzaMGzcucFWlNdcxaO3cDxw4AABOp3PChAnFxcUej+f1118fP3688G2Ll1oo0tq3kttqTd5mcxgI4rcd/t4UzB0jFgnwCrf2+w+y2mCaCEmjUmgnBxs2xJ7C0qVLJVwBs9k8atSo7OzsMWPGiErjIq25jkFr5y584Hl+7dq1OTk56enpU6ZMsVqtwrctXmpv2dYMIa2t1uRtNodBQfy2I9ubOmQf9DmpAK9wa7//1pBgOLFNSD6XIMGxRjEYDAYT18Tk0CgGg8FgMKECO0IMBoPBxDXYEWIwGAwmrsGOEIPBYDBxDXaEGAwGg4lrsCPEYDAYTFyDHSEGg8Fg4hrsCDEYDAYT12BHiMFgMJi4BjtCDAaDwcQ12BFiMBgMJq7BjhCDwWAwcQ12hBgMBoOJa7AjxGAwGExcgx0hBoPBYOIa7AgxGAwGE9dgR4jBYDCYuAY7QgwGg8HENdgRYjAYDCauwY4Qg8FgMHENdoQYDAaDiWuwI8RgMBhMXBN5R1hSUuJ2u5tKeJ7nOK7NghzH8Twf5sNYlkUIhfkwDAaDwbQf0h3hkCFDiKtMmzZNEJrN5jFjxqSmpo4dO9ZsNvsRepk0aVJxcXFTicvlcrlcbSrgdDp9PGiLOBwOiqLaPMxms3k8njYPs1qtDMO0eZjFYmFZts3DTCZTIN4Xg8FgMO2HREeIEDp37lxVVZXdbrfb7atXrxbkK1asuOaaa2pra7t3775y5Uo/wiBhWfbEiRMhqQqDwWAw8YxER1hfX+/xeMaNG9elS5fJkyfbbDZBvmXLlhkzZqhUqhkzZmzevNmPMEi+//77seMnhKQqDAaDwcQzcmnF6urqbr755rfeeqt79+6zZs2aOXPmZ599BgDV1dXXXHMNAAivgMLBLQq9uFyuV1999dNPP/VKnE4nQsj/mGFlZSWPwGaztTkCabPZFApFm+OZwtAoTdNtHsbzvFKpbPMwAFAoFG0eJpfLZTKZVqtt82AMBoPBtAcSHeGAAQP27dsnfF6+fPl1110nfEYIEQQhfPAueGlR+KsGcvnAgQObugG5XA5teRGZTCYc2ab/UCgUAR4mEP7DZDIZSUZ+1RIGg8HEJxId4YkTJyiKGjx4MAAolUqVSiXIs7KyKisrc3Nzq6urs7Oz/Qi9KJXKBx98UKvVeiU8zyOEmkqao1KpCAK0Wq3/wwCAYRiFQtHmYTRNazQatVrt/zC3263Vatt8I3S5XIG85Gk0Gq1WKzh1DAaDwUQEiS8iTqdzwoQJxcXFHo/n9ddfHz9+vCAfM2bMhg0bEEIbNmwYN26cHyEGg8FgMNGAREc4dOjQRYsWjRkzJjs722w2r1ixQpAvXLjw9OnT3bp1O3v27IIFC/wIMRgMBoOJBiQOjRIEMX369OnTp/vIk5OTd+zYEYgwSMxmM+Px2Gw2nU4X2poxGAwGE1fE6hqNl199jaao8vLySCuCwWAwmNgmVh2h0+kEhGOyYDAYDCZYYtURYjAYDAYTErAjxGAwGExcgx0hBoPBYOKa2HaEgWRrwmAwGAzGD7HsCJXajRs3RloJDAaDwcQ2wTrCoqKipjv5JOQjlA4Ry14cg8FgMNFBUL7EarVOnTq1aRLdsOUjxGAwGAwmJEh3hAihqVOnzps3r6kwbPkIMRgMBoMJCRJDrAHAihUrcnJyJk6c2FQoIR+h2+1evXr12rVrvRIhH2Fb7SMAYBjGbrf7P87hcCgUijaX1TgcDpZl20xb6HA4AKDN7BMOh4MkyTazTzgcDqVSKZPJNBqNkHwKg8FgMGFG4s13//79O3fu3L17t4+8eerB5cuXKxSK22+/nSAIhJBMJhs4cGDTIo2Njfv37x86dGjTSgBAqKc1NAqZWkF+8803R48e9a9qILUBAM/zBEEEcpj/3IG5ubkfffQRcRX/tQV4GAaDwWDaD4mOcO/evQcPHvS+GBEEcejQoaFDhzZPPVhZWTlt2rRHHnkkZCpHMSUlJfn5+Xq9nqIovV7f5huhy+XS6/U4HyEGg8FEEImOcOnSpUuXLhU+C696wmch9WB+fn7T1INdu3b1eQvsqOBE8xgMBhNzhPjGjVMPYjAYDCa2CMECjaYLW9oj9SAGg8FgMO0HHsrDYDAYTFyDHSEGg8Fg4hrsCDEYDAYT12BHiMFgMJi4JgLRTN5e885/v9oanrZIgFdfmDl27NjwNIfBYDCYmCMCjvBM8fkTqr7Qf3QY2lIeeK+ysjIMDWEwGAwmRolQfMvMXPjdyDC0Q57+OgytYDAYDCZ2wXOEv8JxXJ8+fZpKdu7c2bdv3+Tk5L59+/5/e/caE1e57gH8XTPMDAMzAwMou9MibikwrWxaMPqBHmugSnKodKQaooE0Ia1JNc2u1lh3sO32WA7Emlg5hqSmlOyIzUFrwhma6j6tW7ECzdZdiaUtt9gqFJBrBzr3y3rPh2XnjHNZMw5zdf6/T/adh2e9wwf/mVmL9zl//ny0NgYAAOETfBB6DYlwzeANv9bW1rKystHRUecKy7J1dXXvvffe0tLSm2++2dDQEMXtAQBAmAQZhL5CIi5m8D7//PPHjx/n/nv37t3vvPMOIaS4uPjw4cOuZXa7vbOzs6KiwmAwSCSS9PT0KOwVAADCLMh7hM6Q0Ov1riHR3d2t1Wq5GbwajaalpSV0Ww2Zp59+urm5+eWXX7ZYLFqt9ujRo4SQ8vJytzKxWFxVVaXX6xUKBcMwfX19gTSnlOr1eoPBIBQK/U6f4CJWKBQmJydjHiEAQFQE+T9fXyHBP4M3RlRUVNTX18/Ozn7zzTclJSUqlYqnWCaT6fX61tbW/fv3f/vtt36b018LsPi3vQEAAAidVT0sw4VEU1PTFy2LqgAADIdJREFU/v37uRXPwbwxSCwWb9++vaenp6urq76+3lfZjz/++OqrrxJCUlNTd+/ePTw8HEhzgUAgl8tlMpk8AM4yfBwEAIiWIIPQV0hwg3kJIc7BvLHpmWeeOX369IULF2pqanzVqFSqU6dOffXVV5TSjz76KEFGKgIAJJogg9BXSHCDeSmlroN5Y9ATTzxx+fLliooKhULhq0YsFnd3dx84cCAzM7Orq+vkyZOR3CEAAERG8PcIuZC4efOmWq3u6Ojg1o8cOVJXV5eTk1NaWtrZ2Rm6fYZYcnLyxo0bPb8Xdbtd99hjj12+fDmC+wIAgEgL/taU15AIZDBvRkaG5PNO0cQ/g7504Ow3viVPFbkt2my2oaGhycnJysrKCOwBAABiWRSe0fjPvx7avLEwUlf790ceecRt6ezZsy+88MKJEyfEYnGktgEAADEqCkEokUh4ntWMgJ07d+7cuTOKGwAAgNiBs0YBACChIQgBACChIQgBACChIQgBACChIQgBACChBR+EWq22qKgoPT1969atY2Nj3GL8ziMEAIDEFOSfT0xMTNTX158/f37z5s1tbW0NDQ39/f3k7jzCTz755JVXXjl27FhLS4tUKv3444+vXbuWlJRkt9sJIW4HTFutVpFIxB3VHSCr1eo8AoZhmNj5c8CFhYVobwEAAH4bJrgZQL29vadPn+aO35yfn9+wYQOXAYWFhVqtVq1Wj4yMaDSa0dFRk8l0//3379mzJzs7e3Z2tr29/fXXX3dt1dLSUlJSsm3bNucKl3MSicTX1d9597+m7854SpWn/cehv/Bs1Ww2BzIa0GQyiUQiv1MgjEYjN0HQV8E999xTW1u7uLioUCj8XnR+fj4jI4PbHk9PAAAInyCD0MnhcOzbt08gELS1tRFCZDLZ/Py8VCo1mUzZ2dkrKyu+Fp3UanVubm5XV5dzxWAwUEplMpmvi26tePzqteuEEMIwmfeuGb/yL54drqysiEQiqVTK/0Z0Op1UKuVJX87S0pJMJvP7GfS3BqFMJvNbDAAA4bCqk2U+//zzgwcPVlZWNjU1cSte5xHyDylMSUlpaWlRKpXOFZFIRCmVy+W+risQ/v+2GYHA9We9FAsEIpEoJSXF79uRSqXJycn8NSzLyuVyv0HocDjS0tL8ZpvNZlMqlfgsCAAQRUEGIaW0sbGxv7+/q6uroKDAuc7NI8zPz3edR+h1MVSsJkNoGwIAQEIJ8qnRgYGB7u7unp4elUql1+v1ej237nUeYRiHFFJ2ZX4mlA0BACDBBBmEvb29o6OjSqVSfhe3fuTIkStXruTk5Fy7du3QoUM8i6GxqvubAAAAq35YZvVKS0vb29tLS0udK3q9nv8eYVrW3SduKCV2C/9bWF5eDuQe4e3btwO5R7i4uBjIPcKFhYVA7hHOzc1lZmbiHiEAQBTF5ckyRqMx2lsAAIDfibgMQgAAgFCJvyC0Wq2sg3VdcT6qAwAA8FvFXxBOTEyw1CUIBUl9fX3R2w4AAMS3+AtCd2LpqVMd0d4EAADEq/gPQtbxj6/7o70JAACIV/EfhJRlmVUdFAcAAIlsVUHocDjUarXritd5hBhSCAAAMSv4IGxtbS0rKxsdHXVd5OYRzszM3HfffceOHeNZBAAAiAXBf6lYXFycl5dXXV3tutjd3a3VaiUSyb59+zQaTUtLi69FJ7vd3tfX9+CDDzpXLBYLpdTX6S1Wq9VthVJqsVh87dNisbAs6/f0FovFIhAI/M4Htlqt3PYCKWNZNpAybh6hQBD/X1MDAMSh4IOwvLzcc3Fqaio3N5cQwn0E5Fl0MpvNH374YX19vXPFaDRSSn0Fg8lk8lw0GHzOoDAajdxcJ/63w13Uc0qU54UYhrHZbH7LApkG7BzzG8iMQwAACIcQP2YSxDxCmUx24sSJjIwM54pYLOY5a/TmzZtuKwzDuP64Gy6Q/J41yjBMIGeNchvzG1osywZy1qjdbucG8/KXAQBA+IT46zhu9CAhxHMeodti0D777LNVdgAAAHAKcRBGYB7h3Nyc24r5zrLdbl9lWwAASEwhDsIIzCO80HvxV/+mrMWw7PbwKgAAQIBWe4/Q7SGU9PT0c+fOudV4XQza8vKK+5JU8ZfGQ2e13aG6BAAAJI44e2R/bm5uZtr9uVPCCPz8mQIAAIAPcRaEs7OzXtctFgvPX1AAAAD4EmdB6J3d9o/er1/Y9+do7wMAAOLP7yIIWTsRMH//3/MLCwvR3goAAMSZ30UQEkIomZ+55Xn6GgAAAL84C8J3333X+wsOG0kSX7p0KbLbAQCAuBdnQfi30//t/QVKCCHT09OR3AwAAPwORCIIQzWP0GazUZ55DozgpYONtbW1er0+6EsAAECiiUQQhmQe4eTk5P1/fIB/hARrt575n7P3/XG9NCV1ZGQkuAsBAEBCYfzOJ1q9wsJCrVarVqtHRkY0Go3bcWibNm06ePBgbW0tIYRl2TNnzmRlZSkUCoPBMDQ0JJVKBwYGvu4fmJi8RQkhDEMoQ5xDAyklhBLCEMItUsIICOsgQpFQKExLTsr+wxq90cjabfnr11dWVl69erWmpmb9+vUXL17U6/W5ubnj4+NlZWVarXbbtm3r1q2TSCQTExMrKytFRUXEZW6Gc3qGTqeTyWRCodBt3flrZBiGYZilpaXl5eWVlZXi4mLXJlyZxWK5evXqc889t7S0lJmZKRQKhUIh5hECAERFJIJQJpPNz89LpVKTyZSdnb2y8qsz0vLz8wkh3HMuFovl3x59dOOGDUqlcmpqanR8XCIWT/88a7dYKCO4GzWEkF/yh/xyc/CXdYYwhFBCnYtUyDAOlkokIrudvSdTuXh7+aGSTWq1+tPP/s4wTFpa+s8/z5SWbB64dGnro4+qVCqBQHB9eNhmtW/a9CdngLkym80ikSgpKYl7lfvtMQzDzeB1hpnZbL5x44bJbC4tKSF3I9D5Izqd7vsrQxe/6jUYDEqlUigUKhQKzCMEAIiKEM8j9Ip/HqFcLm9vb8/KyuL+efPGjXPnzjkcjh07dvC3PXPmjFKpfPzxx/nLPvjgg7y8vC1btvCXvf/++w8//HBpaSl/2fHjx6uqqgoLC/nLmpub33rrrZycHP6yw4cPv/TSS5mZmfxlAAAQPpH4RJifn//pp5/m5+ePj49v3759bGzM9dUnn3xSp9O5Ds69desWpdRvivz0009isXjNmjX8ZT/88INcLr/33nv5y8bGxrKysngG/HKuX7++bt06hULBXzY0NPTAAw+kpqbylw0ODm7cuFEikbzxxhtlZWX8xQAAEA6RCMIDBw5IJJLm5ubGxka73f7222+7vjo/Pz84OBjuPcS4zZs3+41qAAAIh0gEoU6nq6ur+/7770tLSzs7O9PS0sJ9RQAAgABFIggBAABiFh7ZBwCAhIYgBACAhIYgBACAhBajQbhlyxbmrr1793oWBHh+KX8fh8OhVqv9tvUs82yr1WqLiorS09O3bt3q/PsQz25ey/y+WQAACJ9YDEJK6cjIyK1bt+7cuXPnzh2vo5cCOb+Uv09ra2tZWZnbeW+ebT3LPNtOTEzU19efPHlyZmZmx44dDQ0NXrt5LQvkzQIAQBjR2DMzMyOTyR566CGZTKbRaGZnZz1rCgoKhoeHKaXDw8MFBQVB9Pniiy/Onj3r9hvwbOtZ5tn2yy+/3LNnD/fq3NxcZmam125eywJ5swAAED6xGISDg4Pl5eWDg4OLi4u7du169tlnPWtSU1ONRiOl1Gg0yuXyoPu4BaGvtq5lPG3tdvvevXtffPFF/m6uZYFsEgAAwidWgtB5eqfb+vT0tFKp9KxPSUkxmUyUUoPBkJKS4re/rz5uV/TV1tdHZ9e2Fy5cKCkpee2112w2G083zzK/mwQAgPCJxKHbgXAdH/jdd9+ZzWbu7E2xWCyRSDzrVSrV5ORkfn7+1NTU2rVrvfYMpE9I2lJKGxsb+/v7u7q6CgoKfHXzWhbEJgEAIIRi8WEZg8FQU1MzPDxstVqPHj361FNPedZUV1d3dHRQSjs6OjQaTdB9QtJ2YGCgu7u7p6dHpVLp9Xq9Xu+1m9eyIDYJAAChFNXPo96xLNvW1paXl5eVlbVr167l5WXPmtu3b1dVVa1du7a6ulqn0wXdx+034Kuta5ln26amJq+/VbduXssC2SQAAIQPzhoFAICEFotfjQIAAEQMghAAABIaghAAABIaghAAABIaghAAABIaghAAABIaghAAABIaghAAABIaghAAABIaghAAABIaghAAABLa/wG2jEm26H1ojAAAAABJRU5ErkJggg==" }, - "execution_count": 18, + "execution_count": 65, "metadata": {}, "output_type": "execute_result" } @@ -691,24 +668,16 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 66, "metadata": {}, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "WARNING: Base.I is deprecated, run `using LinearAlgebra` to load linear algebra functionality.\n", - " likely near /Users/juhyun-kim/.julia/packages/IJulia/GIANC/src/kernel.jl:41\n" - ] - }, { "data": { "text/plain": [ "(:Y, :X, :V)" ] }, - "execution_count": 21, + "execution_count": 66, "metadata": {}, "output_type": "execute_result" } @@ -723,7 +692,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 67, "metadata": {}, "outputs": [ { @@ -732,7 +701,7 @@ "VarianceComponentVariate{Float64,2,Array{Float64,2},Array{Float64,2},Array{Float64,2}}([-1.81573 -0.94615 … -1.02853 -0.394049; -1.2444 0.10966 … 1.09065 0.0256616; … ; 0.886626 0.487408 … -0.636874 -0.439825; -1.24394 0.213697 … 0.299931 0.392809], Array{Float64}(6670,0), ([1.00547 0.00656224 … -0.000121918 -0.00555374; 0.00656224 0.99614 … 0.00176999 0.00683183; … ; -0.000121918 0.00176999 … 0.998578 0.000962983; -0.00555374 0.00683183 … 0.000962983 0.999599], [1.0 0.0 … 0.0 0.0; 0.0 1.0 … 0.0 0.0; … ; 0.0 0.0 … 1.0 0.0; 0.0 0.0 … 0.0 1.0]))" ] }, - "execution_count": 22, + "execution_count": 67, "metadata": {}, "output_type": "execute_result" } @@ -750,14 +719,14 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 68, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - " 53.245058 seconds (29 allocations: 1021.427 MiB, 0.24% gc time)\n" + " 51.732018 seconds (29 allocations: 1021.427 MiB, 0.98% gc time)\n" ] }, { @@ -766,7 +735,7 @@ "(:Yrot, :Xrot, :eigval, :eigvec, :logdetV2)" ] }, - "execution_count": 24, + "execution_count": 68, "metadata": {}, "output_type": "execute_result" } @@ -795,7 +764,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 83, "metadata": {}, "outputs": [ { @@ -808,16 +777,15 @@ "Base & & Module \\\\\n", "Core & & Module \\\\\n", "Main & & Module \\\\\n", - "Plots & 22.513 MiB & Module \\\\\n", - "PyPlot & 790.274 KiB & Module \\\\\n", + "Plots & 22.448 MiB & Module \\\\\n", "Y & 677.461 KiB & 6670×13 Array\\{Float64,2\\} \\\\\n", "cg10k & 1022.983 MiB & 6670×630860 SnpArray \\\\\n", - "cg10k\\_trait & 1.499 MiB & 6670×15 DataFrame \\\\\n", + "cg10k\\_trait & 1.386 MiB & 6670×15 DataFrame \\\\\n", "cg10kdata & 679.508 MiB & VarianceComponentVariate\\{Float64,2,Array\\{Float64,2\\},Array\\{Float64,2\\},Array\\{Float64,2\\}\\} \\\\\n", "cg10kdata\\_rotated & 340.136 MiB & TwoVarCompVariateRotate\\{Float64,Array\\{Float64,2\\},Array\\{Float64,2\\}\\} \\\\\n", "maf\\_cg10k & 4.813 MiB & 630860-element Array\\{Float64,1\\} \\\\\n", "missings\\_by\\_snp & 4.813 MiB & 1×630860 Array\\{Int64,2\\} \\\\\n", - "mp & 28.748 MiB & 6670×630860 SparseArrays.SparseMatrixCSC\\{Bool,Int32\\} \\\\\n", + "mp & 28.748 MiB & 6670×630860 SparseMatrixCSC\\{Bool,Int32\\} \\\\\n", "people & 8 bytes & Int64 \\\\\n", "snps & 8 bytes & Int64 \\\\\n", "startupfile & 57 bytes & String \\\\\n", @@ -830,16 +798,15 @@ "| Base | | Module |\n", "| Core | | Module |\n", "| Main | | Module |\n", - "| Plots | 22.513 MiB | Module |\n", - "| PyPlot | 790.274 KiB | Module |\n", + "| Plots | 22.448 MiB | Module |\n", "| Y | 677.461 KiB | 6670×13 Array{Float64,2} |\n", "| cg10k | 1022.983 MiB | 6670×630860 SnpArray |\n", - "| cg10k_trait | 1.499 MiB | 6670×15 DataFrame |\n", + "| cg10k_trait | 1.386 MiB | 6670×15 DataFrame |\n", "| cg10kdata | 679.508 MiB | VarianceComponentVariate{Float64,2,Array{Float64,2},Array{Float64,2},Array{Float64,2}} |\n", "| cg10kdata_rotated | 340.136 MiB | TwoVarCompVariateRotate{Float64,Array{Float64,2},Array{Float64,2}} |\n", "| maf_cg10k | 4.813 MiB | 630860-element Array{Float64,1} |\n", "| missings_by_snp | 4.813 MiB | 1×630860 Array{Int64,2} |\n", - "| mp | 28.748 MiB | 6670×630860 SparseArrays.SparseMatrixCSC{Bool,Int32} |\n", + "| mp | 28.748 MiB | 6670×630860 SparseMatrixCSC{Bool,Int32} |\n", "| people | 8 bytes | Int64 |\n", "| snps | 8 bytes | Int64 |\n", "| startupfile | 57 bytes | String |\n", @@ -851,76 +818,113 @@ "Base Module \n", "Core Module \n", "Main Module \n", - "Plots 22.513 MiB Module \n", - "PyPlot 790.274 KiB Module \n", + "Plots 22.448 MiB Module \n", "Y 677.461 KiB 6670×13 Array{Float64,2} \n", "cg10k 1022.983 MiB 6670×630860 SnpArray \n", - "cg10k_trait 1.499 MiB 6670×15 DataFrame \n", + "cg10k_trait 1.386 MiB 6670×15 DataFrame \n", "cg10kdata 679.508 MiB VarianceComponentVariate{Float64,2,Array{Float64,2},Array{Float64,2},Array{Float64,2}}\n", "cg10kdata_rotated 340.136 MiB TwoVarCompVariateRotate{Float64,Array{Float64,2},Array{Float64,2}} \n", "maf_cg10k 4.813 MiB 630860-element Array{Float64,1} \n", "missings_by_snp 4.813 MiB 1×630860 Array{Int64,2} \n", - "mp 28.748 MiB 6670×630860 SparseArrays.SparseMatrixCSC{Bool,Int32} \n", + "mp 28.748 MiB 6670×630860 SparseMatrixCSC{Bool,Int32} \n", "people 8 bytes Int64 \n", "snps 8 bytes Int64 \n", "startupfile 57 bytes String \n", "Φgrm 339.423 MiB 6670×6670 Array{Float64,2} " ] }, - "execution_count": 29, + "execution_count": 83, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "using JLD\n", - "@save \"cg10k.jld\"\n", + "#pkg\"add FileIO JLD2\"\n", + "\n", + "using JLD2, FileIO\n", + "save(\"cg10k.jld2\")\n", "varinfo()" ] }, { "cell_type": "code", - "execution_count": 45, + "execution_count": 84, "metadata": {}, "outputs": [ { - "ename": "ErrorException", - "evalue": "unrecognized HDF5 datatype class 4", - "output_type": "error", - "traceback": [ - "unrecognized HDF5 datatype class 4", - "", - "Stacktrace:", - " [1] error(::String, ::Int32) at ./error.jl:42", - " [2] jldatatype(::JLD.JldFile, ::HDF5.HDF5Datatype) at /Users/juhyun-kim/.julia/packages/JLD/1BoSz/src/jld_types.jl:709", - " [3] read_array(::JLD.JldDataset, ::HDF5.HDF5Datatype, ::Int64, ::Int64, ::Tuple{Int64}) at /Users/juhyun-kim/.julia/packages/JLD/1BoSz/src/JLD.jl:411", - " [4] read_array(::JLD.JldDataset, ::HDF5.HDF5Datatype, ::Int64, ::Int64) at /Users/juhyun-kim/.julia/packages/JLD/1BoSz/src/JLD.jl:408", - " [5] read(::JLD.JldDataset) at /Users/juhyun-kim/.julia/packages/JLD/1BoSz/src/JLD.jl:372", - " [6] read_ref(::JLD.JldFile, ::HDF5.HDF5ReferenceObj) at /Users/juhyun-kim/.julia/packages/JLD/1BoSz/src/JLD.jl:502", - " [7] jlconvert(::Type{SparseArrays.SparseMatrixCSC{Bool,Int32}}, ::JLD.JldFile, ::Ptr{UInt8}) at /Users/juhyun-kim/.julia/packages/JLD/1BoSz/src/jld_types.jl:434", - " [8] read_scalar(::JLD.JldDataset, ::HDF5.HDF5Datatype, ::Type) at /Users/juhyun-kim/.julia/packages/JLD/1BoSz/src/JLD.jl:398", - " [9] read(::JLD.JldDataset) at /Users/juhyun-kim/.julia/packages/JLD/1BoSz/src/JLD.jl:370", - " [10] read(::JLD.JldFile, ::String) at /Users/juhyun-kim/.julia/packages/JLD/1BoSz/src/JLD.jl:346", - " [11] #39 at ./none:0 [inlined]", - " [12] iterate at ./generator.jl:47 [inlined]", - " [13] collect_to!(::Array{Tuple{String,Any},1}, ::Base.Generator{Array{String,1},getfield(JLD, Symbol(\"##39#41\")){JLD.JldFile}}, ::Int64, ::Int64) at ./array.jl:656", - " [14] collect_to!(::Array{Tuple{String,AbstractArray{T,2} where T},1}, ::Base.Generator{Array{String,1},getfield(JLD, Symbol(\"##39#41\")){JLD.JldFile}}, ::Int64, ::Int64) at ./array.jl:667 (repeats 2 times)", - " [15] collect_to_with_first!(::Array{Tuple{String,Array{Float64,2}},1}, ::Tuple{String,Array{Float64,2}}, ::Base.Generator{Array{String,1},getfield(JLD, Symbol(\"##39#41\")){JLD.JldFile}}, ::Int64) at ./array.jl:643", - " [16] collect(::Base.Generator{Array{String,1},getfield(JLD, Symbol(\"##39#41\")){JLD.JldFile}}) at ./array.jl:624", - " [17] (::getfield(JLD, Symbol(\"##38#40\")))(::JLD.JldFile) at /Users/juhyun-kim/.julia/packages/JLD/1BoSz/src/JLD.jl:1234", - " [18] #jldopen#14(::Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}}, ::Function, ::getfield(JLD, Symbol(\"##38#40\")), ::String, ::Vararg{String,N} where N) at /Users/juhyun-kim/.julia/packages/JLD/1BoSz/src/JLD.jl:246", - " [19] jldopen at /Users/juhyun-kim/.julia/packages/JLD/1BoSz/src/JLD.jl:244 [inlined]", - " [20] load at /Users/juhyun-kim/.julia/packages/JLD/1BoSz/src/JLD.jl:1233 [inlined]", - " [21] #load#13(::Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}}, ::Function, ::String) at /Users/juhyun-kim/.julia/packages/FileIO/YJO7Z/src/loadsave.jl:118", - " [22] load(::String) at /Users/juhyun-kim/.julia/packages/FileIO/YJO7Z/src/loadsave.jl:118", - " [23] top-level scope at In[45]:1" - ] + "data": { + "text/latex": [ + "\\begin{tabular}\n", + "{l | r | l}\n", + "name & size & summary \\\\\n", + "\\hline\n", + "Base & & Module \\\\\n", + "Core & & Module \\\\\n", + "Main & & Module \\\\\n", + "Plots & 22.448 MiB & Module \\\\\n", + "Y & 677.461 KiB & 6670×13 Array\\{Float64,2\\} \\\\\n", + "cg10k & 1022.983 MiB & 6670×630860 SnpArray \\\\\n", + "cg10k\\_trait & 1.449 MiB & 6670×15 DataFrame \\\\\n", + "cg10kdata & 679.508 MiB & VarianceComponentVariate\\{Float64,2,Array\\{Float64,2\\},Array\\{Float64,2\\},Array\\{Float64,2\\}\\} \\\\\n", + "cg10kdata\\_rotated & 340.136 MiB & TwoVarCompVariateRotate\\{Float64,Array\\{Float64,2\\},Array\\{Float64,2\\}\\} \\\\\n", + "maf\\_cg10k & 4.813 MiB & 630860-element Array\\{Float64,1\\} \\\\\n", + "missings\\_by\\_snp & 4.813 MiB & 1×630860 Array\\{Int64,2\\} \\\\\n", + "mp & 28.748 MiB & 6670×630860 SparseMatrixCSC\\{Bool,Int32\\} \\\\\n", + "people & 8 bytes & Int64 \\\\\n", + "snps & 8 bytes & Int64 \\\\\n", + "startupfile & 57 bytes & String \\\\\n", + "Φgrm & 339.423 MiB & 6670×6670 Array\\{Float64,2\\} \\\\\n", + "\\end{tabular}\n" + ], + "text/markdown": [ + "| name | size | summary |\n", + "|:----------------- | ------------:|:-------------------------------------------------------------------------------------- |\n", + "| Base | | Module |\n", + "| Core | | Module |\n", + "| Main | | Module |\n", + "| Plots | 22.448 MiB | Module |\n", + "| Y | 677.461 KiB | 6670×13 Array{Float64,2} |\n", + "| cg10k | 1022.983 MiB | 6670×630860 SnpArray |\n", + "| cg10k_trait | 1.449 MiB | 6670×15 DataFrame |\n", + "| cg10kdata | 679.508 MiB | VarianceComponentVariate{Float64,2,Array{Float64,2},Array{Float64,2},Array{Float64,2}} |\n", + "| cg10kdata_rotated | 340.136 MiB | TwoVarCompVariateRotate{Float64,Array{Float64,2},Array{Float64,2}} |\n", + "| maf_cg10k | 4.813 MiB | 630860-element Array{Float64,1} |\n", + "| missings_by_snp | 4.813 MiB | 1×630860 Array{Int64,2} |\n", + "| mp | 28.748 MiB | 6670×630860 SparseMatrixCSC{Bool,Int32} |\n", + "| people | 8 bytes | Int64 |\n", + "| snps | 8 bytes | Int64 |\n", + "| startupfile | 57 bytes | String |\n", + "| Φgrm | 339.423 MiB | 6670×6670 Array{Float64,2} |\n" + ], + "text/plain": [ + "name size summary \n", + "––––––––––––––––– –––––––––––– ––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––\n", + "Base Module \n", + "Core Module \n", + "Main Module \n", + "Plots 22.448 MiB Module \n", + "Y 677.461 KiB 6670×13 Array{Float64,2} \n", + "cg10k 1022.983 MiB 6670×630860 SnpArray \n", + "cg10k_trait 1.449 MiB 6670×15 DataFrame \n", + "cg10kdata 679.508 MiB VarianceComponentVariate{Float64,2,Array{Float64,2},Array{Float64,2},Array{Float64,2}}\n", + "cg10kdata_rotated 340.136 MiB TwoVarCompVariateRotate{Float64,Array{Float64,2},Array{Float64,2}} \n", + "maf_cg10k 4.813 MiB 630860-element Array{Float64,1} \n", + "missings_by_snp 4.813 MiB 1×630860 Array{Int64,2} \n", + "mp 28.748 MiB 6670×630860 SparseMatrixCSC{Bool,Int32} \n", + "people 8 bytes Int64 \n", + "snps 8 bytes Int64 \n", + "startupfile 57 bytes String \n", + "Φgrm 339.423 MiB 6670×6670 Array{Float64,2} " + ] + }, + "execution_count": 84, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ - "#using SnpArrays, JLD, DataFrames, VarianceComponentModels, Plots\n", + "#using SnpArrays, JLD2, DataFrames, VarianceComponentModels, Plots\n", "#pyplot()\n", - "#@load \"cg10k.jld\"\n", "load(\"cg10k.jld\")\n", "varinfo()" ] @@ -941,7 +945,7 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": 85, "metadata": {}, "outputs": [ { @@ -949,6 +953,13 @@ "output_type": "stream", "text": [ "Trait1\n", + "\n", + "******************************************************************************\n", + "This program contains Ipopt, a library for large-scale nonlinear optimization.\n", + " Ipopt is released as open source code under the Eclipse Public License (EPL).\n", + " For more information visit http://projects.coin-or.org/Ipopt\n", + "******************************************************************************\n", + "\n", "(σ2a[trait], σ2e[trait]) = (0.25947604776534905, 0.7375938878381869)\n", "Trait2\n", "(σ2a[trait], σ2e[trait]) = (0.18588150448759697, 0.8137880281704691)\n", @@ -974,7 +985,7 @@ "(σ2a[trait], σ2e[trait]) = (0.08224956591863997, 0.9166483378364418)\n", "Trait13\n", "(σ2a[trait], σ2e[trait]) = (0.05687679106196685, 0.9424058676223442)\n", - " 0.152341 seconds (258.64 k allocations: 16.272 MiB, 8.63% gc time)\n" + " 6.559467 seconds (12.64 M allocations: 611.362 MiB, 11.95% gc time)\n" ] } ], @@ -1008,7 +1019,7 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": 86, "metadata": {}, "outputs": [ { @@ -1019,7 +1030,7 @@ " 0.0799434 0.08689 0.0739664 0.0887138 0.0944375 0.0953072" ] }, - "execution_count": 36, + "execution_count": 86, "metadata": {}, "output_type": "execute_result" } @@ -1040,7 +1051,7 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": 87, "metadata": {}, "outputs": [ { @@ -1183,15 +1194,15 @@ "(Σa[i, j], Σe[i, j]) = ([0.193914 -0.00259164; -0.00259164 0.0821237], [0.805992 0.0333257; 0.0333257 0.916773])\n", "Trait8Trait13\n", "(Σa[i, j], Σe[i, j]) = ([0.193913 0.00330925; 0.00330925 0.0569126], [0.805993 0.0386552; 0.0386552 0.94237])\n", - "Trait9Trait10\n" + "Trait9Trait10\n", + "(Σa[i, j], Σe[i, j]) = ([0.246837 -0.00323396; -0.00323396 0.0989732], [0.751834 0.074992; 0.074992 0.901043])\n", + "Trait9Trait11\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "(Σa[i, j], Σe[i, j]) = ([0.246837 -0.00323396; -0.00323396 0.0989732], [0.751834 0.074992; 0.074992 0.901043])\n", - "Trait9Trait11\n", "(Σa[i, j], Σe[i, j]) = ([0.247395 0.0305136; 0.0305136 0.164575], [0.751288 0.153633; 0.153633 0.834308])\n", "Trait9Trait12\n", "(Σa[i, j], Σe[i, j]) = ([0.249928 0.0844216; 0.0844216 0.0880992], [0.748829 0.108018; 0.108018 0.910884])\n", @@ -1209,7 +1220,7 @@ "(Σa[i, j], Σe[i, j]) = ([0.164571 -0.00169939; -0.00169939 0.0575331], [0.834326 0.200725; 0.200725 0.941755])\n", "Trait12Trait13\n", "(Σa[i, j], Σe[i, j]) = ([0.08398 0.0685417; 0.0685417 0.0559416], [0.91494 0.573206; 0.573206 0.943343])\n", - " 3.627466 seconds (2.89 M allocations: 223.634 MiB, 3.17% gc time)\n" + " 4.258555 seconds (4.55 M allocations: 306.219 MiB, 3.55% gc time)\n" ] } ], @@ -1248,7 +1259,7 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": 88, "metadata": {}, "outputs": [ { @@ -1302,11 +1313,11 @@ "Number of equality constraint Jacobian evaluations = 0\n", "Number of inequality constraint Jacobian evaluations = 0\n", "Number of Lagrangian Hessian evaluations = 49\n", - "Total CPU secs in IPOPT (w/o function evaluations) = 0.019\n", - "Total CPU secs in NLP function evaluations = 0.055\n", + "Total CPU secs in IPOPT (w/o function evaluations) = 0.022\n", + "Total CPU secs in NLP function evaluations = 0.059\n", "\n", "EXIT: Optimal Solution Found.\n", - " 0.093736 seconds (46.45 k allocations: 5.051 MiB)\n" + " 0.098447 seconds (46.49 k allocations: 5.053 MiB)\n" ] }, { @@ -1315,7 +1326,7 @@ "VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}(Array{Float64}(0,3), ([0.28137 0.280153 0.232512; 0.280153 0.284916 0.220363; 0.232512 0.220363 0.212921], [0.717042 0.661484 0.674207; 0.661484 0.714964 0.581648; 0.674207 0.581648 0.784373]), Array{Float64}(0,0), Char[], Float64[], -Inf, Inf)" ] }, - "execution_count": 39, + "execution_count": 88, "metadata": {}, "output_type": "execute_result" } @@ -1341,7 +1352,7 @@ }, { "cell_type": "code", - "execution_count": 40, + "execution_count": 89, "metadata": {}, "outputs": [ { @@ -1355,7 +1366,7 @@ " 0 -1.470095e+04\n", " 1 -1.470095e+04\n", "\n", - " 0.534608 seconds (1.06 M allocations: 52.739 MiB, 5.97% gc time)\n" + " 0.548238 seconds (1.04 M allocations: 51.897 MiB, 5.51% gc time)\n" ] }, { @@ -1364,7 +1375,7 @@ "VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}(Array{Float64}(0,3), ([0.28137 0.280153 0.232512; 0.280153 0.284916 0.220363; 0.232512 0.220363 0.212921], [0.717042 0.661484 0.674207; 0.661484 0.714964 0.581648; 0.674207 0.581648 0.784373]), Array{Float64}(0,0), Char[], Float64[], -Inf, Inf)" ] }, - "execution_count": 40, + "execution_count": 89, "metadata": {}, "output_type": "execute_result" } @@ -1384,7 +1395,7 @@ }, { "cell_type": "code", - "execution_count": 41, + "execution_count": 90, "metadata": {}, "outputs": [ { @@ -1517,7 +1528,7 @@ " 1100 -1.470100e+04\n", " 1110 -1.470100e+04\n", "\n", - " 0.746208 seconds (135.30 k allocations: 13.562 MiB, 2.06% gc time)\n" + " 0.794444 seconds (135.15 k allocations: 13.553 MiB, 1.62% gc time)\n" ] }, { @@ -1526,7 +1537,7 @@ "VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}(Array{Float64}(0,3), ([0.281394 0.280169 0.232564; 0.280169 0.285001 0.220511; 0.232564 0.220511 0.213147], [0.717018 0.661467 0.674155; 0.661467 0.714877 0.581502; 0.674155 0.581502 0.784149]), Array{Float64}(0,0), Char[], Float64[], -Inf, Inf)" ] }, - "execution_count": 41, + "execution_count": 90, "metadata": {}, "output_type": "execute_result" } @@ -1547,7 +1558,7 @@ }, { "cell_type": "code", - "execution_count": 42, + "execution_count": 91, "metadata": {}, "outputs": [ { @@ -1558,7 +1569,7 @@ " 0.0777056 0.0772501 0.0840114" ] }, - "execution_count": 42, + "execution_count": 91, "metadata": {}, "output_type": "execute_result" } @@ -1579,7 +1590,7 @@ }, { "cell_type": "code", - "execution_count": 43, + "execution_count": 92, "metadata": {}, "outputs": [ { @@ -1632,7 +1643,7 @@ " [13] #mle_fs!#27(::Int64, ::Symbol, ::Symbol, ::Bool, ::Function, ::VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}, ::TwoVarCompVariateRotate{Float64,Array{Float64,2},Array{Float64,2}}) at /Users/juhyun-kim/.julia/packages/VarianceComponentModels/Mh6LK/src/two_variance_component.jl:942", " [14] (::getfield(VarianceComponentModels, Symbol(\"#kw##mle_fs!\")))(::NamedTuple{(:solver, :verbose),Tuple{Symbol,Bool}}, ::typeof(mle_fs!), ::VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}, ::TwoVarCompVariateRotate{Float64,Array{Float64,2},Array{Float64,2}}) at ./none:0", " [15] top-level scope at util.jl:156", - " [16] top-level scope at In[43]:3" + " [16] top-level scope at In[92]:3" ] } ], @@ -1652,7 +1663,7 @@ }, { "cell_type": "code", - "execution_count": 44, + "execution_count": 93, "metadata": {}, "outputs": [ { @@ -1782,7 +1793,7 @@ " 1070 -4.435625e+04\n", " 1080 -4.435625e+04\n", "\n", - " 3.378533 seconds (131.51 k allocations: 65.569 MiB, 0.70% gc time)\n" + " 3.423491 seconds (131.35 k allocations: 65.559 MiB, 0.86% gc time)\n" ] }, { @@ -1791,7 +1802,7 @@ "(-44356.24416253091, VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}(Array{Float64}(0,13), ([0.272112 0.190023 … -0.128464 -0.0976418; 0.190023 0.216814 … -0.0687833 -0.04341; … ; -0.128464 -0.0687833 … 0.116994 0.0900933; -0.0976418 -0.04341 … 0.0900933 0.105876], [0.725183 0.570497 … -0.0589748 -0.125486; 0.570497 0.783023 … 0.0235685 0.0464638; … ; -0.0589748 0.0235685 … 0.882056 0.551829; -0.125486 0.0464638 … 0.551829 0.893642]), Array{Float64}(0,0), Char[], Float64[], -Inf, Inf), ([0.0111615 0.0131072 … 0.0128913 0.0127584; 0.0131071 0.0151656 … 0.017153 0.0171365; … ; 0.0128911 0.017153 … 0.0174174 0.018212; 0.0127593 0.0171355 … 0.0182119 0.0188], [0.0112229 0.0133031 … 0.013004 0.0127777; 0.0133033 0.015806 … 0.0178516 0.0177826; … ; 0.0130042 0.0178517 … 0.0179556 0.0187634; 0.0127776 0.0177822 … 0.0187633 0.0193475]), [0.00012458 7.242e-5 … -3.70019e-7 -1.40553e-5; 7.23929e-5 0.000171795 … -2.04581e-5 -3.19854e-6; … ; -3.8356e-7 -2.04665e-5 … 0.000352066 -1.46068e-5; -1.40476e-5 -3.19013e-6 … -1.46022e-5 0.000374325], Array{Float64}(0,13), Array{Float64}(0,0))" ] }, - "execution_count": 44, + "execution_count": 93, "metadata": {}, "output_type": "execute_result" } From 8e1771247a4ec9967d0233faa650f30db1f67946 Mon Sep 17 00:00:00 2001 From: Juhyun Kim Date: Tue, 29 Jan 2019 18:20:51 -0800 Subject: [PATCH 47/56] remove .ipynb files using julia v0.6 --- docs/heritability.ipynb | 2020 --------------------------------------- docs/mle_reml.ipynb | 1408 --------------------------- 2 files changed, 3428 deletions(-) delete mode 100644 docs/heritability.ipynb delete mode 100644 docs/mle_reml.ipynb diff --git a/docs/heritability.ipynb b/docs/heritability.ipynb deleted file mode 100644 index bc77f75..0000000 --- a/docs/heritability.ipynb +++ /dev/null @@ -1,2020 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, - "source": [ - "# Heritability Analysis" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, - "source": [ - "As an application of the variance component model, this note demonstrates the workflow for heritability analysis in genetics, using a sample data set `cg10k` with **6,670** individuals and **630,860** SNPs. Person IDs and phenotype names are masked for privacy. `cg10k.bed`, `cg10k.bim`, and `cg10k.fam` is a set of Plink files in binary format. `cg10k_traits.txt` contains 13 phenotypes of the 6,670 individuals." - ] - }, - { - "cell_type": "code", - "execution_count": 31, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "cg10k.bed\n", - "cg10k.bim\n", - "cg10k.fam\n", - "cg10k_traits.txt\n" - ] - } - ], - "source": [ - ";ls cg10k.bed cg10k.bim cg10k.fam cg10k_traits.txt" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, - "source": [ - "Machine information:" - ] - }, - { - "cell_type": "code", - "execution_count": 32, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Julia Version 0.6.0\n", - "Commit 903644385b (2017-06-19 13:05 UTC)\n", - "Platform Info:\n", - " OS: macOS (x86_64-apple-darwin13.4.0)\n", - " CPU: Intel(R) Core(TM) i7-4790K CPU @ 4.00GHz\n", - " WORD_SIZE: 64\n", - " BLAS: libopenblas (USE64BITINT DYNAMIC_ARCH NO_AFFINITY Haswell)\n", - " LAPACK: libopenblas64_\n", - " LIBM: libopenlibm\n", - " LLVM: libLLVM-3.9.1 (ORCJIT, haswell)\n" - ] - } - ], - "source": [ - "versioninfo()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, - "source": [ - "## Read in binary SNP data" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, - "source": [ - "We will use the [`SnpArrays.jl`](https://github.com/OpenMendel/SnpArrays.jl) package to read in binary SNP data and compute the empirical kinship matrix. Issue \n", - "```julia\n", - "Pkg.clone(\"https://github.com/OpenMendel/SnpArrays.jl.git\")\n", - "```\n", - "within `Julia` to install the `SnpArrays` package." - ] - }, - { - "cell_type": "code", - "execution_count": 33, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, - "outputs": [], - "source": [ - "using SnpArrays" - ] - }, - { - "cell_type": "code", - "execution_count": 34, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " 22.902730 seconds (51.62 k allocations: 1005.845 MiB, 0.11% gc time)\n" - ] - }, - { - "data": { - "text/plain": [ - "6670×630860 SnpArrays.SnpArray{2}:\n", - " (false, true) (false, true) … (true, true) (true, true) \n", - " (true, true) (true, true) (false, true) (true, false)\n", - " (true, true) (true, true) (true, true) (true, true) \n", - " (true, true) (true, true) (false, true) (true, true) \n", - " (true, true) (true, true) (true, true) (false, true)\n", - " (false, true) (false, true) … (true, true) (true, true) \n", - " (false, false) (false, false) (true, true) (true, true) \n", - " (true, true) (true, true) (true, true) (false, true)\n", - " (true, true) (true, true) (true, true) (true, true) \n", - " (true, true) (true, true) (false, true) (true, true) \n", - " (true, true) (true, true) … (true, true) (true, true) \n", - " (false, true) (false, true) (true, true) (false, true)\n", - " (true, true) (true, true) (true, true) (false, true)\n", - " ⋮ ⋱ \n", - " (false, true) (false, true) (false, true) (false, true)\n", - " (false, true) (false, true) (false, true) (true, true) \n", - " (true, true) (true, true) … (false, true) (true, true) \n", - " (false, true) (false, true) (true, true) (false, true)\n", - " (true, true) (true, true) (false, true) (true, true) \n", - " (true, true) (true, true) (false, false) (false, true)\n", - " (true, true) (true, true) (true, true) (false, true)\n", - " (true, true) (true, true) … (true, true) (true, true) \n", - " (true, true) (true, true) (false, true) (true, true) \n", - " (true, true) (true, true) (true, true) (false, true)\n", - " (false, true) (false, true) (true, true) (true, true) \n", - " (true, true) (true, true) (true, true) (true, true) " - ] - }, - "execution_count": 34, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# read in genotype data from Plink binary file (~50 secs on my laptop)\n", - "@time cg10k = SnpArray(\"cg10k\")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, - "source": [ - "## Summary statistics of SNP data" - ] - }, - { - "cell_type": "code", - "execution_count": 35, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, - "outputs": [ - { - "data": { - "text/plain": [ - "(6670, 630860)" - ] - }, - "execution_count": 35, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "people, snps = size(cg10k)" - ] - }, - { - "cell_type": "code", - "execution_count": 36, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " 24" - ] - } - ], - "source": [ - "# summary statistics (~50 secs on my laptop)\n", - "@time maf, _, missings_by_snp, = summarize(cg10k);" - ] - }, - { - "cell_type": "code", - "execution_count": 37, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, - "outputs": [ - { - "data": { - "text/plain": [ - "([0.00841726 0.124063 … 0.364253 0.5], 0.24536516625042462)" - ] - }, - "execution_count": 37, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# 5 number summary and average MAF (minor allele frequencies)\n", - "quantile(maf, [0.0 .25 .5 .75 1.0]), mean(maf)" - ] - }, - { - "cell_type": "code", - "execution_count": 38, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, - "outputs": [ - { - "data": { - "text/html": [ - "" - ] - }, - "execution_count": 38, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# Pkg.add(\"Plots\")\n", - "# Pkg.add(\"PyPlot\")\n", - "using Plots\n", - "pyplot()\n", - "\n", - "histogram(maf, xlab = \"Minor Allele Frequency (MAF)\", label = \"MAF\")" - ] - }, - { - "cell_type": "code", - "execution_count": 39, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, - "outputs": [ - { - "data": { - "text/plain": [ - "0.0013128198764010824" - ] - }, - "execution_count": 39, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# proportion of missing genotypes\n", - "sum(missings_by_snp) / length(cg10k)" - ] - }, - { - "cell_type": "code", - "execution_count": 40, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, - "outputs": [ - { - "data": { - "text/plain": [ - "0.07228069619249913" - ] - }, - "execution_count": 40, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# proportion of rare SNPs with maf < 0.05\n", - "countnz(maf .< 0.05) / length(maf)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, - "source": [ - "## Empirical kinship matrix\n", - "\n", - "We estimate empirical kinship based on all SNPs by the genetic relation matrix (GRM). Missing genotypes are imputed on the fly by drawing according to the minor allele frequencies." - ] - }, - { - "cell_type": "code", - "execution_count": 41, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "396.943890 seconds (8.43 G allocations: 127.378 GiB, 4.38% gc time)\n" - ] - }, - { - "data": { - "text/plain": [ - "6670×6670 Array{Float64,2}:\n", - " 0.503024 0.00335505 -0.000120075 … -5.45185e-5 -0.00278072 \n", - " 0.00335505 0.498958 -0.00195952 0.000868471 0.0034285 \n", - " -0.000120075 -0.00195952 0.493828 0.000174648 -0.000381467\n", - " 0.000923828 -0.00329169 -0.00194166 -0.00223595 -0.00123508 \n", - " -8.39649e-5 -0.00353358 0.0018709 0.00222858 -0.00171176 \n", - " 0.00204208 0.000572952 0.00254025 … 0.000861385 2.99785e-5 \n", - " 0.000569323 0.0024786 -0.00185743 0.00117649 -0.00118027 \n", - " -0.000642144 0.00317992 -0.00099777 0.00354182 -0.000260645\n", - " -0.00102913 -0.00123475 -0.00061138 0.00173885 0.00177727 \n", - " -0.00139442 0.00208423 0.000124525 -0.00145156 -0.001011 \n", - " -0.00204555 0.00011055 -0.000419398 … -0.000198235 -0.00110353 \n", - " 0.000947587 0.00167346 0.00184451 -0.000690143 -0.00304087 \n", - " 0.000322759 -0.000899805 0.00303981 0.000739331 -0.00118835 \n", - " ⋮ ⋱ \n", - " 0.00298012 0.00130003 0.000998861 4.18454e-6 0.00303991 \n", - " -0.00207748 0.00274717 -0.00191741 -0.00107073 0.00368267 \n", - " 0.000545569 -0.00244439 -0.00299578 … -0.000669885 0.00221027 \n", - " -0.00423186 -0.00208514 -0.00108833 -0.000622127 -0.000567483\n", - " -0.00325644 -0.000781353 0.0030423 0.000501423 -0.00010267 \n", - " 0.00041055 -0.00200772 0.00274867 -0.00624933 -0.00521365 \n", - " 0.00210519 0.000879889 -0.00107817 -0.000797878 -0.000557352\n", - " -0.00230058 -0.000119132 0.000116817 … 0.000867087 -0.00233512 \n", - " -0.0020119 0.00230772 -0.00128837 0.00194798 -0.00048733 \n", - " -0.000944942 -0.000928073 -0.000175096 0.00126911 -0.00303766 \n", - " -5.45185e-5 0.000868471 0.000174648 0.500829 0.000469478\n", - " -0.00278072 0.0034285 -0.000381467 0.000469478 0.500627 " - ] - }, - "execution_count": 41, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# GRM using SNPs with maf > 0.01 (default) (~10 mins on my laptop)\n", - "srand(123)\n", - "@time Φgrm = grm(cg10k; method = :GRM)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, - "source": [ - "## Phenotypes" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, - "source": [ - "Read in the phenotype data and compute descriptive statistics." - ] - }, - { - "cell_type": "code", - "execution_count": 42, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, - "outputs": [ - { - "data": { - "text/html": [ - "
Trait1Trait2Trait3Trait4Trait5Trait6Trait7Trait8Trait9Trait10Trait11Trait12Trait13
1-1.81573145026234-0.946150461472831.11363077580442-2.098671211191590.7444166141117480.001391718840801310.934732480409667-1.226773154181031.1160784277875-0.44362803350290.824465656443384-1.02852542216546-0.394049201727681
2-1.244400943787290.1096599925471790.467119394241789-1.621313040975891.05667583556830.9789469794191811.000146339460470.324874271402281.162321752196962.69227069487053.082636724610471.090649547860130.0256616415357438
31.455669145023051.538669329232431.094029593765550.586655272226893-0.32796454430367-0.30337709778827-0.0334354881314741-0.464463064285437-0.3319396273436-0.486839089635991-1.10648681564373-1.42015780427231-0.687463456644413
4-0.7688092766985480.5134908855142490.244263028382142-1.317402544756911.193937743268451.173441277342881.087374266752320.5360225837322610.8027592407620680.2341594117498150.394174866891074-0.7673658924760290.0635385761884935
5-0.264415132547719-0.348240421825694-0.02390650834136060.004739158022449481.256191917121931.20388836676311.298007390426270.3101136602473110.6261598610593520.8992891298312240.549967833508120.5406878095420480.179675416046033
6-1.37617270917293-1.471919677445640.291179894254146-0.803110740704731-0.264239977442213-0.260573027836772-0.165372266287781-0.2192572941183621.04702422290318-0.09858155346164820.9473934380684480.5940148120314380.245407436348479
70.1009416296374-0.191615722103455-0.5674213215966770.378571487240382-0.246656179817904-0.6088107500538580.189081058215596-1.27077787326519-0.4524761991439650.7025628772977240.3326362189571790.00269165036261810.317117176705358
8-0.3198182763674641.357744806572830.818689545938528-1.155655316443520.634483681022590.2914619086346790.933323714954726-0.7410832896824920.647477683507572-0.9708776270779660.2208611654113040.852512250237764-0.225904624283945
9-0.2883341733420320.5660825380907520.254958336116175-0.6525783028697140.6689215592773470.9783091991705580.1228629660419381.47909263782140.06721324241734490.07959039175278270.1675324552432320.2469155794421390.539932616458363
10-1.15759732583991-0.781198583545165-0.595807759833517-1.005549802604020.7898288859333210.5710584133790440.951304176233755-0.2959629829848160.990420024797070.5613093669889830.733100030623233-1.73467772245684-1.35278484330654
110.7405691504590311.408738467554150.7346899994400880.0208322841295094-0.337440968561619-0.458304040611395-0.142582512772326-0.580392297464107-0.684684998101516-0.00785381461893456-0.712244337518008-0.313345561230878-0.345419463162219
12-0.6758924864549950.2798926138296820.267915996308248-1.041036653929850.9107417156458880.8660276185131711.074144317020050.03817510035383020.766355377018601-0.340118016143495-0.8090139585050590.548521663785885-0.0201828675962336
13-0.795410435603455-0.6999899397627380.3991295030063-0.5104762619007361.515522454168441.287430329394671.537723932509030.1339891601177021.020257368860370.499018733899186-0.36948273277931-1.10153460436318-0.598132438886619
14-0.193483122930324-0.286021160323518-0.6914942252629950.01315816787006991.523374706867821.40106380722621.531146204518960.3330664834780751.043724803810990.163206783570466-0.422883765001728-0.383527976713573-0.489221907788158
150.1512462033797182.091851089936142.03800472474384-1.124747171435311.665570243907131.625356751095761.587510704836550.6358521860437760.8425777846059790.450761870778952-1.39479033623028-0.5609841075677680.289349776549287
16-0.4646087408127120.361276947723031.2327673928287-0.8260337310863831.434752247099831.744518238188460.2110968874846382.648164251405481.025114331460960.119757316031840.0596832073448267-0.631231612661616-0.207878671782927
17-0.732977488012215-0.5262234258897790.61657871336593-0.554479743325930.9474848590251040.9368332141381730.9725168063355240.2902510138652271.012853597257230.516207422283291-0.03006891719881940.87873225245830.450254629309513
18-0.1673264596221190.1753271654872370.287467725892572-0.4026525320842460.5511815094180560.5222047432909750.4368376600946530.2995649338455790.583109520896067-0.704415820005353-0.730810367994577-1.95140580379896-0.933504665700164
191.411594857874181.787224079010170.843976395853640.481278083772991-0.0887673728508268-0.499577574268580.304195897924847-1.23884208383369-0.153475724036624-0.8704861027883290.0955473331150403-0.983708050882817-0.3563445644514
20-1.42997091652825-0.4901470450342130.272730237607695-1.610299929541530.9907878171977480.7116875326081841.1885836012715-0.3712291880756381.24703459239952-0.03891623322715160.8834957490728722.589880263210173.33539552370368
21-0.1472472881767650.123284304156520.617549051912237-0.187130771782620.2564381075866940.177949837350830.412611806463263-0.2448091245597370.09476248061364920.723017223849532-0.6839483546334360.0873751276309269-0.262209652750371
22-0.187112676773894-0.270777264595619-1.015568185516060.06028505686002330.2724197577579780.869133161879197-0.6575194614142342.32388522018189-0.9999360115250341.446718441783060.971157886040772-0.358747904241515-0.439657942096136
23-1.82434047163768-0.9334804460680671.29474003766977-1.945452211510360.335846511896540.3592016543028440.513652924365886-0.0731976966969581.571390428120051.533293713267281.820768218595282.227403018678291.50063347195857
24-2.29344084351335-2.491618423444180.40383988742336-2.364880747529481.41052548319561.422441171477921.170241662721720.844766501768551.790268754324950.648181858970515-0.0857231057403538-1.027895352926170.491288088952859
25-0.4341359328883050.7408819890346520.699576357578518-1.024055431877750.7595292239837130.9566561108952880.6332995686565890.7707339322685160.8249885117145261.842874376347691.91045942063443-0.5023172078693660.132670133448219
26-2.1920969546557-2.494656642722710.354854763893431-1.931558486357140.9419794002899380.9789171014141060.8948600972897360.4632394028318731.125371333171631.705284461919550.7177927144791230.6458880491082610.783968250169388
27-1.46602269088422-1.249216771018970.307977693653039-1.550973646609890.6189084944747980.6625081716620420.4759571739060780.4847186745977070.4015648920282490.55987973254026-0.376938143754217-0.9339826292282180.390013151672955
28-1.83317744236881-1.532687878287012.55674262685865-1.518277457838350.7894096017464550.9087477997285880.6499719229414790.6683736499316671.200583035199030.2779632560756371.25049531982753.313704450716382.22035828885342
29-0.7845466282431780.2765825795439313.01104958800057-1.119788432067580.9208238584227070.7502176898861511.26153730009639-0.4033638829224170.400667296857811-0.217597941303479-0.724669537565068-0.391945338467193-0.650023936358253
300.4644559163451351.3326356122229-1.23059563374303-0.3579759589374141.182497469771041.54315938069757-0.603390411540623.383088459584220.823740765148641-0.129951318508883-0.657979878422938-0.499534924074273-0.414476569095651
⋮⋮⋮⋮⋮⋮⋮⋮⋮⋮⋮⋮⋮⋮
" - ], - "text/plain": [ - "6670×13 DataFrames.DataFrame\n", - "│ Row │ Trait1 │ Trait2 │ Trait3 │ Trait4 │ Trait5 │\n", - "├──────┼───────────┼───────────┼────────────┼────────────┼────────────┤\n", - "│ 1 │ -1.81573 │ -0.94615 │ 1.11363 │ -2.09867 │ 0.744417 │\n", - "│ 2 │ -1.2444 │ 0.10966 │ 0.467119 │ -1.62131 │ 1.05668 │\n", - "│ 3 │ 1.45567 │ 1.53867 │ 1.09403 │ 0.586655 │ -0.327965 │\n", - "│ 4 │ -0.768809 │ 0.513491 │ 0.244263 │ -1.3174 │ 1.19394 │\n", - "│ 5 │ -0.264415 │ -0.34824 │ -0.0239065 │ 0.00473916 │ 1.25619 │\n", - "│ 6 │ -1.37617 │ -1.47192 │ 0.29118 │ -0.803111 │ -0.26424 │\n", - "│ 7 │ 0.100942 │ -0.191616 │ -0.567421 │ 0.378571 │ -0.246656 │\n", - "│ 8 │ -0.319818 │ 1.35774 │ 0.81869 │ -1.15566 │ 0.634484 │\n", - "│ 9 │ -0.288334 │ 0.566083 │ 0.254958 │ -0.652578 │ 0.668922 │\n", - "│ 10 │ -1.1576 │ -0.781199 │ -0.595808 │ -1.00555 │ 0.789829 │\n", - "│ 11 │ 0.740569 │ 1.40874 │ 0.73469 │ 0.0208323 │ -0.337441 │\n", - "⋮\n", - "│ 6659 │ -0.131005 │ 0.425378 │ -1.09015 │ -0.42093 │ 0.310943 │\n", - "│ 6660 │ -0.52427 │ 1.04173 │ 1.13749 │ -1.27262 │ -0.607382 │\n", - "│ 6661 │ 1.32516 │ 0.905899 │ 0.84261 │ 0.869143 │ -0.691461 │\n", - "│ 6662 │ -1.44368 │ -2.55708 │ -0.868193 │ 0.390457 │ -0.0331364 │\n", - "│ 6663 │ -1.8518 │ -1.25726 │ 1.81724 │ -2.17285 │ 0.948696 │\n", - "│ 6664 │ -0.810034 │ 0.0896703 │ 0.530939 │ -1.14212 │ 1.26049 │\n", - "│ 6665 │ -1.22395 │ -1.48953 │ -2.95847 │ -0.645307 │ -2.67022 │\n", - "│ 6666 │ -0.282847 │ -1.54129 │ -1.38819 │ 1.48327 │ -1.23406 │\n", - "│ 6667 │ 0.475008 │ 1.46697 │ 0.497403 │ -0.46861 │ 0.496553 │\n", - "│ 6668 │ -0.408154 │ -0.325323 │ 0.0850869 │ -0.182984 │ -0.46577 │\n", - "│ 6669 │ 0.886626 │ 0.487408 │ -0.0977307 │ 0.830857 │ -0.168597 │\n", - "│ 6670 │ -1.24394 │ 0.213697 │ 2.74965 │ -1.80285 │ 1.11401 │\n", - "\n", - "│ Row │ Trait6 │ Trait7 │ Trait8 │ Trait9 │ Trait10 │\n", - "├──────┼────────────┼────────────┼───────────┼───────────┼─────────────┤\n", - "│ 1 │ 0.00139172 │ 0.934732 │ -1.22677 │ 1.11608 │ -0.443628 │\n", - "│ 2 │ 0.978947 │ 1.00015 │ 0.324874 │ 1.16232 │ 2.69227 │\n", - "│ 3 │ -0.303377 │ -0.0334355 │ -0.464463 │ -0.33194 │ -0.486839 │\n", - "│ 4 │ 1.17344 │ 1.08737 │ 0.536023 │ 0.802759 │ 0.234159 │\n", - "│ 5 │ 1.20389 │ 1.29801 │ 0.310114 │ 0.62616 │ 0.899289 │\n", - "│ 6 │ -0.260573 │ -0.165372 │ -0.219257 │ 1.04702 │ -0.0985816 │\n", - "│ 7 │ -0.608811 │ 0.189081 │ -1.27078 │ -0.452476 │ 0.702563 │\n", - "│ 8 │ 0.291462 │ 0.933324 │ -0.741083 │ 0.647478 │ -0.970878 │\n", - "│ 9 │ 0.978309 │ 0.122863 │ 1.47909 │ 0.0672132 │ 0.0795904 │\n", - "│ 10 │ 0.571058 │ 0.951304 │ -0.295963 │ 0.99042 │ 0.561309 │\n", - "│ 11 │ -0.458304 │ -0.142583 │ -0.580392 │ -0.684685 │ -0.00785381 │\n", - "⋮\n", - "│ 6659 │ 0.735302 │ -0.0923875 │ 1.35506 │ 1.0652 │ 0.643122 │\n", - "│ 6660 │ -0.613654 │ -0.233233 │ -0.72178 │ 0.948663 │ -1.19573 │\n", - "│ 6661 │ -0.433578 │ -0.890664 │ 0.4462 │ -1.71787 │ -0.457925 │\n", - "│ 6662 │ -2.6614 │ 0.750482 │ -5.62549 │ 0.86061 │ 0.506262 │\n", - "│ 6663 │ -0.351239 │ -0.16338 │ -0.376714 │ 0.781865 │ 1.62379 │\n", - "│ 6664 │ 1.30857 │ 1.00127 │ 0.876074 │ 1.37296 │ -0.727299 │\n", - "│ 6665 │ -1.74132 │ -4.1342 │ 2.52424 │ -1.4819 │ 1.32056 │\n", - "│ 6666 │ -1.10252 │ -0.717352 │ -0.905295 │ -1.07487 │ 0.505224 │\n", - "│ 6667 │ 0.195008 │ 0.661188 │ -0.544159 │ 1.54089 │ -0.290803 │\n", - "│ 6668 │ 0.251995 │ -0.915712 │ 1.62863 │ -2.2893 │ -0.813678 │\n", - "│ 6669 │ -0.586483 │ 0.256554 │ -1.32053 │ -1.10432 │ -1.7194 │\n", - "│ 6670 │ 1.16462 │ 1.1404 │ 0.451617 │ 1.40992 │ -0.482502 │\n", - "\n", - "│ Row │ Trait11 │ Trait12 │ Trait13 │\n", - "├──────┼───────────┼────────────┼───────────┤\n", - "│ 1 │ 0.824466 │ -1.02853 │ -0.394049 │\n", - "│ 2 │ 3.08264 │ 1.09065 │ 0.0256616 │\n", - "│ 3 │ -1.10649 │ -1.42016 │ -0.687463 │\n", - "│ 4 │ 0.394175 │ -0.767366 │ 0.0635386 │\n", - "│ 5 │ 0.549968 │ 0.540688 │ 0.179675 │\n", - "│ 6 │ 0.947393 │ 0.594015 │ 0.245407 │\n", - "│ 7 │ 0.332636 │ 0.00269165 │ 0.317117 │\n", - "│ 8 │ 0.220861 │ 0.852512 │ -0.225905 │\n", - "│ 9 │ 0.167532 │ 0.246916 │ 0.539933 │\n", - "│ 10 │ 0.7331 │ -1.73468 │ -1.35278 │\n", - "│ 11 │ -0.712244 │ -0.313346 │ -0.345419 │\n", - "⋮\n", - "│ 6659 │ 0.35674 │ 0.456428 │ 0.882577 │\n", - "│ 6660 │ 0.366737 │ 1.78286 │ 1.90764 │\n", - "│ 6661 │ -0.418756 │ -0.275519 │ -0.912778 │\n", - "│ 6662 │ 1.31914 │ -1.44981 │ -1.77373 │\n", - "│ 6663 │ 0.770329 │ -0.0470789 │ 1.50496 │\n", - "│ 6664 │ 0.757479 │ 1.10001 │ 1.29115 │\n", - "│ 6665 │ 1.29209 │ 0.697478 │ 0.228819 │\n", - "│ 6666 │ 1.00973 │ -0.362158 │ -1.55022 │\n", - "│ 6667 │ 0.141684 │ 0.183218 │ 0.122664 │\n", - "│ 6668 │ -0.2214 │ -0.575183 │ 0.399583 │\n", - "│ 6669 │ -0.985545 │ -0.636874 │ -0.439825 │\n", - "│ 6670 │ 1.39201 │ 0.299931 │ 0.392809 │" - ] - }, - "execution_count": 42, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# Pkg.add(\"DataFrames\")\n", - "using DataFrames\n", - "\n", - "cg10k_trait = readtable(\n", - " \"cg10k_traits.txt\"; \n", - " separator = ' ',\n", - " names = [:FID; :IID; :Trait1; :Trait2; :Trait3; :Trait4; :Trait5; :Trait6; \n", - " :Trait7; :Trait8; :Trait9; :Trait10; :Trait11; :Trait12; :Trait13], \n", - " eltypes = [String; String; Float64; Float64; Float64; Float64; Float64; \n", - " Float64; Float64; Float64; Float64; Float64; Float64; Float64; Float64]\n", - " )\n", - "# do not display FID and IID for privacy\n", - "cg10k_trait[:, 3:end]" - ] - }, - { - "cell_type": "code", - "execution_count": 43, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Trait1\n", - "Summary Stats:\n", - "Mean: 0.002211\n", - "Minimum: -3.204128\n", - "1st Quartile: -0.645771\n", - "Median: 0.125010\n", - "3rd Quartile: 0.723315\n", - "Maximum: 3.479398\n", - "Length: 6670\n", - "Type: Float64\n", - "Number Missing: 0\n", - "% Missing: 0.000000\n", - "\n", - "Trait2\n", - "Summary Stats:\n", - "Mean: 0.001353\n", - "Minimum: -3.511659\n", - "1st Quartile: -0.642621\n", - "Median: 0.033517\n", - "3rd Quartile: 0.657467\n", - "Maximum: 4.913423\n", - "Length: 6670\n", - "Type: Float64\n", - "Number Missing: 0\n", - "% Missing: 0.000000\n", - "\n", - "Trait3\n", - "Summary Stats:\n", - "Mean: -0.001296\n", - "Minimum: -3.938436\n", - "1st Quartile: -0.640907\n", - "Median: -0.000782\n", - "3rd Quartile: 0.637108\n", - "Maximum: 7.916299\n", - "Length: 6670\n", - "Type: Float64\n", - "Number Missing: 0\n", - "% Missing: 0.000000\n", - "\n", - "Trait4\n", - "Summary Stats:\n", - "Mean: 0.002309\n", - "Minimum: -3.608403\n", - "1st Quartile: -0.546086\n", - "Median: 0.228165\n", - "3rd Quartile: 0.715291\n", - "Maximum: 3.127688\n", - "Length: 6670\n", - "Type: Float64\n", - "Number Missing: 0\n", - "% Missing: 0.000000\n", - "\n", - "Trait5\n", - "Summary Stats:\n", - "Mean: -0.001790\n", - "Minimum: -4.148749\n", - "1st Quartile: -0.690765\n", - "Median: 0.031034\n", - "3rd Quartile: 0.734916\n", - "Maximum: 2.717184\n", - "Length: 6670\n", - "Type: Float64\n", - "Number Missing: 0\n", - "% Missing: 0.000000\n", - "\n", - "Trait6\n", - "Summary Stats:\n", - "Mean: -0.001196\n", - "Minimum: -3.824792\n", - "1st Quartile: -0.662796\n", - "Median: 0.036242\n", - "3rd Quartile: 0.741176\n", - "Maximum: 2.589728\n", - "Length: 6670\n", - "Type: Float64\n", - "Number Missing: 0\n", - "% Missing: 0.000000\n", - "\n", - "Trait7\n", - "Summary Stats:\n", - "Mean: -0.001989\n", - "Minimum: -4.272455\n", - "1st Quartile: -0.638923\n", - "Median: 0.069801\n", - "3rd Quartile: 0.710423\n", - "Maximum: 2.653779\n", - "Length: 6670\n", - "Type: Float64\n", - "Number Missing: 0\n", - "% Missing: 0.000000\n", - "\n", - "Trait8\n", - "Summary Stats:\n", - "Mean: 0.000614\n", - "Minimum: -5.625488\n", - "1st Quartile: -0.601575\n", - "Median: -0.038630\n", - "3rd Quartile: 0.527342\n", - "Maximum: 5.805702\n", - "Length: 6670\n", - "Type: Float64\n", - "Number Missing: 0\n", - "% Missing: 0.000000\n", - "\n", - "Trait9\n", - "Summary Stats:\n", - "Mean: -0.001810\n", - "Minimum: -5.381968\n", - "1st Quartile: -0.601429\n", - "Median: 0.106571\n", - "3rd Quartile: 0.698567\n", - "Maximum: 2.571936\n", - "Length: 6670\n", - "Type: Float64\n", - "Number Missing: 0\n", - "% Missing: 0.000000\n", - "\n", - "Trait10\n", - "Summary Stats:\n", - "Mean: -0.000437\n", - "Minimum: -3.548506\n", - "1st Quartile: -0.633641\n", - "Median: -0.096651\n", - "3rd Quartile: 0.498610\n", - "Maximum: 6.537820\n", - "Length: 6670\n", - "Type: Float64\n", - "Number Missing: 0\n", - "% Missing: 0.000000\n", - "\n", - "Trait11\n", - "Summary Stats:\n", - "Mean: -0.000616\n", - "Minimum: -3.264910\n", - "1st Quartile: -0.673685\n", - "Median: -0.068044\n", - "3rd Quartile: 0.655486\n", - "Maximum: 4.262410\n", - "Length: 6670\n", - "Type: Float64\n", - "Number Missing: 0\n", - "% Missing: 0.000000\n", - "\n", - "Trait12\n", - "Summary Stats:\n", - "Mean: -0.000589\n", - "Minimum: -8.851909\n", - "1st Quartile: -0.539686\n", - "Median: -0.141099\n", - "3rd Quartile: 0.350779\n", - "Maximum: 13.211402\n", - "Length: 6670\n", - "Type: Float64\n", - "Number Missing: 0\n", - "% Missing: 0.000000\n", - "\n", - "Trait13\n", - "Summary Stats:\n", - "Mean: -0.000151\n", - "Minimum: -5.592104\n", - "1st Quartile: -0.492289\n", - "Median: -0.141022\n", - "3rd Quartile: 0.324804\n", - "Maximum: 24.174436\n", - "Length: 6670\n", - "Type: Float64\n", - "Number Missing: 0\n", - "% Missing: 0.000000\n", - "\n" - ] - } - ], - "source": [ - "describe(cg10k_trait[:, 3:end])" - ] - }, - { - "cell_type": "code", - "execution_count": 44, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, - "outputs": [ - { - "data": { - "text/html": [ - "" - ] - }, - "execution_count": 44, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "Y = convert(Matrix{Float64}, cg10k_trait[:, 3:15])\n", - "histogram(Y, layout = 13)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, - "source": [ - "## Pre-processing data for heritability analysis" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, - "source": [ - "To prepare variance component model fitting, we form an instance of `VarianceComponentVariate`. The two variance components are $(2\\Phi, I)$." - ] - }, - { - "cell_type": "code", - "execution_count": 45, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, - "outputs": [ - { - "data": { - "text/plain": [ - "3-element Array{Symbol,1}:\n", - " :Y\n", - " :X\n", - " :V" - ] - }, - "execution_count": 45, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "using VarianceComponentModels\n", - "\n", - "# form data as VarianceComponentVariate\n", - "cg10kdata = VarianceComponentVariate(Y, (2Φgrm, eye(size(Y, 1))))\n", - "fieldnames(cg10kdata)" - ] - }, - { - "cell_type": "code", - "execution_count": 46, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, - "outputs": [ - { - "data": { - "text/plain": [ - "VarianceComponentModels.VarianceComponentVariate{Float64,2,Array{Float64,2},Array{Float64,2},Array{Float64,2}}([-1.81573 -0.94615 … -1.02853 -0.394049; -1.2444 0.10966 … 1.09065 0.0256616; … ; 0.886626 0.487408 … -0.636874 -0.439825; -1.24394 0.213697 … 0.299931 0.392809], Array{Float64}(6670,0), ([1.00605 0.00671009 … -0.000109037 -0.00556144; 0.00671009 0.997916 … 0.00173694 0.00685701; … ; -0.000109037 0.00173694 … 1.00166 0.000938955; -0.00556144 0.00685701 … 0.000938955 1.00125], [1.0 0.0 … 0.0 0.0; 0.0 1.0 … 0.0 0.0; … ; 0.0 0.0 … 1.0 0.0; 0.0 0.0 … 0.0 1.0]))" - ] - }, - "execution_count": 46, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "cg10kdata" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, - "source": [ - "Before fitting the variance component model, we pre-compute the eigen-decomposition of $2\\Phi_{\\text{GRM}}$, the rotated responses, and the constant part in log-likelihood, and store them as a `TwoVarCompVariateRotate` instance, which is re-used in various variane component estimation procedures." - ] - }, - { - "cell_type": "code", - "execution_count": 47, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " 48.837361 seconds (39 allocations: 1021.427 MiB, 0.57% gc time)\n" - ] - }, - { - "data": { - "text/plain": [ - "5-element Array{Symbol,1}:\n", - " :Yrot \n", - " :Xrot \n", - " :eigval \n", - " :eigvec \n", - " :logdetV2" - ] - }, - "execution_count": 47, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# pre-compute eigen-decomposition (~50 secs on my laptop)\n", - "@time cg10kdata_rotated = TwoVarCompVariateRotate(cg10kdata)\n", - "fieldnames(cg10kdata_rotated)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, - "source": [ - "## Save intermediate results\n", - "\n", - "We don't want to re-compute SnpArray and empirical kinship matrices again and again for heritibility analysis." - ] - }, - { - "cell_type": "code", - "execution_count": 48, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true, - "scrolled": true - }, - "outputs": [], - "source": [ - "# # Pkg.add(\"JLD\")\n", - "# using JLD\n", - "# @save \"cg10k.jld\"\n", - "# whos()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, - "source": [ - "To load workspace" - ] - }, - { - "cell_type": "code", - "execution_count": 49, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true, - "scrolled": true - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " Base Module\n", - " BinDeps 41348 KB Module\n", - " Blosc 41202 KB Module\n", - " ColorTypes 41457 KB Module\n", - " Colors 41480 KB Module\n", - " Compat 41196 KB Module\n", - " Conda 41205 KB Module\n", - " Core Module\n", - " DataArrays 41456 KB Module\n", - " DataFrames 41684 KB Module\n", - " DataStructures 41356 KB Module\n", - " FileIO 41310 KB Module\n", - " FixedPointNumbers 41695 KB Module\n", - " GZip 41181 KB Module\n", - " HDF5 41403 KB Module\n", - " IJulia 4185781 KB Module\n", - " Ipopt 41172 KB Module\n", - " JLD 41376 KB Module\n", - " JSON 41245 KB Module\n", - " LaTeXStrings 4058 bytes Module\n", - " LegacyStrings 41212 KB Module\n", - " LinearMaps 22 KB Module\n", - " MacroTools 41606 KB Module\n", - " Main Module\n", - " MathProgBase 41353 KB Module\n", - " MbedTLS 41269 KB Module\n", - " Measures 41175 KB Module\n", - " NaNMath 41200 KB Module\n", - " PlotThemes 41167 KB Module\n", - " PlotUtils 41332 KB Module\n", - " Plots 42960 KB Module\n", - " PyCall 41711 KB Module\n", - " PyPlot 41771 KB Module\n", - " RecipesBase 41283 KB Module\n", - " Reexport 41160 KB Module\n", - " Requires 41172 KB Module\n", - " SHA 62 KB Module\n", - " Showoff 41163 KB Module\n", - " SnpArrays 41218 KB Module\n", - " SortingAlgorithms 41178 KB Module\n", - " SpecialFunctions 41252 KB Module\n", - " StaticArrays 41744 KB Module\n", - " StatsBase 41810 KB Module\n", - " URIParser 41171 KB Module\n", - " VarianceComponentModels 41278 KB Module\n", - " Y 677 KB 6670×13 Array{Float64,2}\n", - " ZMQ 41223 KB Module\n", - " _ 77 KB 630860-element BitArray{1}\n", - " cg10k 1027303 KB 6670×630860 SnpArrays.SnpArray{2}\n", - " cg10k_trait 978 KB 6670×15 DataFrames.DataFrame\n", - " cg10kdata 695816 KB VarianceComponentModels.VarianceCo…\n", - " cg10kdata_rotated 348299 KB VarianceComponentModels.TwoVarComp…\n", - " h 24 bytes 3-element Array{Float64,1}\n", - " hST 104 bytes 13-element Array{Float64,1}\n", - " hST_se 104 bytes 13-element Array{Float64,1}\n", - " hse 24 bytes 3-element Array{Float64,1}\n", - " maf 4928 KB 630860-element Array{Float64,1}\n", - " missings_by_snp 4928 KB 630860-element Array{Int64,1}\n", - " people 8 bytes Int64\n", - " snps 8 bytes Int64\n", - " trait57_data 347778 KB VarianceComponentModels.TwoVarComp…\n", - " trait57_model 232 bytes VarianceComponentModels.VarianceCo…\n", - " traitall_model 2792 bytes VarianceComponentModels.VarianceCo…\n", - " traitidx 16 bytes 3-element UnitRange{Int64}\n", - " Σa 3848 bytes 13×13 Array{Array{Float64,2},2}\n", - " Σcov 2592 bytes 18×18 Array{Float64,2}\n", - " Σe 3848 bytes 13×13 Array{Array{Float64,2},2}\n", - " Φgrm 347569 KB 6670×6670 Array{Float64,2}\n", - " σ2a 104 bytes 13-element Array{Float64,1}\n", - " σ2e 104 bytes 13-element Array{Float64,1}\n" - ] - } - ], - "source": [ - "using SnpArrays, JLD, DataFrames, VarianceComponentModels, Plots\n", - "pyplot()\n", - "@load \"cg10k.jld\"\n", - "whos()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, - "source": [ - "## Heritability of single traits" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, - "source": [ - "We use Fisher scoring algorithm to fit variance component model for each single trait." - ] - }, - { - "cell_type": "code", - "execution_count": 50, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true, - "scrolled": true - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Trait1\n", - "(σ2a[trait], σ2e[trait]) = (0.25978160614793233, 0.7369535197912689)\n", - "Trait2\n", - "(σ2a[trait], σ2e[trait]) = (0.18647130348299173, 0.8129591079735827)\n", - "Trait3\n", - "(σ2a[trait], σ2e[trait]) = (0.3188368159422607, 0.6798809726936244)\n", - "Trait4\n", - "(σ2a[trait], σ2e[trait]) = (0.2651357653143703, 0.7308007669086968)\n", - "Trait5\n", - "(σ2a[trait], σ2e[trait]) = (0.28083388108246, 0.7172036435586534)\n", - "Trait6\n", - "(σ2a[trait], σ2e[trait]) = (0.2824159905728832, 0.7170988773569172)\n", - "Trait7\n", - "(σ2a[trait], σ2e[trait]) = (0.2155274336968625, 0.7815346282986375)\n", - "Trait8\n", - "(σ2a[trait], σ2e[trait]) = (0.194687807263945, 0.8049690651320599)\n", - "Trait9\n", - "(σ2a[trait], σ2e[trait]) = (0.24706855916591713, 0.7512942998567308)\n", - "Trait10\n", - "(σ2a[trait], σ2e[trait]) = (0.098712236297271, 0.9011756660217387)\n", - "Trait11\n", - "(σ2a[trait], σ2e[trait]) = (0.1664264642608195, 0.8322427413046204)\n", - "Trait12\n", - "(σ2a[trait], σ2e[trait]) = (0.0834296761650666, 0.9153609794266608)\n", - "Trait13\n", - "(σ2a[trait], σ2e[trait]) = (0.05893968504298988, 0.940270012443928)\n", - "elapsed time: 0.160999612 seconds\n" - ] - }, - { - "data": { - "text/plain": [ - "0.160999612" - ] - }, - "execution_count": 50, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# heritability from single trait analysis\n", - "hST = zeros(13)\n", - "# standard errors of estimated heritability\n", - "hST_se = zeros(13)\n", - "# additive genetic effects\n", - "σ2a = zeros(13)\n", - "# enviromental effects\n", - "σ2e = zeros(13)\n", - "\n", - "tic()\n", - "for trait in 1:13\n", - " println(names(cg10k_trait)[trait + 2])\n", - " # form data set for trait j\n", - " traitj_data = TwoVarCompVariateRotate(cg10kdata_rotated.Yrot[:, trait], cg10kdata_rotated.Xrot, \n", - " cg10kdata_rotated.eigval, cg10kdata_rotated.eigvec, cg10kdata_rotated.logdetV2)\n", - " # initialize model parameters\n", - " traitj_model = VarianceComponentModel(traitj_data)\n", - " # estimate variance components\n", - " _, _, _, Σcov, _, _ = mle_fs!(traitj_model, traitj_data; solver=:Ipopt, verbose=false)\n", - " σ2a[trait] = traitj_model.Σ[1][1]\n", - " σ2e[trait] = traitj_model.Σ[2][1]\n", - " @show σ2a[trait], σ2e[trait]\n", - " h, hse = heritability(traitj_model.Σ, Σcov)\n", - " hST[trait] = h[1]\n", - " hST_se[trait] = hse[1]\n", - "end\n", - "toc()" - ] - }, - { - "cell_type": "code", - "execution_count": 51, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, - "outputs": [ - { - "data": { - "text/plain": [ - "2×13 Array{Float64,2}:\n", - " 0.260633 0.186578 0.319246 … 0.166648 0.0835307 0.0589863\n", - " 0.0799732 0.0869002 0.0741007 0.08862 0.0944407 0.0953238" - ] - }, - "execution_count": 51, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# heritability and standard errors\n", - "[hST'; hST_se']" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, - "source": [ - "## Pairwise traits\n", - "\n", - "Joint analysis of multiple traits is subject to intensive research recently. Following code snippet does joint analysis of all pairs of traits, a total of 78 bivariate variane component models." - ] - }, - { - "cell_type": "code", - "execution_count": 52, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Trait1Trait2\n", - "(Σa[i, j], Σe[i, j]) = ([0.258822 0.174358; 0.174358 0.185108], [0.737892 0.585751; 0.585751 0.814301])\n", - "Trait1Trait3\n", - "(Σa[i, j], Σe[i, j]) = ([0.260236 -0.0144726; -0.0144726 0.319245], [0.736512 -0.11979; -0.11979 0.679488])\n", - "Trait1Trait4\n", - "(Σa[i, j], Σe[i, j]) = ([0.259615 0.222203; 0.222203 0.265149], [0.737116 0.599854; 0.599854 0.730788])\n", - "Trait1Trait5\n", - "(Σa[i, j], Σe[i, j]) = ([0.259574 -0.146827; -0.146827 0.28153], [0.737153 -0.254777; -0.254777 0.71653])\n", - "Trait1Trait6\n", - "(Σa[i, j], Σe[i, j]) = ([0.259476 -0.129115; -0.129115 0.282688], [0.73725 -0.23161; -0.23161 0.716837])\n", - "Trait1Trait7\n", - "(Σa[i, j], Σe[i, j]) = ([0.259115 -0.140455; -0.140455 0.215297], [0.737606 -0.197616; -0.197616 0.781774])\n", - "Trait1Trait8\n", - "(Σa[i, j], Σe[i, j]) = ([0.259778 -0.0327756; -0.0327756 0.194698], [0.736957 -0.127026; -0.127026 0.804959])\n", - "Trait1Trait9\n", - "(Σa[i, j], Σe[i, j]) = ([0.261858 -0.204589; -0.204589 0.246027], [0.734961 -0.307734; -0.307734 0.75232])\n", - "Trait1Trait10\n", - "(Σa[i, j], Σe[i, j]) = ([0.259649 -0.0994858; -0.0994858 0.0956585], [0.737083 -0.303942; -0.303942 0.904218])\n", - "Trait1Trait11\n", - "(Σa[i, j], Σe[i, j]) = ([0.25947 -0.138603; -0.138603 0.164709], [0.737257 -0.359557; -0.359557 0.83395])\n", - "Trait1Trait12\n", - "(Σa[i, j], Σe[i, j]) = ([0.261779 -0.145414; -0.145414 0.0807748], [0.735076 -0.041823; -0.041823 0.9181])\n", - "Trait1Trait13\n", - "(Σa[i, j], Σe[i, j]) = ([0.261125 -0.108774; -0.108774 0.0538214], [0.735674 -0.114123; -0.114123 0.945416])\n", - "Trait2Trait3\n", - "(Σa[i, j], Σe[i, j]) = ([0.186541 0.144056; 0.144056 0.320627], [0.812888 0.0995944; 0.0995944 0.678167])\n", - "Trait2Trait4\n", - "(Σa[i, j], Σe[i, j]) = ([0.186131 0.0746032; 0.0746032 0.265122], [0.813293 0.221109; 0.221109 0.730814])\n", - "Trait2Trait5\n", - "(Σa[i, j], Σe[i, j]) = ([0.186442 -0.0118093; -0.0118093 0.280842], [0.812987 -0.0365191; -0.0365191 0.717195])\n", - "Trait2Trait6\n", - "(Σa[i, j], Σe[i, j]) = ([0.18649 -0.00366533; -0.00366533 0.282471], [0.812941 -0.0206271; -0.0206271 0.717046])\n", - "Trait2Trait7\n", - "(Σa[i, j], Σe[i, j]) = ([0.186104 -0.030665; -0.030665 0.215304], [0.81332 -0.000667009; -0.000667009 0.781755])\n", - "Trait2Trait8\n", - "(Σa[i, j], Σe[i, j]) = ([0.187023 0.0331783; 0.0331783 0.195259], [0.812421 -0.0326343; -0.0326343 0.804415])\n", - "Trait2Trait9\n", - "(Σa[i, j], Σe[i, j]) = ([0.185032 -0.085334; -0.085334 0.245909], [0.814386 -0.0809638; -0.0809638 0.752433])\n", - "Trait2Trait10\n", - "(Σa[i, j], Σe[i, j]) = ([0.186587 -0.123303; -0.123303 0.0987387], [0.812872 -0.273083; -0.273083 0.901229])\n", - "Trait2Trait11\n", - "(Σa[i, j], Σe[i, j]) = ([0.185484 -0.117256; -0.117256 0.167776], [0.81393 -0.296772; -0.296772 0.830934])\n", - "Trait2Trait12\n", - "(Σa[i, j], Σe[i, j]) = ([0.185907 -0.0909104; -0.0909104 0.0827171], [0.813555 0.0457924; 0.0457924 0.916135])\n", - "Trait2Trait13\n", - "(Σa[i, j], Σe[i, j]) = ([0.185979 -0.0720811; -0.0720811 0.0568238], [0.8135 0.0751703; 0.0751703 0.942424])\n", - "Trait3Trait4\n", - "(Σa[i, j], Σe[i, j]) = ([0.3188 -0.154562; -0.154562 0.264323], [0.679917 -0.303223; -0.303223 0.731591])\n", - "Trait3Trait5\n", - "(Σa[i, j], Σe[i, j]) = ([0.319216 0.183527; 0.183527 0.282063], [0.679514 0.33724; 0.33724 0.716008])\n", - "Trait3Trait6\n", - "(Σa[i, j], Σe[i, j]) = ([0.319776 0.165672; 0.165672 0.284448], [0.678972 0.298667; 0.298667 0.715124])\n", - "Trait3Trait7\n", - "(Σa[i, j], Σe[i, j]) = ([0.318838 0.166283; 0.166283 0.215261], [0.67988 0.347706; 0.347706 0.781796])\n", - "Trait3Trait8\n", - "(Σa[i, j], Σe[i, j]) = ([0.320718 0.0566397; 0.0566397 0.197764], [0.678063 0.0451569; 0.0451569 0.801956])\n", - "Trait3Trait9\n", - "(Σa[i, j], Σe[i, j]) = ([0.319001 0.137699; 0.137699 0.246142], [0.679722 0.266704; 0.266704 0.752197])\n", - "Trait3Trait10\n", - "(Σa[i, j], Σe[i, j]) = ([0.31908 -0.076513; -0.076513 0.0996001], [0.679646 -0.142905; -0.142905 0.900298])\n", - "Trait3Trait11\n", - "(Σa[i, j], Σe[i, j]) = ([0.318094 -0.0177494; -0.0177494 0.16629], [0.6806 -0.1144; -0.1144 0.832376])\n", - "Trait3Trait12\n", - "(Σa[i, j], Σe[i, j]) = ([0.321164 0.0843842; 0.0843842 0.0874609], [0.677639 0.0341558; 0.0341558 0.911368])\n", - "Trait3Trait13\n", - "(Σa[i, j], Σe[i, j]) = ([0.323273 0.109443; 0.109443 0.0634295], [0.675635 -0.0060525; -0.0060525 0.935819])\n", - "Trait4Trait5\n", - "(Σa[i, j], Σe[i, j]) = ([0.26525 -0.215125; -0.215125 0.282572], [0.73068 -0.377406; -0.377406 0.715518])\n", - "Trait4Trait6\n", - "(Σa[i, j], Σe[i, j]) = ([0.265715 -0.199714; -0.199714 0.283942], [0.730231 -0.347732; -0.347732 0.715619])\n", - "Trait4Trait7\n", - "(Σa[i, j], Σe[i, j]) = ([0.26407 -0.18238; -0.18238 0.214324], [0.731843 -0.32655; -0.32655 0.782733])\n", - "Trait4Trait8\n", - "(Σa[i, j], Σe[i, j]) = ([0.266229 -0.0965381; -0.0965381 0.196655], [0.729739 -0.151461; -0.151461 0.803044])\n", - "Trait4Trait9\n", - "(Σa[i, j], Σe[i, j]) = ([0.269627 -0.226931; -0.226931 0.247265], [0.726443 -0.416085; -0.416085 0.751086])\n", - "Trait4Trait10\n", - "(Σa[i, j], Σe[i, j]) = ([0.265098 -0.0352926; -0.0352926 0.0981462], [0.730847 -0.226248; -0.226248 0.901736])\n", - "Trait4Trait11\n", - "(Σa[i, j], Σe[i, j]) = ([0.265178 -0.0970634; -0.0970634 0.164885], [0.73076 -0.272291; -0.272291 0.833762])\n", - "Trait4Trait12\n", - "(Σa[i, j], Σe[i, j]) = ([0.267732 -0.140985; -0.140985 0.081029], [0.728323 -0.0834791; -0.0834791 0.917815])\n", - "Trait4Trait13\n", - "(Σa[i, j], Σe[i, j]) = ([0.265695 -0.0970238; -0.0970238 0.0564809], [0.730259 -0.226115; -0.226115 0.942736])\n", - "Trait5Trait6\n", - "(Σa[i, j], Σe[i, j]) = ([0.281198 0.280259; 0.280259 0.281764], [0.716855 0.661013; 0.661013 0.717735])\n", - "Trait5Trait7\n", - "(Σa[i, j], Σe[i, j]) = ([0.280442 0.231918; 0.231918 0.211837], [0.717597 0.674491; 0.674491 0.785172])\n", - "Trait5Trait8\n", - "(Σa[i, j], Σe[i, j]) = ([0.280958 0.163168; 0.163168 0.193315], [0.717089 0.221817; 0.221817 0.806314])\n", - "Trait5Trait9\n", - "(Σa[i, j], Σe[i, j]) = ([0.283544 0.243884; 0.243884 0.240564], [0.714585 0.509072; 0.509072 0.757631])\n", - "Trait5Trait10\n", - "(Σa[i, j], Σe[i, j]) = ([0.281378 -0.0454427; -0.0454427 0.100081], [0.716678 -0.0579778; -0.0579778 0.899822])\n", - "Trait5Trait11\n", - "(Σa[i, j], Σe[i, j]) = ([0.280066 0.0195669; 0.0195669 0.165607], [0.71795 -0.0345589; -0.0345589 0.833047])\n", - "Trait5Trait12\n", - "(Σa[i, j], Σe[i, j]) = ([0.28101 0.0592641; 0.0592641 0.0831831], [0.717036 0.0552788; 0.0552788 0.915608])\n", - "Trait5Trait13\n", - "(Σa[i, j], Σe[i, j]) = ([0.281854 0.0680641; 0.0680641 0.0591899], [0.716223 0.0551992; 0.0551992 0.940027])\n", - "Trait6Trait7\n", - "(Σa[i, j], Σe[i, j]) = ([0.282435 0.220236; 0.220236 0.213997], [0.71708 0.581507; 0.581507 0.783041])\n", - "Trait6Trait8\n", - "(Σa[i, j], Σe[i, j]) = ([0.282435 0.18375; 0.18375 0.192999], [0.717081 0.436932; 0.436932 0.80663])\n", - "Trait6Trait9\n", - "(Σa[i, j], Σe[i, j]) = ([0.284516 0.233768; 0.233768 0.242478], [0.715071 0.477502; 0.477502 0.755765])\n", - "Trait6Trait10\n", - "(Σa[i, j], Σe[i, j]) = ([0.283087 -0.0427658; -0.0427658 0.100634], [0.716449 -0.0599491; -0.0599491 0.899275])\n", - "Trait6Trait11\n", - "(Σa[i, j], Σe[i, j]) = ([0.281046 0.0272144; 0.0272144 0.165044], [0.71843 -0.0516242; -0.0516242 0.833601])\n", - "Trait6Trait12\n", - "(Σa[i, j], Σe[i, j]) = ([0.28256 0.0548537; 0.0548537 0.083133], [0.716961 0.0502064; 0.0502064 0.915658])\n", - "Trait6Trait13\n", - "(Σa[i, j], Σe[i, j]) = ([0.283231 0.0585667; 0.0585667 0.0592752], [0.716314 0.055827; 0.055827 0.939942])\n", - "Trait7Trait8\n", - "(Σa[i, j], Σe[i, j]) = ([0.213998 0.0875641; 0.0875641 0.192993], [0.78304 -0.055939; -0.055939 0.806635])\n", - "Trait7Trait9\n", - "(Σa[i, j], Σe[i, j]) = ([0.219039 0.216925; 0.216925 0.243338], [0.778156 0.463024; 0.463024 0.754935])\n", - "Trait7Trait10\n", - "(Σa[i, j], Σe[i, j]) = ([0.216296 -0.0412106; -0.0412106 0.100663], [0.780785 -0.0868086; -0.0868086 0.899246])\n", - "Trait7Trait11\n", - "(Σa[i, j], Σe[i, j]) = ([0.2142 0.0204227; 0.0204227 0.165077], [0.782833 -0.0478727; -0.0478727 0.833569])\n", - "Trait7Trait12\n", - "(Σa[i, j], Σe[i, j]) = ([0.215054 0.0738562; 0.0738562 0.0814228], [0.782012 0.0366272; 0.0366272 0.917365])\n", - "Trait7Trait13\n", - "(Σa[i, j], Σe[i, j]) = ([0.216093 0.0728515; 0.0728515 0.0570272], [0.781006 0.0409945; 0.0409945 0.942189])\n", - "Trait8Trait9\n", - "(Σa[i, j], Σe[i, j]) = ([0.195154 0.111756; 0.111756 0.246453], [0.804528 0.185842; 0.185842 0.751896])\n", - "Trait8Trait10\n", - "(Σa[i, j], Σe[i, j]) = ([0.195015 -0.015506; -0.015506 0.0990776], [0.804651 0.0118538; 0.0118538 0.900815])\n", - "Trait8Trait11\n", - "(Σa[i, j], Σe[i, j]) = ([0.194421 0.0215044; 0.0215044 0.166226], [0.805231 -0.026247; -0.026247 0.83244])\n", - "Trait8Trait12\n", - "(Σa[i, j], Σe[i, j]) = ([0.194491 -0.00425152; -0.00425152 0.0832711], [0.805162 0.0349872; 0.0349872 0.915518])\n", - "Trait8Trait13\n", - "(Σa[i, j], Σe[i, j]) = ([0.19448 0.00235501; 0.00235501 0.0589351], [0.805173 0.0396048; 0.0396048 0.940275])\n", - "Trait9Trait10\n", - "(Σa[i, j], Σe[i, j]) = ([0.246455 -0.00257997; -0.00257997 0.0984563], [0.751895 0.0743439; 0.0743439 0.901429])\n", - "Trait9Trait11\n", - "(Σa[i, j], Σe[i, j]) = ([0.247001 0.0303415; 0.0303415 0.166421], [0.75136 0.153765; 0.153765 0.832248])\n", - "Trait9Trait12\n", - "(Σa[i, j], Σe[i, j]) = ([0.249421 0.0829968; 0.0829968 0.0890874], [0.749007 0.109331; 0.109331 0.909778])\n", - "Trait9Trait13\n", - "(Σa[i, j], Σe[i, j]) = ([0.24861 0.0916799; 0.0916799 0.0602352], [0.749811 0.100027; 0.100027 0.939032])\n", - "Trait10Trait11\n", - "(Σa[i, j], Σe[i, j]) = ([0.0914658 0.100613; 0.100613 0.166501], [0.908376 0.473847; 0.473847 0.83217])\n", - "Trait10Trait12\n", - "(Σa[i, j], Σe[i, j]) = ([0.0951392 0.0588424; 0.0588424 0.0796744], [0.904735 0.0828862; 0.0828862 0.919115])\n", - "Trait10Trait13\n", - "(Σa[i, j], Σe[i, j]) = ([0.0995192 -0.0257171; -0.0257171 0.0598595], [0.900397 0.163778; 0.163778 0.939368])\n", - "Trait11Trait12\n", - "(Σa[i, j], Σe[i, j]) = ([0.165386 0.0579914; 0.0579914 0.0796005], [0.83327 0.144637; 0.144637 0.919166])\n", - "Trait11Trait13\n", - "(Σa[i, j], Σe[i, j]) = ([0.166417 -0.000985185; -0.000985185 0.0595681], [0.832265 0.200012; 0.200012 0.939646])\n", - "Trait12Trait13\n", - "(Σa[i, j], Σe[i, j]) = ([0.085082 0.0696185; 0.0696185 0.0569655], [0.913729 0.572041; 0.572041 0.942247])\n", - "elapsed time: 3.587337102 seconds\n" - ] - }, - { - "data": { - "text/plain": [ - "3.587337102" - ] - }, - "execution_count": 52, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# additive genetic effects (2x2 psd matrices) from bavariate trait analysis;\n", - "Σa = Array{Matrix{Float64}}(13, 13)\n", - "# environmental effects (2x2 psd matrices) from bavariate trait analysis;\n", - "Σe = Array{Matrix{Float64}}(13, 13)\n", - "\n", - "tic()\n", - "for i in 1:13\n", - " for j in (i+1):13\n", - " println(names(cg10k_trait)[i + 2], names(cg10k_trait)[j + 2])\n", - " # form data set for (trait1, trait2)\n", - " traitij_data = TwoVarCompVariateRotate(cg10kdata_rotated.Yrot[:, [i;j]], cg10kdata_rotated.Xrot, \n", - " cg10kdata_rotated.eigval, cg10kdata_rotated.eigvec, cg10kdata_rotated.logdetV2)\n", - " # initialize model parameters\n", - " traitij_model = VarianceComponentModel(traitij_data)\n", - " # estimate variance components\n", - " mle_fs!(traitij_model, traitij_data; solver=:Ipopt, verbose=false)\n", - " Σa[i, j] = traitij_model.Σ[1]\n", - " Σe[i, j] = traitij_model.Σ[2]\n", - " @show Σa[i, j], Σe[i, j]\n", - " end\n", - "end\n", - "toc()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, - "source": [ - "## 3-trait analysis\n", - "\n", - "Researchers want to jointly analyze traits 5-7. Our strategy is to try both Fisher scoring and MM algorithm with different starting point, and choose the best local optimum. We first form the data set and run Fisher scoring, which yields a final objective value -1.4700991+04." - ] - }, - { - "cell_type": "code", - "execution_count": 53, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "This is Ipopt version 3.12.4, running with linear solver mumps.\n", - "NOTE: Other linear solvers might be more efficient (see Ipopt documentation).\n", - "\n", - "Number of nonzeros in equality constraint Jacobian...: 0\n", - "Number of nonzeros in inequality constraint Jacobian.: 0\n", - "Number of nonzeros in Lagrangian Hessian.............: 78\n", - "\n", - "Total number of variables............................: 12\n", - " variables with only lower bounds: 0\n", - " variables with lower and upper bounds: 0\n", - " variables with only upper bounds: 0\n", - "Total number of equality constraints.................: 0\n", - "Total number of inequality constraints...............: 0\n", - " inequality constraints with only lower bounds: 0\n", - " inequality constraints with lower and upper bounds: 0\n", - " inequality constraints with only upper bounds: 0\n", - "\n", - "iter objective inf_pr inf_du lg(mu) ||d|| lg(rg) alpha_du alpha_pr ls\n", - " 0 3.0247565e+04 0.00e+00 1.00e+02 0.0 0.00e+00 - 0.00e+00 0.00e+00 0 \n", - " 5 1.6835078e+04 0.00e+00 4.08e+02 -11.0 3.64e-01 - 1.00e+00 1.00e+00f 1 MaxS\n", - " 10 1.4742941e+04 0.00e+00 1.10e+02 -11.0 2.35e-01 - 1.00e+00 1.00e+00f 1 MaxS\n", - " 15 1.4701394e+04 0.00e+00 1.16e+01 -11.0 7.78e-02 -4.5 1.00e+00 1.00e+00f 1 MaxS\n", - " 20 1.4701019e+04 0.00e+00 5.75e-01 -11.0 1.51e-04 -6.9 1.00e+00 1.00e+00f 1 MaxS\n", - " 25 1.4701018e+04 0.00e+00 2.40e-02 -11.0 6.38e-06 -9.2 1.00e+00 1.00e+00f 1 MaxS\n", - " 30 1.4701018e+04 0.00e+00 9.98e-04 -11.0 2.66e-07 -11.6 1.00e+00 1.00e+00f 1 MaxS\n", - " 35 1.4701018e+04 0.00e+00 4.15e-05 -11.0 1.10e-08 -14.0 1.00e+00 1.00e+00h 1 MaxS\n", - " 40 1.4701018e+04 0.00e+00 1.72e-06 -11.0 4.59e-10 -16.4 1.00e+00 1.00e+00f 1 MaxSA\n", - " 45 1.4701018e+04 0.00e+00 7.17e-08 -11.0 1.91e-11 -18.8 1.00e+00 1.00e+00h 1 MaxSA\n", - "iter objective inf_pr inf_du lg(mu) ||d|| lg(rg) alpha_du alpha_pr ls\n", - "\n", - "Number of Iterations....: 49\n", - "\n", - " (scaled) (unscaled)\n", - "Objective...............: 4.4720359684330265e+02 1.4701017692082147e+04\n", - "Dual infeasibility......: 5.6081357364421780e-09 1.8435742302386474e-07\n", - "Constraint violation....: 0.0000000000000000e+00 0.0000000000000000e+00\n", - "Complementarity.........: 0.0000000000000000e+00 0.0000000000000000e+00\n", - "Overall NLP error.......: 5.6081357364421780e-09 1.8435742302386474e-07\n", - "\n", - "\n", - "Number of objective function evaluations = 50\n", - "Number of objective gradient evaluations = 50\n", - "Number of equality constraint evaluations = 0\n", - "Number of inequality constraint evaluations = 0\n", - "Number of equality constraint Jacobian evaluations = 0\n", - "Number of inequality constraint Jacobian evaluations = 0\n", - "Number of Lagrangian Hessian evaluations = 49\n", - "Total CPU secs in IPOPT (w/o function evaluations) = 0.014\n", - "Total CPU secs in NLP function evaluations = 0.076\n", - "\n", - "EXIT: Optimal Solution Found.\n", - " 0.097955 seconds (55.15 k allocations: 5.632 MiB)\n" - ] - }, - { - "data": { - "text/plain": [ - "VarianceComponentModels.VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}(Array{Float64}(0,3), ([0.280777 0.279441 0.232208; 0.279441 0.28422 0.219831; 0.232208 0.219831 0.212832], [0.717266 0.66183 0.674206; 0.66183 0.715287 0.581891; 0.674206 0.581891 0.784183]), Array{Float64}(0,0), Char[], Float64[], -Inf, Inf)" - ] - }, - "execution_count": 53, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "traitidx = 5:7\n", - "# form data set\n", - "trait57_data = TwoVarCompVariateRotate(cg10kdata_rotated.Yrot[:, traitidx], cg10kdata_rotated.Xrot, \n", - " cg10kdata_rotated.eigval, cg10kdata_rotated.eigvec, cg10kdata_rotated.logdetV2)\n", - "# initialize model parameters\n", - "trait57_model = VarianceComponentModel(trait57_data)\n", - "# estimate variance components\n", - "@time mle_fs!(trait57_model, trait57_data; solver=:Ipopt, verbose=true)\n", - "trait57_model" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, - "source": [ - "We then run the MM algorithm, starting from the Fisher scoring answer. MM finds an improved solution with objective value 8.955397e+03." - ] - }, - { - "cell_type": "code", - "execution_count": 54, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - " MM Algorithm\n", - " Iter Objective \n", - "-------- -------------\n", - " 0 -1.470102e+04\n", - " 1 -1.470102e+04\n", - "\n", - " 0.003006 seconds (21.01 k allocations: 1.551 MiB)\n" - ] - }, - { - "data": { - "text/plain": [ - "VarianceComponentModels.VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}(Array{Float64}(0,3), ([0.280777 0.279441 0.232208; 0.279441 0.28422 0.219831; 0.232208 0.219831 0.212832], [0.717266 0.66183 0.674206; 0.66183 0.715287 0.581891; 0.674206 0.581891 0.784183]), Array{Float64}(0,0), Char[], Float64[], -Inf, Inf)" - ] - }, - "execution_count": 54, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# trait59_model contains the fitted model by Fisher scoring now\n", - "@time mle_mm!(trait57_model, trait57_data; verbose=true)\n", - "trait57_model" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, - "source": [ - "Do another run of MM algorithm from default starting point. It leads to a slightly better local optimum -1.470104e+04, slighly worse than the Fisher scoring result. Follow up anlaysis should use the Fisher scoring result." - ] - }, - { - "cell_type": "code", - "execution_count": 55, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - " MM Algorithm\n", - " Iter Objective \n", - "-------- -------------\n", - " 0 -3.024757e+04\n", - " 1 -2.040300e+04\n", - " 2 -1.656070e+04\n", - " 3 -1.528529e+04\n", - " 4 -1.490986e+04\n", - " 5 -1.480638e+04\n", - " 6 -1.477811e+04\n", - " 7 -1.476968e+04\n", - " 8 -1.476639e+04\n", - " 9 -1.476444e+04\n", - " 10 -1.476286e+04\n", - " 20 -1.475000e+04\n", - " 30 -1.474011e+04\n", - " 40 -1.473248e+04\n", - " 50 -1.472658e+04\n", - " 60 -1.472200e+04\n", - " 70 -1.471840e+04\n", - " 80 -1.471555e+04\n", - " 90 -1.471328e+04\n", - " 100 -1.471145e+04\n", - " 110 -1.470997e+04\n", - " 120 -1.470875e+04\n", - " 130 -1.470775e+04\n", - " 140 -1.470691e+04\n", - " 150 -1.470621e+04\n", - " 160 -1.470562e+04\n", - " 170 -1.470511e+04\n", - " 180 -1.470469e+04\n", - " 190 -1.470432e+04\n", - " 200 -1.470400e+04\n", - " 210 -1.470372e+04\n", - " 220 -1.470348e+04\n", - " 230 -1.470326e+04\n", - " 240 -1.470308e+04\n", - " 250 -1.470291e+04\n", - " 260 -1.470276e+04\n", - " 270 -1.470263e+04\n", - " 280 -1.470251e+04\n", - " 290 -1.470241e+04\n", - " 300 -1.470231e+04\n", - " 310 -1.470223e+04\n", - " 320 -1.470215e+04\n", - " 330 -1.470208e+04\n", - " 340 -1.470201e+04\n", - " 350 -1.470195e+04\n", - " 360 -1.470190e+04\n", - " 370 -1.470185e+04\n", - " 380 -1.470180e+04\n", - " 390 -1.470176e+04\n", - " 400 -1.470172e+04\n", - " 410 -1.470168e+04\n", - " 420 -1.470165e+04\n", - " 430 -1.470162e+04\n", - " 440 -1.470159e+04\n", - " 450 -1.470156e+04\n", - " 460 -1.470153e+04\n", - " 470 -1.470151e+04\n", - " 480 -1.470149e+04\n", - " 490 -1.470147e+04\n", - " 500 -1.470145e+04\n", - " 510 -1.470143e+04\n", - " 520 -1.470141e+04\n", - " 530 -1.470139e+04\n", - " 540 -1.470138e+04\n", - " 550 -1.470136e+04\n", - " 560 -1.470135e+04\n", - " 570 -1.470133e+04\n", - " 580 -1.470132e+04\n", - " 590 -1.470131e+04\n", - " 600 -1.470130e+04\n", - " 610 -1.470129e+04\n", - " 620 -1.470128e+04\n", - " 630 -1.470127e+04\n", - " 640 -1.470126e+04\n", - " 650 -1.470125e+04\n", - " 660 -1.470124e+04\n", - " 670 -1.470123e+04\n", - " 680 -1.470122e+04\n", - " 690 -1.470122e+04\n", - " 700 -1.470121e+04\n", - " 710 -1.470120e+04\n", - " 720 -1.470120e+04\n", - " 730 -1.470119e+04\n", - " 740 -1.470118e+04\n", - " 750 -1.470118e+04\n", - " 760 -1.470117e+04\n", - " 770 -1.470117e+04\n", - " 780 -1.470116e+04\n", - " 790 -1.470116e+04\n", - " 800 -1.470115e+04\n", - " 810 -1.470115e+04\n", - " 820 -1.470114e+04\n", - " 830 -1.470114e+04\n", - " 840 -1.470114e+04\n", - " 850 -1.470113e+04\n", - " 860 -1.470113e+04\n", - " 870 -1.470112e+04\n", - " 880 -1.470112e+04\n", - " 890 -1.470112e+04\n", - " 900 -1.470111e+04\n", - " 910 -1.470111e+04\n", - " 920 -1.470111e+04\n", - " 930 -1.470111e+04\n", - " 940 -1.470110e+04\n", - " 950 -1.470110e+04\n", - " 960 -1.470110e+04\n", - " 970 -1.470109e+04\n", - " 980 -1.470109e+04\n", - " 990 -1.470109e+04\n", - " 1000 -1.470109e+04\n", - " 1010 -1.470109e+04\n", - " 1020 -1.470108e+04\n", - " 1030 -1.470108e+04\n", - " 1040 -1.470108e+04\n", - " 1050 -1.470108e+04\n", - " 1060 -1.470108e+04\n", - " 1070 -1.470107e+04\n", - " 1080 -1.470107e+04\n", - " 1090 -1.470107e+04\n", - " 1100 -1.470107e+04\n", - " 1110 -1.470107e+04\n", - " 1120 -1.470107e+04\n", - "\n", - " 0.794377 seconds (168.12 k allocations: 15.640 MiB, 0.80% gc time)\n" - ] - }, - { - "data": { - "text/plain": [ - "VarianceComponentModels.VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}(Array{Float64}(0,3), ([0.2808 0.279454 0.232256; 0.279454 0.284312 0.219977; 0.232256 0.219977 0.213052], [0.717243 0.661816 0.674158; 0.661816 0.715193 0.581746; 0.674158 0.581746 0.783965]), Array{Float64}(0,0), Char[], Float64[], -Inf, Inf)" - ] - }, - "execution_count": 55, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# default starting point\n", - "trait57_model = VarianceComponentModel(trait57_data)\n", - "@time _, _, _, Σcov, = mle_mm!(trait57_model, trait57_data; verbose=true)\n", - "trait57_model" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, - "source": [ - "Heritability from 3-variate estimate and their standard errors." - ] - }, - { - "cell_type": "code", - "execution_count": 56, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, - "outputs": [ - { - "data": { - "text/plain": [ - "2×3 Array{Float64,2}:\n", - " 0.281351 0.284453 0.213689\n", - " 0.0778252 0.077378 0.084084" - ] - }, - "execution_count": 56, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "h, hse = heritability(trait57_model.Σ, Σcov)\n", - "[h'; hse']" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 13-trait joint analysis\n", - "\n", - "In some situations, such as studying the genetic covariance, we need to jointly analyze 13 traits. We first try the **Fisher scoring algorithm**." - ] - }, - { - "cell_type": "code", - "execution_count": 57, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "This is Ipopt version 3.12.4, running with linear solver mumps.\n", - "NOTE: Other linear solvers might be more efficient (see Ipopt documentation).\n", - "\n", - "Number of nonzeros in equality constraint Jacobian...: 0\n", - "Number of nonzeros in inequality constraint Jacobian.: 0\n", - "Number of nonzeros in Lagrangian Hessian.............: 16653\n", - "\n", - "Total number of variables............................: 182\n", - " variables with only lower bounds: 0\n", - " variables with lower and upper bounds: 0\n", - " variables with only upper bounds: 0\n", - "Total number of equality constraints.................: 0\n", - "Total number of inequality constraints...............: 0\n", - " inequality constraints with only lower bounds: 0\n", - " inequality constraints with lower and upper bounds: 0\n", - " inequality constraints with only upper bounds: 0\n", - "\n", - "iter objective inf_pr inf_du lg(mu) ||d|| lg(rg) alpha_du alpha_pr ls\n", - " 0 1.3113371e+05 0.00e+00 1.00e+02 0.0 0.00e+00 - 0.00e+00 0.00e+00 0 \n", - " 5 8.2233766e+04 0.00e+00 6.03e+02 -11.0 2.32e+00 - 1.00e+00 1.00e+00f 1 MaxS\n", - " 10 1.1960260e+05 0.00e+00 8.76e+02 -11.0 6.20e+01 -5.4 1.00e+00 1.00e+00h 1 MaxS\n", - " 15 2.4416551e+05 0.00e+00 2.50e+02 -11.0 8.69e+02 -7.8 1.00e+00 1.00e+00f 1 MaxS\n" - ] - }, - { - "ename": "LoadError", - "evalue": "DomainError:\nlog will only return a complex result if called with a complex argument. Try log(complex(x)).", - "output_type": "error", - "traceback": [ - "DomainError:\nlog will only return a complex result if called with a complex argument. Try log(complex(x)).", - "", - "Stacktrace:", - " [1] \u001b[1mnan_dom_err\u001b[22m\u001b[22m at \u001b[1m./math.jl:300\u001b[22m\u001b[22m [inlined]", - " [2] \u001b[1mlog\u001b[22m\u001b[22m at \u001b[1m./math.jl:419\u001b[22m\u001b[22m [inlined]", - " [3] \u001b[1mlogdet\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::Array{Float64,2}\u001b[1m)\u001b[22m\u001b[22m at \u001b[1m./linalg/generic.jl:1244\u001b[22m\u001b[22m", - " [4] \u001b[1mVarianceComponentModels.TwoVarCompModelRotate\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::VarianceComponentModels.VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}\u001b[1m)\u001b[22m\u001b[22m at \u001b[1m/Users/huazhou/.julia/v0.6/VarianceComponentModels/src/VarianceComponentModels.jl:127\u001b[22m\u001b[22m", - " [5] \u001b[1meval_f\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::VarianceComponentModels.TwoVarCompOptProb{VarianceComponentModels.VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}},VarianceComponentModels.TwoVarCompVariateRotate{Float64,Array{Float64,2},Array{Float64,2}},Array{Float64,2},Array{Float64,1},VarianceComponentModels.VarianceComponentAuxData{Array{Float64,2},Array{Float64,1}}}, ::Array{Float64,1}\u001b[1m)\u001b[22m\u001b[22m at \u001b[1m/Users/huazhou/.julia/v0.6/VarianceComponentModels/src/two_variance_component.jl:683\u001b[22m\u001b[22m", - " [6] \u001b[1m(::Ipopt.#eval_f_cb#4{VarianceComponentModels.TwoVarCompOptProb{VarianceComponentModels.VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}},VarianceComponentModels.TwoVarCompVariateRotate{Float64,Array{Float64,2},Array{Float64,2}},Array{Float64,2},Array{Float64,1},VarianceComponentModels.VarianceComponentAuxData{Array{Float64,2},Array{Float64,1}}}})\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::Array{Float64,1}\u001b[1m)\u001b[22m\u001b[22m at \u001b[1m/Users/huazhou/.julia/v0.6/Ipopt/src/IpoptSolverInterface.jl:53\u001b[22m\u001b[22m", - " [7] \u001b[1meval_f_wrapper\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::Int32, ::Ptr{Float64}, ::Int32, ::Ptr{Float64}, ::Ptr{Void}\u001b[1m)\u001b[22m\u001b[22m at \u001b[1m/Users/huazhou/.julia/v0.6/Ipopt/src/Ipopt.jl:89\u001b[22m\u001b[22m", - " [8] \u001b[1msolveProblem\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::Ipopt.IpoptProblem\u001b[1m)\u001b[22m\u001b[22m at \u001b[1m/Users/huazhou/.julia/v0.6/Ipopt/src/Ipopt.jl:304\u001b[22m\u001b[22m", - " [9] \u001b[1moptimize!\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::Ipopt.IpoptMathProgModel\u001b[1m)\u001b[22m\u001b[22m at \u001b[1m/Users/huazhou/.julia/v0.6/Ipopt/src/IpoptSolverInterface.jl:120\u001b[22m\u001b[22m", - " [10] \u001b[1m#mle_fs!#29\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::Int64, ::Symbol, ::Symbol, ::Bool, ::Function, ::VarianceComponentModels.VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}, ::VarianceComponentModels.TwoVarCompVariateRotate{Float64,Array{Float64,2},Array{Float64,2}}\u001b[1m)\u001b[22m\u001b[22m at \u001b[1m/Users/huazhou/.julia/v0.6/VarianceComponentModels/src/two_variance_component.jl:893\u001b[22m\u001b[22m", - " [11] \u001b[1m(::VarianceComponentModels.#kw##mle_fs!)\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::Array{Any,1}, ::VarianceComponentModels.#mle_fs!, ::VarianceComponentModels.VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}, ::VarianceComponentModels.TwoVarCompVariateRotate{Float64,Array{Float64,2},Array{Float64,2}}\u001b[1m)\u001b[22m\u001b[22m at \u001b[1m./:0\u001b[22m\u001b[22m", - " [12] \u001b[1minclude_string\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::String, ::String\u001b[1m)\u001b[22m\u001b[22m at \u001b[1m./loading.jl:515\u001b[22m\u001b[22m" - ] - } - ], - "source": [ - "# initialize model parameters\n", - "traitall_model = VarianceComponentModel(cg10kdata_rotated)\n", - "# estimate variance components using Fisher scoring algorithm\n", - "@time mle_fs!(traitall_model, cg10kdata_rotated; solver=:Ipopt, verbose=true)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "From the output we can see the Fisher scoring algorithm ran into some numerical issues. Let's try the **MM algorithm**." - ] - }, - { - "cell_type": "code", - "execution_count": 58, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - " MM Algorithm\n", - " Iter Objective \n", - "-------- -------------\n", - " 0 -1.311337e+05\n", - " 1 -8.002108e+04\n", - " 2 -5.806935e+04\n", - " 3 -4.926111e+04\n", - " 4 -4.611059e+04\n", - " 5 -4.511606e+04\n", - " 6 -4.482679e+04\n", - " 7 -4.474294e+04\n", - " 8 -4.471496e+04\n", - " 9 -4.470174e+04\n", - " 10 -4.469246e+04\n", - " 20 -4.462243e+04\n", - " 30 -4.456888e+04\n", - " 40 -4.452774e+04\n", - " 50 -4.449601e+04\n", - " 60 -4.447134e+04\n", - " 70 -4.445199e+04\n", - " 80 -4.443665e+04\n", - " 90 -4.442436e+04\n", - " 100 -4.441442e+04\n", - " 110 -4.440630e+04\n", - " 120 -4.439961e+04\n", - " 130 -4.439405e+04\n", - " 140 -4.438938e+04\n", - " 150 -4.438544e+04\n", - " 160 -4.438210e+04\n", - " 170 -4.437923e+04\n", - " 180 -4.437676e+04\n", - " 190 -4.437463e+04\n", - " 200 -4.437277e+04\n", - " 210 -4.437115e+04\n", - " 220 -4.436972e+04\n", - " 230 -4.436846e+04\n", - " 240 -4.436735e+04\n", - " 250 -4.436636e+04\n", - " 260 -4.436548e+04\n", - " 270 -4.436469e+04\n", - " 280 -4.436399e+04\n", - " 290 -4.436335e+04\n", - " 300 -4.436278e+04\n", - " 310 -4.436226e+04\n", - " 320 -4.436179e+04\n", - " 330 -4.436137e+04\n", - " 340 -4.436098e+04\n", - " 350 -4.436063e+04\n", - " 360 -4.436030e+04\n", - " 370 -4.436001e+04\n", - " 380 -4.435974e+04\n", - " 390 -4.435949e+04\n", - " 400 -4.435926e+04\n", - " 410 -4.435905e+04\n", - " 420 -4.435886e+04\n", - " 430 -4.435868e+04\n", - " 440 -4.435851e+04\n", - " 450 -4.435836e+04\n", - " 460 -4.435822e+04\n", - " 470 -4.435809e+04\n", - " 480 -4.435797e+04\n", - " 490 -4.435785e+04\n", - " 500 -4.435775e+04\n", - " 510 -4.435765e+04\n", - " 520 -4.435756e+04\n", - " 530 -4.435747e+04\n", - " 540 -4.435739e+04\n", - " 550 -4.435732e+04\n", - " 560 -4.435725e+04\n", - " 570 -4.435718e+04\n", - " 580 -4.435712e+04\n", - " 590 -4.435706e+04\n", - " 600 -4.435701e+04\n", - " 610 -4.435696e+04\n", - " 620 -4.435691e+04\n", - " 630 -4.435687e+04\n", - " 640 -4.435683e+04\n", - " 650 -4.435679e+04\n", - " 660 -4.435675e+04\n", - " 670 -4.435671e+04\n", - " 680 -4.435668e+04\n", - " 690 -4.435665e+04\n", - " 700 -4.435662e+04\n", - " 710 -4.435659e+04\n", - " 720 -4.435657e+04\n", - " 730 -4.435654e+04\n", - " 740 -4.435652e+04\n", - " 750 -4.435649e+04\n", - " 760 -4.435647e+04\n", - " 770 -4.435645e+04\n", - " 780 -4.435643e+04\n", - " 790 -4.435642e+04\n", - " 800 -4.435640e+04\n", - " 810 -4.435638e+04\n", - " 820 -4.435637e+04\n", - " 830 -4.435635e+04\n", - " 840 -4.435634e+04\n", - " 850 -4.435633e+04\n", - " 860 -4.435631e+04\n", - " 870 -4.435630e+04\n", - " 880 -4.435629e+04\n", - " 890 -4.435628e+04\n", - " 900 -4.435627e+04\n", - " 910 -4.435626e+04\n", - " 920 -4.435625e+04\n", - " 930 -4.435624e+04\n", - " 940 -4.435623e+04\n", - " 950 -4.435622e+04\n", - " 960 -4.435621e+04\n", - " 970 -4.435621e+04\n", - " 980 -4.435620e+04\n", - " 990 -4.435619e+04\n", - " 1000 -4.435619e+04\n", - " 1010 -4.435618e+04\n", - " 1020 -4.435617e+04\n", - " 1030 -4.435617e+04\n", - " 1040 -4.435616e+04\n", - " 1050 -4.435616e+04\n", - " 1060 -4.435615e+04\n", - " 1070 -4.435615e+04\n", - " 1080 -4.435614e+04\n", - " 1090 -4.435614e+04\n", - "\n", - " 3.551301 seconds (178.42 k allocations: 70.115 MiB, 0.42% gc time)\n" - ] - }, - { - "data": { - "text/plain": [ - "(-44356.138529861186, VarianceComponentModels.VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}(Array{Float64}(0,13), ([0.272384 0.190358 … -0.128222 -0.0980655; 0.190358 0.21692 … -0.0689912 -0.0444349; … ; -0.128222 -0.0689912 … 0.118227 0.0909188; -0.0980655 -0.0444349 … 0.0909188 0.107456], [0.724562 0.56992 … -0.0590518 -0.124939; 0.56992 0.782639 … 0.0238629 0.0475408; … ; -0.0590518 0.0238629 … 0.880671 0.550889; -0.124939 0.0475408 … 0.550889 0.891929]), Array{Float64}(0,0), Char[], Float64[], -Inf, Inf), ([0.0111619 0.0131088 … 0.0128956 0.0127641; 0.0131091 0.0151759 … 0.017162 0.0171466; … ; 0.0128956 0.017162 … 0.0173994 0.0182002; 0.0127643 0.0171461 … 0.0182003 0.0187848], [0.0112235 0.0133094 … 0.0130111 0.0127861; 0.01331 0.0158262 … 0.017867 0.017798; … ; 0.013011 0.0178666 … 0.0179487 0.0187579; 0.012786 0.0177975 … 0.0187578 0.0193328]), [0.000124587 7.24074e-5 … -3.35716e-7 -1.40982e-5; 7.24411e-5 0.000171849 … -2.05381e-5 -3.17975e-6; … ; -3.60221e-7 -2.05683e-5 … 0.000351859 -1.5168e-5; -1.40799e-5 -3.16738e-6 … -1.51641e-5 0.000373756], Array{Float64}(0,13), Array{Float64}(0,0))" - ] - }, - "execution_count": 58, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# reset model parameters\n", - "traitall_model = VarianceComponentModel(cg10kdata_rotated)\n", - "# estimate variance components using Fisher scoring algorithm\n", - "@time mle_mm!(traitall_model, cg10kdata_rotated; verbose=true)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "It converges after ~1000 iterations." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, - "source": [ - "## Save analysis results" - ] - }, - { - "cell_type": "code", - "execution_count": 59, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, - "outputs": [], - "source": [ - "#using JLD\n", - "#@save \"copd.jld\"\n", - "#whos()" - ] - } - ], - "metadata": { - "anaconda-cloud": {}, - "kernelspec": { - "display_name": "Julia 0.6.0", - "language": "julia", - "name": "julia-0.6" - }, - "language_info": { - "file_extension": ".jl", - "mimetype": "application/julia", - "name": "julia", - "version": "0.6.0" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/docs/mle_reml.ipynb b/docs/mle_reml.ipynb deleted file mode 100644 index d7f69dc..0000000 --- a/docs/mle_reml.ipynb +++ /dev/null @@ -1,1408 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# MLE and REML" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Machine information" - ] - }, - { - "cell_type": "code", - "execution_count": 37, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Julia Version 0.6.0\n", - "Commit 903644385b (2017-06-19 13:05 UTC)\n", - "Platform Info:\n", - " OS: macOS (x86_64-apple-darwin13.4.0)\n", - " CPU: Intel(R) Core(TM) i7-4790K CPU @ 4.00GHz\n", - " WORD_SIZE: 64\n", - " BLAS: libopenblas (USE64BITINT DYNAMIC_ARCH NO_AFFINITY Haswell)\n", - " LAPACK: libopenblas64_\n", - " LIBM: libopenlibm\n", - " LLVM: libLLVM-3.9.1 (ORCJIT, haswell)\n" - ] - } - ], - "source": [ - "versioninfo()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Demo data" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "For demonstration, we generate a random data set." - ] - }, - { - "cell_type": "code", - "execution_count": 38, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "# generate data from a d-variate response variane component model\n", - "srand(123)\n", - "n = 1000 # no. observations\n", - "d = 2 # dimension of responses\n", - "m = 2 # no. variance components\n", - "p = 2 # no. covariates\n", - "# n-by-p design matrix\n", - "X = randn(n, p)\n", - "# p-by-d mean component regression coefficient\n", - "B = ones(p, d) \n", - "# a tuple of m covariance matrices\n", - "V = ntuple(x -> zeros(n, n), m) \n", - "for i = 1:m-1\n", - " Vi = randn(n, 50)\n", - " copy!(V[i], Vi * Vi')\n", - "end\n", - "copy!(V[m], eye(n)) # last covarianec matrix is idendity\n", - "# a tuple of m d-by-d variance component parameters\n", - "Σ = ntuple(x -> zeros(d, d), m) \n", - "for i in 1:m\n", - " Σi = randn(d, d)\n", - " copy!(Σ[i], Σi' * Σi)\n", - "end\n", - "# form overall nd-by-nd covariance matrix Ω\n", - "Ω = zeros(n * d, n * d)\n", - "for i = 1:m\n", - " Ω += kron(Σ[i], V[i])\n", - "end\n", - "Ωchol = cholfact(Ω)\n", - "# n-by-d responses\n", - "Y = X * B + reshape(Ωchol[:L] * randn(n*d), n, d);" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Maximum likelihood estimation (MLE)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "To find the MLE of parameters $(B,\\Sigma_1,\\ldots,\\Sigma_m)$, we take 3 steps: \n", - "\n", - "**Step 1 (Construct data)**. Construct an instance of `VarianceComponentVariate`, which consists fields \n", - "\n", - "* `Y`: $n$-by-$d$ responses \n", - "* `X`: $n$-by-$p$ covariate matrix \n", - "* `V=(V[1],...,V[m])`: a tuple of $n$-by-$n$ covariance matrices. The last covariance matrix must be positive definite and usually is the identity matrix. " - ] - }, - { - "cell_type": "code", - "execution_count": 39, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "3-element Array{Symbol,1}:\n", - " :Y\n", - " :X\n", - " :V" - ] - }, - "execution_count": 39, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "using VarianceComponentModels\n", - "vcdata = VarianceComponentVariate(Y, X, V)\n", - "fieldnames(vcdata)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "In the absence of covariates $X$, we can simply initialize by `vcdata = VarianceComponentVariate(Y, V)`." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "**Step 2 (Construct a model)**. Construct an instance of `VarianceComponentModel`, which consists of fields \n", - "\n", - "* `B`: $n$-by-$p$ mean regression coefficients \n", - "* `Σ=(Σ[1],...,Σ[m])`: variane component parameters respectively. \n", - "\n", - "When constructed from a `VarianceComponentVariate` instance, the mean parameters $B$ are initialized to be zero and the tuple of variance component parameters $\\Sigma$ to be `(eye(d),...,eye(d))`." - ] - }, - { - "cell_type": "code", - "execution_count": 40, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "7-element Array{Symbol,1}:\n", - " :B \n", - " :Σ \n", - " :A \n", - " :sense\n", - " :b \n", - " :lb \n", - " :ub " - ] - }, - "execution_count": 40, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "vcmodel = VarianceComponentModel(vcdata)\n", - "fieldnames(vcmodel)" - ] - }, - { - "cell_type": "code", - "execution_count": 41, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "VarianceComponentModels.VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}([0.0 0.0; 0.0 0.0], ([1.0 0.0; 0.0 1.0], [1.0 0.0; 0.0 1.0]), Array{Float64}(0,4), Char[], Float64[], -Inf, Inf)" - ] - }, - "execution_count": 41, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "vcmodel" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The remaining fields `A`, `sense`, `b`, `lb`, `ub` specify (optional) constraints on the mean parameters `B`:\n", - "\n", - "$A * \\text{vec}(B) \\,\\, =(\\text{or } \\ge \\text{or } \\le) \\,\\, b$\n", - "\n", - "$lb \\le \\text{vec}(B) \\le ub$\n", - "\n", - "`A` is an constraint matrix with $pd$ columns, `sense` is a vector of charaters taking values `'<'`, `'='` or `'>'`, and `lb` and `ub` are the lower and upper bounds for `vec(B)`. By default, `A`, `sense`, `b` are empty, `lb` is `-Inf`, and `ub` is `Inf`. If any constraits are non-trivial, final estimates of `B` are enforced to satisfy them." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "When a better initial guess is available, we can initialize by calling `vcmodel=VarianceComponentModel(B0, Σ0)` directly." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "**Step 3 (Fit model)**. Call optmization routine `fit_mle!`. The keywork `algo` dictates the optimization algorithm: `:MM` (minorization-maximization algorithm) or `:FS` (Fisher scoring algorithm)." - ] - }, - { - "cell_type": "code", - "execution_count": 42, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - " MM Algorithm\n", - " Iter Objective \n", - "-------- -------------\n", - " 0 -6.253551e+03\n", - " 1 -3.881454e+03\n", - " 2 -3.853179e+03\n", - " 3 -3.846525e+03\n", - " 4 -3.844906e+03\n", - " 5 -3.844506e+03\n", - " 6 -3.844406e+03\n", - " 7 -3.844381e+03\n", - " 8 -3.844375e+03\n", - " 9 -3.844374e+03\n", - " 10 -3.844373e+03\n", - "\n", - " 0.290970 seconds (10.45 k allocations: 24.036 MiB, 4.73% gc time)\n" - ] - } - ], - "source": [ - "vcmodel_mle = deepcopy(vcmodel)\n", - "@time logl, vcmodel_mle, Σse, Σcov, Bse, Bcov = fit_mle!(vcmodel_mle, vcdata; algo = :MM);" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The output of `fit_mle!` contains \n", - "\n", - "* final log-likelihood " - ] - }, - { - "cell_type": "code", - "execution_count": 43, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "-3844.3731814180805" - ] - }, - "execution_count": 43, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "logl" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "* fitted model" - ] - }, - { - "cell_type": "code", - "execution_count": 44, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "7-element Array{Symbol,1}:\n", - " :B \n", - " :Σ \n", - " :A \n", - " :sense\n", - " :b \n", - " :lb \n", - " :ub " - ] - }, - "execution_count": 44, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "fieldnames(vcmodel_mle)" - ] - }, - { - "cell_type": "code", - "execution_count": 45, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "VarianceComponentModels.VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}([1.092 1.04727; 0.955346 1.01632], ([0.380637 -0.305465; -0.305465 4.51938], [1.84009 0.265569; 0.265569 2.17275]), Array{Float64}(0,4), Char[], Float64[], -Inf, Inf)" - ] - }, - "execution_count": 45, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "vcmodel_mle" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "* standard errors of the estimated varianec component parameters" - ] - }, - { - "cell_type": "code", - "execution_count": 46, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "([0.0765136 0.263047; 0.263047 0.904332], [0.0844292 0.0917441; 0.0917441 0.0996927])" - ] - }, - "execution_count": 46, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "Σse" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "* covariance matrix of the variance component parameters estimates" - ] - }, - { - "cell_type": "code", - "execution_count": 47, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "8×8 Array{Float64,2}:\n", - " 0.00585433 -0.00467019 -0.00467019 … -1.07903e-6 -1.557e-7 \n", - " -0.00467019 0.0691937 0.00372555 -1.557e-7 -1.27444e-6 \n", - " -0.00467019 0.00372555 0.0691937 -8.83212e-6 -1.27444e-6 \n", - " 0.00372555 -0.055198 -0.055198 -1.27444e-6 -1.04316e-5 \n", - " -7.4779e-6 -1.07903e-6 -1.07903e-6 0.00102878 0.000148477\n", - " -1.07903e-6 -8.83212e-6 -1.557e-7 … 0.000148477 0.00121477 \n", - " -1.07903e-6 -1.557e-7 -8.83212e-6 0.00841698 0.00121477 \n", - " -1.557e-7 -1.27444e-6 -1.27444e-6 0.00121477 0.00993864 " - ] - }, - "execution_count": 47, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "Σcov" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "* standard errors of the estimated mean parameters" - ] - }, - { - "cell_type": "code", - "execution_count": 48, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "2×2 Array{Float64,2}:\n", - " 0.042559 0.0487086\n", - " 0.0430588 0.049178 " - ] - }, - "execution_count": 48, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "Bse" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "* covariance matrix of the mean parameter estimates" - ] - }, - { - "cell_type": "code", - "execution_count": 49, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "4×4 Array{Float64,2}:\n", - " 0.00181127 -1.98035e-5 0.000240705 -2.59506e-6 \n", - " -1.98035e-5 0.00185406 -2.59506e-6 0.000247285\n", - " 0.000240705 -2.59506e-6 0.00237252 -2.63542e-5 \n", - " -2.59506e-6 0.000247285 -2.63542e-5 0.00241848 " - ] - }, - "execution_count": 49, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "Bcov" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Restricted maximum likelihood estimation (REML)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "[REML (restricted maximum likelihood estimation)](https://en.wikipedia.org/wiki/Restricted_maximum_likelihood) is a popular alternative to the MLE. To find the REML of a variane component model, we replace the above step 3 by " - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "**Step 3**. Call optmization routine `fit_reml!`." - ] - }, - { - "cell_type": "code", - "execution_count": 50, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - " MM Algorithm\n", - " Iter Objective \n", - "-------- -------------\n", - " 0 -4.215053e+03\n", - " 1 -3.925799e+03\n", - " 2 -3.865114e+03\n", - " 3 -3.851105e+03\n", - " 4 -3.847732e+03\n", - " 5 -3.846903e+03\n", - " 6 -3.846698e+03\n", - " 7 -3.846647e+03\n", - " 8 -3.846634e+03\n", - " 9 -3.846631e+03\n", - " 10 -3.846630e+03\n", - "\n", - " 0" - ] - } - ], - "source": [ - "vcmodel_reml = deepcopy(vcmodel)\n", - "@time logl, vcmodel_reml, Σse, Σcov, Bse, Bcov = fit_reml!(vcmodel_reml, vcdata; algo = :MM);" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The output of `fit_reml!` contains" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "* the final log-likelihood at REML estimate" - ] - }, - { - "cell_type": "code", - "execution_count": 51, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "-3844.3777179025096" - ] - }, - "execution_count": 51, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "logl" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "* REML estimates" - ] - }, - { - "cell_type": "code", - "execution_count": 52, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "7-element Array{Symbol,1}:\n", - " :B \n", - " :Σ \n", - " :A \n", - " :sense\n", - " :b \n", - " :lb \n", - " :ub " - ] - }, - "execution_count": 52, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "fieldnames(vcmodel_reml)" - ] - }, - { - "cell_type": "code", - "execution_count": 53, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "VarianceComponentModels.VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}([1.092 1.04727; 0.955345 1.01632], ([0.380594 -0.305485; -0.305485 4.51994], [1.84285 0.261963; 0.261963 2.17842]), Array{Float64}(0,4), Char[], Float64[], -Inf, Inf)" - ] - }, - "execution_count": 53, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "vcmodel_reml" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "* standard errors of the estimated varianec component parameters" - ] - }, - { - "cell_type": "code", - "execution_count": 54, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "([0.0765055 0.26305; 0.26305 0.904446], [0.0845559 0.0919325; 0.0919325 0.0999526])" - ] - }, - "execution_count": 54, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "Σse" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "* covariance matrix of the variance component parameters estimates" - ] - }, - { - "cell_type": "code", - "execution_count": 55, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "8×8 Array{Float64,2}:\n", - " 0.0058531 -0.00467005 -0.00467005 … -1.06597e-6 -1.51499e-7 \n", - " -0.00467005 0.0691951 0.00372613 -1.51499e-7 -1.26041e-6 \n", - " -0.00467005 0.00372613 0.0691951 -8.86843e-6 -1.26041e-6 \n", - " 0.00372613 -0.0552092 -0.0552092 -1.26041e-6 -1.0486e-5 \n", - " -7.50035e-6 -1.06597e-6 -1.06597e-6 0.00101633 0.000144472\n", - " -1.06597e-6 -8.86843e-6 -1.51499e-7 … 0.000144472 0.0012014 \n", - " -1.06597e-6 -1.51499e-7 -8.86843e-6 0.00845158 0.0012014 \n", - " -1.51499e-7 -1.26041e-6 -1.26041e-6 0.0012014 0.00999052 " - ] - }, - "execution_count": 55, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "Σcov" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "* standard errors of the estimated mean parameters" - ] - }, - { - "cell_type": "code", - "execution_count": 56, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "2×2 Array{Float64,2}:\n", - " 0.0425909 0.0487744\n", - " 0.043091 0.0492444" - ] - }, - "execution_count": 56, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "Bse" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "* covariance matrix of the mean parameter estimates" - ] - }, - { - "cell_type": "code", - "execution_count": 57, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "4×4 Array{Float64,2}:\n", - " 0.00181398 -1.98331e-5 0.000237127 -2.55589e-6 \n", - " -1.98331e-5 0.00185683 -2.55589e-6 0.000243624\n", - " 0.000237127 -2.55589e-6 0.00237894 -2.6426e-5 \n", - " -2.55589e-6 0.000243624 -2.6426e-5 0.00242501 " - ] - }, - "execution_count": 57, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "Bcov" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Optimization algorithms" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Finding the MLE or REML of variance component models is a non-trivial nonlinear optimization problem. The main complications are the non-convexity of objective function and the positive semi-definiteness constraint of variane component parameters $\\Sigma_1,\\ldots,\\Sigma_m$. In specific applications, users should try different algorithms with different starting points in order to find a better solution. Here are some tips for efficient computation. " - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "In general the optimization algorithm needs to invert the $nd$ by $nd$ overall covariance matrix $\\Omega = \\Sigma_1 \\otimes V_1 + \\cdots + \\Sigma_m \\otimes V_m$ in each iteration. Inverting a matrix is an expensive operation with $O(n^3 d^3)$ floating operations. When there are only **two** varianec components ($m=2$), this tedious task can be avoided by taking one (generalized) eigendecomposion of $(V_1, V_2)$ and rotating data $(Y, X)$ by the eigen-vectors. " - ] - }, - { - "cell_type": "code", - "execution_count": 58, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "5-element Array{Symbol,1}:\n", - " :Yrot \n", - " :Xrot \n", - " :eigval \n", - " :eigvec \n", - " :logdetV2" - ] - }, - "execution_count": 58, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "vcdatarot = TwoVarCompVariateRotate(vcdata)\n", - "fieldnames(vcdatarot)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Two optimization algorithms are implemented: [Fisher scoring](https://books.google.com/books?id=QYqeYTftPNwC&lpg=PP1&pg=PA142#v=onepage&q&f=false) (`mle_fs!`) and the [minorization-maximization (MM) algorithm](http://arxiv.org/abs/1509.07426) (`mle_mm!`). Both take the rotated data as input. These two functions give finer control of the optimization algorithms. Generally speaking, MM algorithm is more stable while Fisher scoring (if it converges) yields more accurate answer." - ] - }, - { - "cell_type": "code", - "execution_count": 59, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - " MM Algorithm\n", - " Iter Objective \n", - "-------- -------------\n", - " 0 -6.253551e+03\n", - " 1 -3.881454e+03\n", - " 2 -3.853179e+03\n", - " 3 -3.846525e+03\n", - " 4 -3.844906e+03\n", - " 5 -3.844506e+03\n", - " 6 -3.844406e+03\n", - " 7 -3.844381e+03\n", - " 8 -3.844375e+03\n", - " 9 -3.844374e+03\n", - " 10 -3.844373e+03\n", - "\n", - " 0.018754 seconds (9.15 k allocations: 680.172 KiB)\n" - ] - } - ], - "source": [ - "vcmodel_mm = deepcopy(vcmodel)\n", - "@time mle_mm!(vcmodel_mm, vcdatarot; maxiter=10000, funtol=1e-8, verbose = true);" - ] - }, - { - "cell_type": "code", - "execution_count": 60, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "2×2 Array{Float64,2}:\n", - " 1.092 1.04727\n", - " 0.955346 1.01632" - ] - }, - "execution_count": 60, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# MM estimates\n", - "vcmodel_mm.B" - ] - }, - { - "cell_type": "code", - "execution_count": 61, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "([0.380637 -0.305465; -0.305465 4.51938], [1.84009 0.265569; 0.265569 2.17275])" - ] - }, - "execution_count": 61, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# MM estimates\n", - "vcmodel_mm.Σ" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Fisher scoring (`mle_fs!`) uses either [Ipopt.jl](https://github.com/JuliaOpt/Ipopt.jl) (keyword `solver=:Ipopt`) or [KNITRO.jl](https://github.com/JuliaOpt/KNITRO.jl) (keyword `solver=:Knitro`) as the backend solver. Ipopt is open source and installation of [Ipopt.jl](https://github.com/JuliaOpt/Ipopt.jl) package alone is sufficient." - ] - }, - { - "cell_type": "code", - "execution_count": 62, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "This is Ipopt version 3.12.4, running with linear solver mumps.\n", - "NOTE: Other linear solvers might be more efficient (see Ipopt documentation).\n", - "\n", - "Number of nonzeros in equality constraint Jacobian...: 0\n", - "Number of nonzeros in inequality constraint Jacobian.: 0\n", - "Number of nonzeros in Lagrangian Hessian.............: 21\n", - "\n", - "Total number of variables............................: 6\n", - " variables with only lower bounds: 0\n", - " variables with lower and upper bounds: 0\n", - " variables with only upper bounds: 0\n", - "Total number of equality constraints.................: 0\n", - "Total number of inequality constraints...............: 0\n", - " inequality constraints with only lower bounds: 0\n", - " inequality constraints with lower and upper bounds: 0\n", - " inequality constraints with only upper bounds: 0\n", - "\n", - "iter objective inf_pr inf_du lg(mu) ||d|| lg(rg) alpha_du alpha_pr ls\n", - " 0 4.2109423e+03 0.00e+00 1.00e+02 0.0 0.00e+00 - 0.00e+00 0.00e+00 0 \n", - " 5 3.8445586e+03 0.00e+00 7.87e-01 -11.0 4.94e-02 - 1.00e+00 1.00e+00f 1 MaxS\n", - " 10 3.8443870e+03 0.00e+00 2.25e-01 -11.0 1.38e-02 - 1.00e+00 1.00e+00f 1 MaxS\n", - " 15 3.8443742e+03 0.00e+00 6.23e-02 -11.0 3.78e-03 - 1.00e+00 1.00e+00f 1 MaxS\n", - " 20 3.8443733e+03 0.00e+00 1.70e-02 -11.0 1.03e-03 - 1.00e+00 1.00e+00f 1 MaxS\n", - " 25 3.8443732e+03 0.00e+00 4.61e-03 -11.0 2.79e-04 - 1.00e+00 1.00e+00f 1 MaxS\n", - " 30 3.8443732e+03 0.00e+00 1.25e-03 -11.0 7.56e-05 - 1.00e+00 1.00e+00f 1 MaxS\n", - " 35 3.8443732e+03 0.00e+00 3.39e-04 -11.0 2.05e-05 - 1.00e+00 1.00e+00f 1 MaxS\n", - " 40 3.8443732e+03 0.00e+00 9.19e-05 -11.0 5.55e-06 - 1.00e+00 1.00e+00f 1 MaxS\n", - " 45 3.8443732e+03 0.00e+00 2.49e-05 -11.0 1.51e-06 - 1.00e+00 1.00e+00f 1 MaxS\n", - "iter objective inf_pr inf_du lg(mu) ||d|| lg(rg) alpha_du alpha_pr ls\n", - " 50 3.8443732e+03 0.00e+00 6.76e-06 -11.0 4.08e-07 - 1.00e+00 1.00e+00f 1 MaxSA\n", - " 55 3.8443732e+03 0.00e+00 1.83e-06 -11.0 1.11e-07 - 1.00e+00 1.00e+00f 1 MaxSA\n", - " 60 3.8443732e+03 0.00e+00 4.97e-07 -11.0 3.00e-08 - 1.00e+00 1.00e+00f 1 MaxSA\n", - "\n", - "Number of Iterations....: 63\n", - "\n", - " (scaled) (unscaled)\n", - "Objective...............: 3.4496886481728075e+02 3.8443731733053696e+03\n", - "Dual infeasibility......: 2.2693631692678575e-07 2.5290047242499938e-06\n", - "Constraint violation....: 0.0000000000000000e+00 0.0000000000000000e+00\n", - "Complementarity.........: 0.0000000000000000e+00 0.0000000000000000e+00\n", - "Overall NLP error.......: 2.2693631692678575e-07 2.5290047242499938e-06\n", - "\n", - "\n", - "Number of objective function evaluations = 64\n", - "Number of objective gradient evaluations = 64\n", - "Number of equality constraint evaluations = 0\n", - "Number of inequality constraint evaluations = 0\n", - "Number of equality constraint Jacobian evaluations = 0\n", - "Number of inequality constraint Jacobian evaluations = 0\n", - "Number of Lagrangian Hessian evaluations = 63\n", - "Total CPU secs in IPOPT (w/o function evaluations) = 0.020\n", - "Total CPU secs in NLP function evaluations = 0.256\n", - "\n", - "EXIT: Solved To Acceptable Level.\n", - " " - ] - } - ], - "source": [ - "# Fisher scoring using Ipopt\n", - "vcmodel_ipopt = deepcopy(vcmodel)\n", - "@time mle_fs!(vcmodel_ipopt, vcdatarot; solver=:Ipopt, maxiter=1000, verbose=true);" - ] - }, - { - "cell_type": "code", - "execution_count": 63, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "2×2 Array{Float64,2}:\n", - " 1.092 1.04727\n", - " 0.955346 1.01632" - ] - }, - "execution_count": 63, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# Ipopt estimates\n", - "vcmodel_ipopt.B" - ] - }, - { - "cell_type": "code", - "execution_count": 64, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "([0.380552 -0.305594; -0.305594 4.52106], [1.84008 0.265385; 0.265385 2.17287])" - ] - }, - "execution_count": 64, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# Ipopt estimates\n", - "vcmodel_ipopt.Σ" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Knitro is a commercial software and users need to follow instructions at [KNITRO.jl](https://github.com/JuliaOpt/KNITRO.jl) for proper functioning. Following code invokes Knitro as the backend optimization solver.\n", - "```julia\n", - "using KNITRO\n", - "\n", - "# Fisher scoring using Knitro\n", - "vcmodel_knitro = deepcopy(vcmodel)\n", - "@time mle_fs!(vcmodel_knitro, vcdatarot; solver=:Knitro, maxiter=1000, verbose=true);\n", - "\n", - "# Knitro estimates\n", - "vcmodel_knitro.B\n", - "\n", - "# Knitro estimates\n", - "vcmodel_knitro.Σ\n", - "```" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Starting point" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Here are a few strategies for successful optimization. " - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "* For $d>1$ (multivariate response), initialize $B, \\Sigma$ from univariate estimates. \n", - "* Use REML estimate as starting point for MLE. \n", - "* When there are only $m=2$ variance components, pre-compute `TwoVarCompVariateRotate` and use it for optimization." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "collapsed": true - }, - "source": [ - "## Constrained estimation of `B`" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Many applications invoke constraints on the mean parameters `B`. For demonstration, we enforce `B[1,1]=B[1,2]` and all entries of `B` are within [0, 2]." - ] - }, - { - "cell_type": "code", - "execution_count": 65, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "VarianceComponentModels.VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}([0.0 0.0; 0.0 0.0], ([1.0 0.0; 0.0 1.0], [1.0 0.0; 0.0 1.0]), [1.0 0.0 -1.0 0.0], '=', 0.0, 0.0, 2.0)" - ] - }, - "execution_count": 65, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# set up constraints on B\n", - "vcmodel_constr = deepcopy(vcmodel)\n", - "vcmodel_constr.A = [1.0 0.0 -1.0 0.0]\n", - "vcmodel_constr.sense = '='\n", - "vcmodel_constr.b = 0.0\n", - "vcmodel_constr.lb = 0.0\n", - "vcmodel_constr.ub = 2.0\n", - "vcmodel_constr" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We first try the MM algorithm." - ] - }, - { - "cell_type": "code", - "execution_count": 66, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - " MM Algorithm\n", - " Iter Objective \n", - "-------- -------------\n", - " 0 -6.253551e+03\n", - " 1 -3.881820e+03\n", - " 2 -3.853477e+03\n", - " 3 -3.846807e+03\n", - " 4 -3.845184e+03\n", - " 5 -3.844783e+03\n", - " 6 -3.844683e+03\n", - " 7 -3.844658e+03\n", - " 8 -3.844652e+03\n", - " 9 -3.844650e+03\n", - " 10 -3.844650e+03\n", - "\n", - " 0.031954 seconds (10.70 k allocations: 781.828 KiB)\n" - ] - } - ], - "source": [ - "# MM algorithm for constrained estimation of B\n", - "@time mle_mm!(vcmodel_constr, vcdatarot; maxiter=10000, funtol=1e-8, verbose = true);" - ] - }, - { - "cell_type": "code", - "execution_count": 67, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "7-element Array{Symbol,1}:\n", - " :B \n", - " :Σ \n", - " :A \n", - " :sense\n", - " :b \n", - " :lb \n", - " :ub " - ] - }, - "execution_count": 67, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "fieldnames(vcmodel_constr)" - ] - }, - { - "cell_type": "code", - "execution_count": 68, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "2×2 Array{Float64,2}:\n", - " 1.07177 1.07177\n", - " 0.955683 1.01591" - ] - }, - "execution_count": 68, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "vcmodel_constr.B" - ] - }, - { - "cell_type": "code", - "execution_count": 69, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "([0.380624 -0.305498; -0.305498 4.51948], [1.84051 0.265065; 0.265065 2.17336])" - ] - }, - "execution_count": 69, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "vcmodel_constr.Σ" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now let's try Fisher scoring." - ] - }, - { - "cell_type": "code", - "execution_count": 70, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "This is Ipopt version 3.12.4, running with linear solver mumps.\n", - "NOTE: Other linear solvers might be more efficient (see Ipopt documentation).\n", - "\n", - "Number of nonzeros in equality constraint Jacobian...: 0\n", - "Number of nonzeros in inequality constraint Jacobian.: 0\n", - "Number of nonzeros in Lagrangian Hessian.............: 21\n", - "\n", - "Total number of variables............................: 6\n", - " variables with only lower bounds: 0\n", - " variables with lower and upper bounds: 0\n", - " variables with only upper bounds: 0\n", - "Total number of equality constraints.................: 0\n", - "Total number of inequality constraints...............: 0\n", - " inequality constraints with only lower bounds: 0\n", - " inequality constraints with lower and upper bounds: 0\n", - " inequality constraints with only upper bounds: 0\n", - "\n", - "iter objective inf_pr inf_du lg(mu) ||d|| lg(rg) alpha_du alpha_pr ls\n", - " 0 4.2114270e+03 0.00e+00 1.00e+02 0.0 0.00e+00 - 0.00e+00 0.00e+00 0 \n", - " 5 3.8448353e+03 0.00e+00 7.87e-01 -11.0 4.94e-02 - 1.00e+00 1.00e+00f 1 MaxS\n", - " 10 3.8446636e+03 0.00e+00 2.25e-01 -11.0 1.38e-02 - 1.00e+00 1.00e+00f 1 MaxS\n", - " 15 3.8446509e+03 0.00e+00 6.23e-02 -11.0 3.78e-03 - 1.00e+00 1.00e+00f 1 MaxS\n", - " 20 3.8446499e+03 0.00e+00 1.70e-02 -11.0 1.03e-03 - 1.00e+00 1.00e+00f 1 MaxS\n", - " 25 3.8446498e+03 0.00e+00 4.61e-03 -11.0 2.79e-04 - 1.00e+00 1.00e+00f 1 MaxS\n", - " 30 3.8446498e+03 0.00e+00 1.25e-03 -11.0 7.56e-05 - 1.00e+00 1.00e+00f 1 MaxS\n", - " 35 3.8446498e+03 0.00e+00 3.39e-04 -11.0 2.05e-05 - 1.00e+00 1.00e+00f 1 MaxS\n", - " 40 3.8446498e+03 0.00e+00 9.19e-05 -11.0 5.56e-06 - 1.00e+00 1.00e+00f 1 MaxS\n", - " 45 3.8446498e+03 0.00e+00 2.49e-05 -11.0 1.51e-06 - 1.00e+00 1.00e+00f 1 MaxS\n", - "iter objective inf_pr inf_du lg(mu) ||d|| lg(rg) alpha_du alpha_pr ls\n", - " 50 3.8446498e+03 0.00e+00 6.76e-06 -11.0 4.08e-07 - 1.00e+00 1.00e+00f 1 MaxSA\n", - " 55 3.8446498e+03 0.00e+00 1.83e-06 -11.0 1.11e-07 - 1.00e+00 1.00e+00f 1 MaxSA\n", - " 60 3.8446498e+03 0.00e+00 4.97e-07 -11.0 3.00e-08 - 1.00e+00 1.00e+00f 1 MaxSA\n", - "\n", - "Number of Iterations....: 63\n", - "\n", - " (scaled) (unscaled)\n", - "Objective...............: 3.4484507551949008e+02 3.8446498170293403e+03\n", - "Dual infeasibility......: 2.2694405349430929e-07 2.5301808715939735e-06\n", - "Constraint violation....: 0.0000000000000000e+00 0.0000000000000000e+00\n", - "Complementarity.........: 0.0000000000000000e+00 0.0000000000000000e+00\n", - "Overall NLP error.......: 2.2694405349430929e-07 2.5301808715939735e-06\n", - "\n", - "\n", - "Number of objective function evaluations = 64\n", - "Number of objective gradient evaluations = 64\n", - "Number of equality constraint evaluations = 0\n", - "Number of inequality constraint evaluations = 0\n", - "Number of equality constraint Jacobian evaluations = 0\n", - "Number of inequality constraint Jacobian evaluations = 0\n", - "Number of Lagrangian Hessian evaluations = 63\n", - "Total CPU secs in IPOPT (w/o function evaluations) = 0.016\n", - "Total CPU secs in NLP function evaluations = 0.417\n", - "\n", - "EXIT: Solved To Acceptable Level.\n", - " " - ] - } - ], - "source": [ - "# Fisher scoring using Ipopt for constrained estimation of B\n", - "vcmodel_constr = deepcopy(vcmodel)\n", - "vcmodel_constr.A = [1.0 0.0 -1.0 0.0]\n", - "vcmodel_constr.sense = '='\n", - "vcmodel_constr.b = 0.0\n", - "vcmodel_constr.lb = 0.0\n", - "vcmodel_constr.ub = 2.0\n", - "vcmodel_constr\n", - "@time mle_fs!(vcmodel_constr, vcdatarot; solver=:Ipopt, maxiter=1000, verbose=true);" - ] - }, - { - "cell_type": "code", - "execution_count": 71, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "2×2 Array{Float64,2}:\n", - " 1.07177 1.07177\n", - " 0.955683 1.01591" - ] - }, - "execution_count": 71, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "vcmodel_constr.B" - ] - }, - { - "cell_type": "code", - "execution_count": 72, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "([0.380539 -0.305626; -0.305626 4.52116], [1.8405 0.264881; 0.264881 2.17348])" - ] - }, - "execution_count": 72, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "vcmodel_constr.Σ" - ] - } - ], - "metadata": { - "anaconda-cloud": {}, - "kernelspec": { - "display_name": "Julia 0.6.0", - "language": "julia", - "name": "julia-0.6" - }, - "language_info": { - "file_extension": ".jl", - "mimetype": "application/julia", - "name": "julia", - "version": "0.6.0" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} From 9f34a7307e9a8c745b5f64e71bbb93605911b80e Mon Sep 17 00:00:00 2001 From: Juhyun Kim Date: Tue, 29 Jan 2019 18:24:29 -0800 Subject: [PATCH 48/56] update save analysis results chunk --- docs/heritability_v0.7.ipynb | 127 ++++++++++++++++++++++++++++++++--- 1 file changed, 119 insertions(+), 8 deletions(-) diff --git a/docs/heritability_v0.7.ipynb b/docs/heritability_v0.7.ipynb index 6a821c1..d0aa5c7 100644 --- a/docs/heritability_v0.7.ipynb +++ b/docs/heritability_v0.7.ipynb @@ -1830,15 +1830,126 @@ }, { "cell_type": "code", - "execution_count": 11, - "metadata": { - "collapsed": true - }, - "outputs": [], + "execution_count": 95, + "metadata": {}, + "outputs": [ + { + "data": { + "text/latex": [ + "\\begin{tabular}\n", + "{l | r | l}\n", + "name & size & summary \\\\\n", + "\\hline\n", + "Base & & Module \\\\\n", + "Core & & Module \\\\\n", + "Main & & Module \\\\\n", + "Plots & 22.448 MiB & Module \\\\\n", + "Y & 677.461 KiB & 6670×13 Array\\{Float64,2\\} \\\\\n", + "\\_ & 240 bytes & Tuple\\{Array\\{Float64,2\\},Array\\{Float64,2\\}\\} \\\\\n", + "cg10k & 1022.983 MiB & 6670×630860 SnpArray \\\\\n", + "cg10k\\_trait & 1.443 MiB & 6670×15 DataFrame \\\\\n", + "cg10kdata & 679.508 MiB & VarianceComponentVariate\\{Float64,2,Array\\{Float64,2\\},Array\\{Float64,2\\},Array\\{Float64,2\\}\\} \\\\\n", + "cg10kdata\\_rotated & 340.136 MiB & TwoVarCompVariateRotate\\{Float64,Array\\{Float64,2\\},Array\\{Float64,2\\}\\} \\\\\n", + "h & 64 bytes & 3-element Array\\{Float64,1\\} \\\\\n", + "hST & 144 bytes & 13-element Array\\{Float64,1\\} \\\\\n", + "hST\\_se & 144 bytes & 13-element Array\\{Float64,1\\} \\\\\n", + "hse & 64 bytes & 3-element Array\\{Float64,1\\} \\\\\n", + "maf\\_cg10k & 4.813 MiB & 630860-element Array\\{Float64,1\\} \\\\\n", + "missings\\_by\\_snp & 4.813 MiB & 1×630860 Array\\{Int64,2\\} \\\\\n", + "mp & 28.748 MiB & 6670×630860 SparseMatrixCSC\\{Bool,Int32\\} \\\\\n", + "people & 8 bytes & Int64 \\\\\n", + "snps & 8 bytes & Int64 \\\\\n", + "startupfile & 57 bytes & String \\\\\n", + "trait57\\_data & 339.627 MiB & TwoVarCompVariateRotate\\{Float64,Array\\{Float64,2\\},Array\\{Float64,2\\}\\} \\\\\n", + "trait57\\_model & 472 bytes & VarianceComponentModel\\{Float64,2,Array\\{Float64,2\\},Array\\{Float64,2\\}\\} \\\\\n", + "traitall\\_model & 2.961 KiB & VarianceComponentModel\\{Float64,2,Array\\{Float64,2\\},Array\\{Float64,2\\}\\} \\\\\n", + "traitidx & 16 bytes & 3-element UnitRange\\{Int64\\} \\\\\n", + "Σa & 6.844 KiB & 13×13 Array\\{Array\\{Float64,2\\},2\\} \\\\\n", + "Σcov & 2.570 KiB & 18×18 Array\\{Float64,2\\} \\\\\n", + "Σe & 6.844 KiB & 13×13 Array\\{Array\\{Float64,2\\},2\\} \\\\\n", + "Φgrm & 339.423 MiB & 6670×6670 Array\\{Float64,2\\} \\\\\n", + "σ2a & 144 bytes & 13-element Array\\{Float64,1\\} \\\\\n", + "σ2e & 144 bytes & 13-element Array\\{Float64,1\\} \\\\\n", + "\\end{tabular}\n" + ], + "text/markdown": [ + "| name | size | summary |\n", + "|:----------------- | ------------:|:-------------------------------------------------------------------------------------- |\n", + "| Base | | Module |\n", + "| Core | | Module |\n", + "| Main | | Module |\n", + "| Plots | 22.448 MiB | Module |\n", + "| Y | 677.461 KiB | 6670×13 Array{Float64,2} |\n", + "| _ | 240 bytes | Tuple{Array{Float64,2},Array{Float64,2}} |\n", + "| cg10k | 1022.983 MiB | 6670×630860 SnpArray |\n", + "| cg10k_trait | 1.443 MiB | 6670×15 DataFrame |\n", + "| cg10kdata | 679.508 MiB | VarianceComponentVariate{Float64,2,Array{Float64,2},Array{Float64,2},Array{Float64,2}} |\n", + "| cg10kdata_rotated | 340.136 MiB | TwoVarCompVariateRotate{Float64,Array{Float64,2},Array{Float64,2}} |\n", + "| h | 64 bytes | 3-element Array{Float64,1} |\n", + "| hST | 144 bytes | 13-element Array{Float64,1} |\n", + "| hST_se | 144 bytes | 13-element Array{Float64,1} |\n", + "| hse | 64 bytes | 3-element Array{Float64,1} |\n", + "| maf_cg10k | 4.813 MiB | 630860-element Array{Float64,1} |\n", + "| missings_by_snp | 4.813 MiB | 1×630860 Array{Int64,2} |\n", + "| mp | 28.748 MiB | 6670×630860 SparseMatrixCSC{Bool,Int32} |\n", + "| people | 8 bytes | Int64 |\n", + "| snps | 8 bytes | Int64 |\n", + "| startupfile | 57 bytes | String |\n", + "| trait57_data | 339.627 MiB | TwoVarCompVariateRotate{Float64,Array{Float64,2},Array{Float64,2}} |\n", + "| trait57_model | 472 bytes | VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}} |\n", + "| traitall_model | 2.961 KiB | VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}} |\n", + "| traitidx | 16 bytes | 3-element UnitRange{Int64} |\n", + "| Σa | 6.844 KiB | 13×13 Array{Array{Float64,2},2} |\n", + "| Σcov | 2.570 KiB | 18×18 Array{Float64,2} |\n", + "| Σe | 6.844 KiB | 13×13 Array{Array{Float64,2},2} |\n", + "| Φgrm | 339.423 MiB | 6670×6670 Array{Float64,2} |\n", + "| σ2a | 144 bytes | 13-element Array{Float64,1} |\n", + "| σ2e | 144 bytes | 13-element Array{Float64,1} |\n" + ], + "text/plain": [ + "name size summary \n", + "––––––––––––––––– –––––––––––– ––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––\n", + "Base Module \n", + "Core Module \n", + "Main Module \n", + "Plots 22.448 MiB Module \n", + "Y 677.461 KiB 6670×13 Array{Float64,2} \n", + "_ 240 bytes Tuple{Array{Float64,2},Array{Float64,2}} \n", + "cg10k 1022.983 MiB 6670×630860 SnpArray \n", + "cg10k_trait 1.443 MiB 6670×15 DataFrame \n", + "cg10kdata 679.508 MiB VarianceComponentVariate{Float64,2,Array{Float64,2},Array{Float64,2},Array{Float64,2}}\n", + "cg10kdata_rotated 340.136 MiB TwoVarCompVariateRotate{Float64,Array{Float64,2},Array{Float64,2}} \n", + "h 64 bytes 3-element Array{Float64,1} \n", + "hST 144 bytes 13-element Array{Float64,1} \n", + "hST_se 144 bytes 13-element Array{Float64,1} \n", + "hse 64 bytes 3-element Array{Float64,1} \n", + "maf_cg10k 4.813 MiB 630860-element Array{Float64,1} \n", + "missings_by_snp 4.813 MiB 1×630860 Array{Int64,2} \n", + "mp 28.748 MiB 6670×630860 SparseMatrixCSC{Bool,Int32} \n", + "people 8 bytes Int64 \n", + "snps 8 bytes Int64 \n", + "startupfile 57 bytes String \n", + "trait57_data 339.627 MiB TwoVarCompVariateRotate{Float64,Array{Float64,2},Array{Float64,2}} \n", + "trait57_model 472 bytes VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}} \n", + "traitall_model 2.961 KiB VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}} \n", + "traitidx 16 bytes 3-element UnitRange{Int64} \n", + "Σa 6.844 KiB 13×13 Array{Array{Float64,2},2} \n", + "Σcov 2.570 KiB 18×18 Array{Float64,2} \n", + "Σe 6.844 KiB 13×13 Array{Array{Float64,2},2} \n", + "Φgrm 339.423 MiB 6670×6670 Array{Float64,2} \n", + "σ2a 144 bytes 13-element Array{Float64,1} \n", + "σ2e 144 bytes 13-element Array{Float64,1} " + ] + }, + "execution_count": 95, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "#using JLD\n", - "#@save \"copd.jld\"\n", - "#whos()" + "#using JLD2, FileIO\n", + "#save(\"copd.jld2\")\n", + "#varinfo()" ] } ], From 8a310f8404342d44d6175285509ef6a4da272619 Mon Sep 17 00:00:00 2001 From: Juhyun Kim Date: Tue, 29 Jan 2019 18:25:50 -0800 Subject: [PATCH 49/56] rename .ipynb files --- docs/{heritability_v0.7.ipynb => heritability.ipynb} | 0 docs/{mle_reml_v0.7.ipynb => mle_reml.ipynb} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename docs/{heritability_v0.7.ipynb => heritability.ipynb} (100%) rename docs/{mle_reml_v0.7.ipynb => mle_reml.ipynb} (100%) diff --git a/docs/heritability_v0.7.ipynb b/docs/heritability.ipynb similarity index 100% rename from docs/heritability_v0.7.ipynb rename to docs/heritability.ipynb diff --git a/docs/mle_reml_v0.7.ipynb b/docs/mle_reml.ipynb similarity index 100% rename from docs/mle_reml_v0.7.ipynb rename to docs/mle_reml.ipynb From d9cedc25ae20857584d23229a885111b7258a543 Mon Sep 17 00:00:00 2001 From: Juhyun Kim Date: Tue, 29 Jan 2019 18:40:30 -0800 Subject: [PATCH 50/56] update md files to v0.7 --- docs/src/man/heritability.md | 1444 +++++++++++++-------------- docs/src/man/heritability_hist1.png | Bin 0 -> 12699 bytes docs/src/man/heritability_hist2.png | Bin 0 -> 44302 bytes docs/src/man/mle_reml.md | 170 ++-- 4 files changed, 755 insertions(+), 859 deletions(-) create mode 100644 docs/src/man/heritability_hist1.png create mode 100644 docs/src/man/heritability_hist2.png diff --git a/docs/src/man/heritability.md b/docs/src/man/heritability.md index bf6842d..d53af40 100644 --- a/docs/src/man/heritability.md +++ b/docs/src/man/heritability.md @@ -21,25 +21,23 @@ Machine information: versioninfo() ``` - Julia Version 0.6.0 - Commit 903644385b (2017-06-19 13:05 UTC) + Julia Version 0.7.0 + Commit a4cb80f3ed (2018-08-08 06:46 UTC) Platform Info: - OS: macOS (x86_64-apple-darwin13.4.0) - CPU: Intel(R) Core(TM) i7-4790K CPU @ 4.00GHz + OS: macOS (x86_64-apple-darwin14.5.0) + CPU: Intel(R) Core(TM) i5-6267U CPU @ 2.90GHz WORD_SIZE: 64 - BLAS: libopenblas (USE64BITINT DYNAMIC_ARCH NO_AFFINITY Haswell) - LAPACK: libopenblas64_ LIBM: libopenlibm - LLVM: libLLVM-3.9.1 (ORCJIT, haswell) + LLVM: libLLVM-6.0.0 (ORCJIT, skylake) ## Read in binary SNP data -We will use the [`SnpArrays.jl`](https://github.com/OpenMendel/SnpArrays.jl) package to read in binary SNP data and compute the empirical kinship matrix. Issue +We will use the [`SnpArrays.jl`](https://github.com/OpenMendel/SnpArrays.jl) package to read in binary SNP data and compute the empirical kinship matrix. The package has not yet been registered and must be installed using the repository location. Start julia and use the `]` key to switch to the package manager REPL ```julia -Pkg.clone("https://github.com/OpenMendel/SnpArrays.jl.git") +(v0.7) pkg> add https://github.com/OpenMendel/SnpArrays.jl.git#juliav0.7 ``` -within `Julia` to install the `SnpArrays` package. +Use the backspace key to return to the Julia REPL. ```julia @@ -49,42 +47,42 @@ using SnpArrays ```julia # read in genotype data from Plink binary file (~50 secs on my laptop) -@time cg10k = SnpArray("cg10k") +@time cg10k = SnpArray("cg10k.bed") ``` - 22.902730 seconds (51.62 k allocations: 1005.845 MiB, 0.11% gc time) + 0.080738 seconds (70.17 k allocations: 23.096 MiB, 12.02% gc time) - 6670×630860 SnpArrays.SnpArray{2}: - (false, true) (false, true) … (true, true) (true, true) - (true, true) (true, true) (false, true) (true, false) - (true, true) (true, true) (true, true) (true, true) - (true, true) (true, true) (false, true) (true, true) - (true, true) (true, true) (true, true) (false, true) - (false, true) (false, true) … (true, true) (true, true) - (false, false) (false, false) (true, true) (true, true) - (true, true) (true, true) (true, true) (false, true) - (true, true) (true, true) (true, true) (true, true) - (true, true) (true, true) (false, true) (true, true) - (true, true) (true, true) … (true, true) (true, true) - (false, true) (false, true) (true, true) (false, true) - (true, true) (true, true) (true, true) (false, true) - ⋮ ⋱ - (false, true) (false, true) (false, true) (false, true) - (false, true) (false, true) (false, true) (true, true) - (true, true) (true, true) … (false, true) (true, true) - (false, true) (false, true) (true, true) (false, true) - (true, true) (true, true) (false, true) (true, true) - (true, true) (true, true) (false, false) (false, true) - (true, true) (true, true) (true, true) (false, true) - (true, true) (true, true) … (true, true) (true, true) - (true, true) (true, true) (false, true) (true, true) - (true, true) (true, true) (true, true) (false, true) - (false, true) (false, true) (true, true) (true, true) - (true, true) (true, true) (true, true) (true, true) + 6670×630860 SnpArray: + 0x02 0x02 0x03 0x03 0x02 0x03 … 0x02 0x03 0x03 0x03 0x03 0x03 + 0x03 0x03 0x02 0x03 0x02 0x03 0x03 0x02 0x02 0x03 0x02 0x01 + 0x03 0x03 0x02 0x03 0x02 0x03 0x03 0x03 0x03 0x03 0x03 0x03 + 0x03 0x03 0x03 0x03 0x03 0x03 0x03 0x02 0x02 0x02 0x02 0x03 + 0x03 0x03 0x03 0x03 0x03 0x03 0x03 0x03 0x03 0x03 0x03 0x02 + 0x02 0x02 0x03 0x03 0x02 0x03 … 0x03 0x03 0x03 0x03 0x03 0x03 + 0x00 0x00 0x03 0x03 0x00 0x03 0x03 0x02 0x03 0x03 0x03 0x03 + 0x03 0x03 0x03 0x03 0x03 0x02 0x03 0x03 0x03 0x03 0x03 0x02 + 0x03 0x03 0x02 0x03 0x02 0x03 0x02 0x02 0x03 0x03 0x03 0x03 + 0x03 0x03 0x03 0x00 0x03 0x03 0x02 0x02 0x03 0x03 0x02 0x03 + 0x03 0x03 0x02 0x03 0x02 0x02 … 0x02 0x03 0x03 0x03 0x03 0x03 + 0x02 0x02 0x03 0x03 0x02 0x03 0x02 0x03 0x03 0x03 0x03 0x02 + 0x03 0x03 0x03 0x00 0x03 0x03 0x03 0x03 0x03 0x03 0x03 0x02 + ⋮ ⋮ ⋱ ⋮ + 0x02 0x02 0x03 0x03 0x02 0x03 0x03 0x02 0x03 0x03 0x02 0x02 + 0x02 0x02 0x02 0x03 0x00 0x02 0x02 0x03 0x03 0x03 0x02 0x03 + 0x03 0x03 0x02 0x02 0x02 0x03 … 0x03 0x02 0x03 0x03 0x02 0x03 + 0x02 0x02 0x03 0x03 0x02 0x03 0x03 0x02 0x03 0x03 0x03 0x02 + 0x03 0x03 0x03 0x02 0x03 0x02 0x03 0x03 0x03 0x03 0x02 0x03 + 0x03 0x03 0x01 0x02 0x02 0x01 0x03 0x00 0x00 0x02 0x00 0x02 + 0x03 0x03 0x03 0x02 0x03 0x03 0x03 0x03 0x03 0x03 0x03 0x02 + 0x03 0x03 0x02 0x02 0x02 0x03 … 0x00 0x03 0x03 0x03 0x03 0x03 + 0x03 0x03 0x03 0x02 0x02 0x03 0x02 0x02 0x02 0x02 0x02 0x03 + 0x03 0x03 0x03 0x02 0x03 0x02 0x02 0x03 0x02 0x03 0x03 0x02 + 0x02 0x02 0x03 0x03 0x02 0x03 0x03 0x03 0x01 0x03 0x03 0x03 + 0x03 0x03 0x03 0x03 0x03 0x02 0x03 0x00 0x03 0x03 0x03 0x03 @@ -102,18 +100,105 @@ people, snps = size(cg10k) +The positions of the missing data are evaluated by + ```julia -# summary statistics (~50 secs on my laptop) -@time maf, _, missings_by_snp, = summarize(cg10k); +mp = missingpos(cg10k) ``` - 24 + + + + 6670×630860 SparseMatrixCSC{Bool,Int32} with 5524131 stored entries: + [5688 , 1] = true + [6445 , 1] = true + [109 , 3] = true + [189 , 3] = true + [216 , 3] = true + [254 , 3] = true + [331 , 3] = true + [522 , 3] = true + [525 , 3] = true + [597 , 3] = true + [619 , 3] = true + [672 , 3] = true + ⋮ + [4929 , 630860] = true + [5594 , 630860] = true + [5650 , 630860] = true + [5780 , 630860] = true + [5854 , 630860] = true + [5867 , 630860] = true + [6084 , 630860] = true + [6175 , 630860] = true + [6178 , 630860] = true + [6239 , 630860] = true + [6478 , 630860] = true + [6511 , 630860] = true + + + +The number of missing data values in each column can be evaluated as + + +```julia +missings_by_snp = sum(mp, dims=1) +``` + + + + + 1×630860 Array{Int64,2}: + 2 0 132 77 0 27 2 2 6 27 2 … 6 4 5 11 0 0 4 29 0 5 43 + + + +Minor allele frequencies (MAF) for each SNP. + + +```julia +maf_cg10k = maf(cg10k) +``` + + + + + 630860-element Array{Float64,1}: + 0.1699160167966407 + 0.17098950524737633 + 0.11402569593147749 + 0.2686940694676172 + 0.21926536731634183 + 0.23934969140448592 + 0.19061187762447507 + 0.20200959808038388 + 0.027160864345738278 + 0.2997139846454915 + 0.24625074985003004 + 0.05555555555555558 + 0.3659067046647667 + ⋮ + 0.22547254725472543 + 0.4035864345738295 + 0.20799579957995795 + 0.44801200300075017 + 0.2954647845021775 + 0.14265367316341826 + 0.1709145427286357 + 0.2814281428142814 + 0.06113537117903933 + 0.052473763118440764 + 0.13930982745686427 + 0.1324128564961521 + + ```julia # 5 number summary and average MAF (minor allele frequencies) -quantile(maf, [0.0 .25 .5 .75 1.0]), mean(maf) +using Statistics +Statistics.quantile(maf_cg10k, [0.0 .25 .5 .75 1.0]), mean(maf_cg10k) ``` @@ -128,15 +213,14 @@ quantile(maf, [0.0 .25 .5 .75 1.0]), mean(maf) # Pkg.add("Plots") # Pkg.add("PyPlot") using Plots -pyplot() - -histogram(maf, xlab = "Minor Allele Frequency (MAF)", label = "MAF") +gr(size=(600,500), html_output_format=:png) +histogram(maf_cg10k, xlab = "Minor Allele Frequency (MAF)", label = "MAF") ``` - +![png](heritability_hist1.png) @@ -156,7 +240,7 @@ sum(missings_by_snp) / length(cg10k) ```julia # proportion of rare SNPs with maf < 0.05 -countnz(maf .< 0.05) / length(maf) +count(!iszero, maf_cg10k .< 0.05) / length(maf_cg10k) ``` @@ -172,44 +256,45 @@ We estimate empirical kinship based on all SNPs by the genetic relation matrix ( ```julia -# GRM using SNPs with maf > 0.01 (default) (~10 mins on my laptop) -srand(123) +## GRM using SNPs with maf > 0.01 (default) (~10 mins on my laptop) +using Random +Random.seed!(123) @time Φgrm = grm(cg10k; method = :GRM) ``` - 396.943890 seconds (8.43 G allocations: 127.378 GiB, 4.38% gc time) + 657.345223 seconds (176.95 k allocations: 361.775 MiB, 0.19% gc time) 6670×6670 Array{Float64,2}: - 0.503024 0.00335505 -0.000120075 … -5.45185e-5 -0.00278072 - 0.00335505 0.498958 -0.00195952 0.000868471 0.0034285 - -0.000120075 -0.00195952 0.493828 0.000174648 -0.000381467 - 0.000923828 -0.00329169 -0.00194166 -0.00223595 -0.00123508 - -8.39649e-5 -0.00353358 0.0018709 0.00222858 -0.00171176 - 0.00204208 0.000572952 0.00254025 … 0.000861385 2.99785e-5 - 0.000569323 0.0024786 -0.00185743 0.00117649 -0.00118027 - -0.000642144 0.00317992 -0.00099777 0.00354182 -0.000260645 - -0.00102913 -0.00123475 -0.00061138 0.00173885 0.00177727 - -0.00139442 0.00208423 0.000124525 -0.00145156 -0.001011 - -0.00204555 0.00011055 -0.000419398 … -0.000198235 -0.00110353 - 0.000947587 0.00167346 0.00184451 -0.000690143 -0.00304087 - 0.000322759 -0.000899805 0.00303981 0.000739331 -0.00118835 + 0.502735 0.00328112 -6.79435e-5 … -6.09591e-5 -0.00277687 + 0.00328112 0.49807 -0.00195548 0.000884994 0.00341591 + -6.79435e-5 -0.00195548 0.492348 0.000198191 -0.000337529 + 0.00087876 -0.00322605 -0.00192088 -0.00235314 -0.00124267 + -5.03227e-5 -0.00352498 0.00184588 0.00219109 -0.00163189 + 0.00203199 0.000597697 0.00251237 … 0.00088428 2.21226e-5 + 0.000560862 0.00244517 -0.00183233 0.00120214 -0.00120686 + -0.000656949 0.00322249 -0.00101472 0.00355832 -0.000240444 + -0.00103874 -0.00125164 -0.000599731 0.00176492 0.00176928 + -0.00137058 0.00209596 0.000146711 -0.0014453 -0.00103066 + -0.00209312 0.000140721 -0.000442031 … -0.000214963 -0.00107391 + 0.000933587 0.00168842 0.00185731 -0.000787908 -0.00311063 + 0.000334572 -0.000886623 0.00304182 0.000752285 -0.00123334 ⋮ ⋱ - 0.00298012 0.00130003 0.000998861 4.18454e-6 0.00303991 - -0.00207748 0.00274717 -0.00191741 -0.00107073 0.00368267 - 0.000545569 -0.00244439 -0.00299578 … -0.000669885 0.00221027 - -0.00423186 -0.00208514 -0.00108833 -0.000622127 -0.000567483 - -0.00325644 -0.000781353 0.0030423 0.000501423 -0.00010267 - 0.00041055 -0.00200772 0.00274867 -0.00624933 -0.00521365 - 0.00210519 0.000879889 -0.00107817 -0.000797878 -0.000557352 - -0.00230058 -0.000119132 0.000116817 … 0.000867087 -0.00233512 - -0.0020119 0.00230772 -0.00128837 0.00194798 -0.00048733 - -0.000944942 -0.000928073 -0.000175096 0.00126911 -0.00303766 - -5.45185e-5 0.000868471 0.000174648 0.500829 0.000469478 - -0.00278072 0.0034285 -0.000381467 0.000469478 0.500627 + 0.00298109 0.00121739 0.00102822 9.66935e-6 0.00306953 + -0.00209728 0.00271452 -0.00182325 -0.00108139 0.00366901 + 0.000549425 -0.00244419 -0.00301368 … -0.000631961 0.00215641 + -0.00423362 -0.00208073 -0.00107904 -0.000619315 -0.000593852 + -0.00326697 -0.000769552 0.00310511 0.000520658 -0.000113441 + 0.000430563 -0.0020236 0.00265425 -0.00635493 -0.00520252 + 0.00218746 0.000798767 -0.00105684 -0.000918245 -0.00061484 + -0.00230525 -0.000101149 0.000117936 … 0.000879829 -0.00233479 + -0.00201305 0.00233864 -0.00134496 0.00197044 -0.000486275 + -0.000990534 -0.000924159 -9.12302e-5 0.00122311 -0.00298296 + -6.09591e-5 0.000884994 0.000198191 0.499289 0.000481492 + -0.00277687 0.00341591 -0.000337529 0.000481492 0.499799 @@ -219,17 +304,14 @@ Read in the phenotype data and compute descriptive statistics. ```julia -# Pkg.add("DataFrames") -using DataFrames - -cg10k_trait = readtable( - "cg10k_traits.txt"; - separator = ' ', - names = [:FID; :IID; :Trait1; :Trait2; :Trait3; :Trait4; :Trait5; :Trait6; - :Trait7; :Trait8; :Trait9; :Trait10; :Trait11; :Trait12; :Trait13], - eltypes = [String; String; Float64; Float64; Float64; Float64; Float64; - Float64; Float64; Float64; Float64; Float64; Float64; Float64; Float64] - ) +#using Pkg +#pkg"add CSV DataFrames" +#using CSV, DataFrames + +cg10k_trait = CSV.File("cg10k_traits.txt"; + delim = ' ') |> DataFrame +names!(cg10k_trait, [:FID; :IID; :Trait1; :Trait2; :Trait3; :Trait4; :Trait5; :Trait6; + :Trait7; :Trait8; :Trait9; :Trait10; :Trait11; :Trait12; :Trait13]) # do not display FID and IID for privacy cg10k_trait[:, 3:end] ``` @@ -237,7 +319,7 @@ cg10k_trait[:, 3:end] -
Trait1Trait2Trait3Trait4Trait5Trait6Trait7Trait8Trait9Trait10Trait11Trait12Trait13
1-1.81573145026234-0.946150461472831.11363077580442-2.098671211191590.7444166141117480.001391718840801310.934732480409667-1.226773154181031.1160784277875-0.44362803350290.824465656443384-1.02852542216546-0.394049201727681
2-1.244400943787290.1096599925471790.467119394241789-1.621313040975891.05667583556830.9789469794191811.000146339460470.324874271402281.162321752196962.69227069487053.082636724610471.090649547860130.0256616415357438
31.455669145023051.538669329232431.094029593765550.586655272226893-0.32796454430367-0.30337709778827-0.0334354881314741-0.464463064285437-0.3319396273436-0.486839089635991-1.10648681564373-1.42015780427231-0.687463456644413
4-0.7688092766985480.5134908855142490.244263028382142-1.317402544756911.193937743268451.173441277342881.087374266752320.5360225837322610.8027592407620680.2341594117498150.394174866891074-0.7673658924760290.0635385761884935
5-0.264415132547719-0.348240421825694-0.02390650834136060.004739158022449481.256191917121931.20388836676311.298007390426270.3101136602473110.6261598610593520.8992891298312240.549967833508120.5406878095420480.179675416046033
6-1.37617270917293-1.471919677445640.291179894254146-0.803110740704731-0.264239977442213-0.260573027836772-0.165372266287781-0.2192572941183621.04702422290318-0.09858155346164820.9473934380684480.5940148120314380.245407436348479
70.1009416296374-0.191615722103455-0.5674213215966770.378571487240382-0.246656179817904-0.6088107500538580.189081058215596-1.27077787326519-0.4524761991439650.7025628772977240.3326362189571790.00269165036261810.317117176705358
8-0.3198182763674641.357744806572830.818689545938528-1.155655316443520.634483681022590.2914619086346790.933323714954726-0.7410832896824920.647477683507572-0.9708776270779660.2208611654113040.852512250237764-0.225904624283945
9-0.2883341733420320.5660825380907520.254958336116175-0.6525783028697140.6689215592773470.9783091991705580.1228629660419381.47909263782140.06721324241734490.07959039175278270.1675324552432320.2469155794421390.539932616458363
10-1.15759732583991-0.781198583545165-0.595807759833517-1.005549802604020.7898288859333210.5710584133790440.951304176233755-0.2959629829848160.990420024797070.5613093669889830.733100030623233-1.73467772245684-1.35278484330654
110.7405691504590311.408738467554150.7346899994400880.0208322841295094-0.337440968561619-0.458304040611395-0.142582512772326-0.580392297464107-0.684684998101516-0.00785381461893456-0.712244337518008-0.313345561230878-0.345419463162219
12-0.6758924864549950.2798926138296820.267915996308248-1.041036653929850.9107417156458880.8660276185131711.074144317020050.03817510035383020.766355377018601-0.340118016143495-0.8090139585050590.548521663785885-0.0201828675962336
13-0.795410435603455-0.6999899397627380.3991295030063-0.5104762619007361.515522454168441.287430329394671.537723932509030.1339891601177021.020257368860370.499018733899186-0.36948273277931-1.10153460436318-0.598132438886619
14-0.193483122930324-0.286021160323518-0.6914942252629950.01315816787006991.523374706867821.40106380722621.531146204518960.3330664834780751.043724803810990.163206783570466-0.422883765001728-0.383527976713573-0.489221907788158
150.1512462033797182.091851089936142.03800472474384-1.124747171435311.665570243907131.625356751095761.587510704836550.6358521860437760.8425777846059790.450761870778952-1.39479033623028-0.5609841075677680.289349776549287
16-0.4646087408127120.361276947723031.2327673928287-0.8260337310863831.434752247099831.744518238188460.2110968874846382.648164251405481.025114331460960.119757316031840.0596832073448267-0.631231612661616-0.207878671782927
17-0.732977488012215-0.5262234258897790.61657871336593-0.554479743325930.9474848590251040.9368332141381730.9725168063355240.2902510138652271.012853597257230.516207422283291-0.03006891719881940.87873225245830.450254629309513
18-0.1673264596221190.1753271654872370.287467725892572-0.4026525320842460.5511815094180560.5222047432909750.4368376600946530.2995649338455790.583109520896067-0.704415820005353-0.730810367994577-1.95140580379896-0.933504665700164
191.411594857874181.787224079010170.843976395853640.481278083772991-0.0887673728508268-0.499577574268580.304195897924847-1.23884208383369-0.153475724036624-0.8704861027883290.0955473331150403-0.983708050882817-0.3563445644514
20-1.42997091652825-0.4901470450342130.272730237607695-1.610299929541530.9907878171977480.7116875326081841.1885836012715-0.3712291880756381.24703459239952-0.03891623322715160.8834957490728722.589880263210173.33539552370368
21-0.1472472881767650.123284304156520.617549051912237-0.187130771782620.2564381075866940.177949837350830.412611806463263-0.2448091245597370.09476248061364920.723017223849532-0.6839483546334360.0873751276309269-0.262209652750371
22-0.187112676773894-0.270777264595619-1.015568185516060.06028505686002330.2724197577579780.869133161879197-0.6575194614142342.32388522018189-0.9999360115250341.446718441783060.971157886040772-0.358747904241515-0.439657942096136
23-1.82434047163768-0.9334804460680671.29474003766977-1.945452211510360.335846511896540.3592016543028440.513652924365886-0.0731976966969581.571390428120051.533293713267281.820768218595282.227403018678291.50063347195857
24-2.29344084351335-2.491618423444180.40383988742336-2.364880747529481.41052548319561.422441171477921.170241662721720.844766501768551.790268754324950.648181858970515-0.0857231057403538-1.027895352926170.491288088952859
25-0.4341359328883050.7408819890346520.699576357578518-1.024055431877750.7595292239837130.9566561108952880.6332995686565890.7707339322685160.8249885117145261.842874376347691.91045942063443-0.5023172078693660.132670133448219
26-2.1920969546557-2.494656642722710.354854763893431-1.931558486357140.9419794002899380.9789171014141060.8948600972897360.4632394028318731.125371333171631.705284461919550.7177927144791230.6458880491082610.783968250169388
27-1.46602269088422-1.249216771018970.307977693653039-1.550973646609890.6189084944747980.6625081716620420.4759571739060780.4847186745977070.4015648920282490.55987973254026-0.376938143754217-0.9339826292282180.390013151672955
28-1.83317744236881-1.532687878287012.55674262685865-1.518277457838350.7894096017464550.9087477997285880.6499719229414790.6683736499316671.200583035199030.2779632560756371.25049531982753.313704450716382.22035828885342
29-0.7845466282431780.2765825795439313.01104958800057-1.119788432067580.9208238584227070.7502176898861511.26153730009639-0.4033638829224170.400667296857811-0.217597941303479-0.724669537565068-0.391945338467193-0.650023936358253
300.4644559163451351.3326356122229-1.23059563374303-0.3579759589374141.182497469771041.54315938069757-0.603390411540623.383088459584220.823740765148641-0.129951318508883-0.657979878422938-0.499534924074273-0.414476569095651
⋮⋮⋮⋮⋮⋮⋮⋮⋮⋮⋮⋮⋮⋮
+

6,670 rows × 13 columns

Trait1Trait2Trait3Trait4Trait5Trait6Trait7Trait8Trait9Trait10Trait11Trait12Trait13
Float64⍰Float64⍰Float64⍰Float64⍰Float64⍰Float64⍰Float64⍰Float64⍰Float64⍰Float64⍰Float64⍰Float64⍰Float64⍰
1-1.81573-0.946151.11363-2.098670.7444170.001391720.934732-1.226771.11608-0.4436280.824466-1.02853-0.394049
2-1.24440.109660.467119-1.621311.056680.9789471.000150.3248741.162322.692273.082641.090650.0256616
31.455671.538671.094030.586655-0.327965-0.303377-0.0334355-0.464463-0.33194-0.486839-1.10649-1.42016-0.687463
4-0.7688090.5134910.244263-1.31741.193941.173441.087370.5360230.8027590.2341590.394175-0.7673660.0635386
5-0.264415-0.34824-0.02390650.004739161.256191.203891.298010.3101140.626160.8992890.5499680.5406880.179675
6-1.37617-1.471920.29118-0.803111-0.26424-0.260573-0.165372-0.2192571.04702-0.09858160.9473930.5940150.245407
70.100942-0.191616-0.5674210.378571-0.246656-0.6088110.189081-1.27078-0.4524760.7025630.3326360.002691650.317117
8-0.3198181.357740.81869-1.155660.6344840.2914620.933324-0.7410830.647478-0.9708780.2208610.852512-0.225905
9-0.2883340.5660830.254958-0.6525780.6689220.9783090.1228631.479090.06721320.07959040.1675320.2469160.539933
10-1.1576-0.781199-0.595808-1.005550.7898290.5710580.951304-0.2959630.990420.5613090.7331-1.73468-1.35278
110.7405691.408740.734690.0208323-0.337441-0.458304-0.142583-0.580392-0.684685-0.00785381-0.712244-0.313346-0.345419
12-0.6758920.2798930.267916-1.041040.9107420.8660281.074140.03817510.766355-0.340118-0.8090140.548522-0.0201829
13-0.79541-0.699990.39913-0.5104761.515521.287431.537720.1339891.020260.499019-0.369483-1.10153-0.598132
14-0.193483-0.286021-0.6914940.01315821.523371.401061.531150.3330661.043720.163207-0.422884-0.383528-0.489222
150.1512462.091852.038-1.124751.665571.625361.587510.6358520.8425780.450762-1.39479-0.5609840.28935
16-0.4646090.3612771.23277-0.8260341.434751.744520.2110972.648161.025110.1197570.0596832-0.631232-0.207879
17-0.732977-0.5262230.616579-0.554480.9474850.9368330.9725170.2902511.012850.516207-0.03006890.8787320.450255
18-0.1673260.1753270.287468-0.4026530.5511820.5222050.4368380.2995650.58311-0.704416-0.73081-1.95141-0.933505
191.411591.787220.8439760.481278-0.0887674-0.4995780.304196-1.23884-0.153476-0.8704860.0955473-0.983708-0.356345
20-1.42997-0.4901470.27273-1.61030.9907880.7116881.18858-0.3712291.24703-0.03891620.8834962.589883.3354
21-0.1472470.1232840.617549-0.1871310.2564380.177950.412612-0.2448090.09476250.723017-0.6839480.0873751-0.26221
22-0.187113-0.270777-1.015570.06028510.272420.869133-0.6575192.32389-0.9999361.446720.971158-0.358748-0.439658
23-1.82434-0.933481.29474-1.945450.3358470.3592020.513653-0.07319771.571391.533291.820772.22741.50063
24-2.29344-2.491620.40384-2.364881.410531.422441.170240.8447671.790270.648182-0.0857231-1.02790.491288
25-0.4341360.7408820.699576-1.024060.7595290.9566560.63330.7707340.8249891.842871.91046-0.5023170.13267
26-2.1921-2.494660.354855-1.931560.9419790.9789170.894860.4632391.125371.705280.7177930.6458880.783968
27-1.46602-1.249220.307978-1.550970.6189080.6625080.4759570.4847190.4015650.55988-0.376938-0.9339830.390013
28-1.83318-1.532692.55674-1.518280.789410.9087480.6499720.6683741.200580.2779631.25053.31372.22036
29-0.7845470.2765833.01105-1.119790.9208240.7502181.26154-0.4033640.400667-0.217598-0.72467-0.391945-0.650024
300.4644561.33264-1.2306-0.3579761.18251.54316-0.603393.383090.823741-0.129951-0.65798-0.499535-0.414477
⋮⋮⋮⋮⋮⋮⋮⋮⋮⋮⋮⋮⋮⋮
@@ -246,175 +328,11 @@ cg10k_trait[:, 3:end] describe(cg10k_trait[:, 3:end]) ``` - Trait1 - Summary Stats: - Mean: 0.002211 - Minimum: -3.204128 - 1st Quartile: -0.645771 - Median: 0.125010 - 3rd Quartile: 0.723315 - Maximum: 3.479398 - Length: 6670 - Type: Float64 - Number Missing: 0 - % Missing: 0.000000 - - Trait2 - Summary Stats: - Mean: 0.001353 - Minimum: -3.511659 - 1st Quartile: -0.642621 - Median: 0.033517 - 3rd Quartile: 0.657467 - Maximum: 4.913423 - Length: 6670 - Type: Float64 - Number Missing: 0 - % Missing: 0.000000 - - Trait3 - Summary Stats: - Mean: -0.001296 - Minimum: -3.938436 - 1st Quartile: -0.640907 - Median: -0.000782 - 3rd Quartile: 0.637108 - Maximum: 7.916299 - Length: 6670 - Type: Float64 - Number Missing: 0 - % Missing: 0.000000 - - Trait4 - Summary Stats: - Mean: 0.002309 - Minimum: -3.608403 - 1st Quartile: -0.546086 - Median: 0.228165 - 3rd Quartile: 0.715291 - Maximum: 3.127688 - Length: 6670 - Type: Float64 - Number Missing: 0 - % Missing: 0.000000 - - Trait5 - Summary Stats: - Mean: -0.001790 - Minimum: -4.148749 - 1st Quartile: -0.690765 - Median: 0.031034 - 3rd Quartile: 0.734916 - Maximum: 2.717184 - Length: 6670 - Type: Float64 - Number Missing: 0 - % Missing: 0.000000 - - Trait6 - Summary Stats: - Mean: -0.001196 - Minimum: -3.824792 - 1st Quartile: -0.662796 - Median: 0.036242 - 3rd Quartile: 0.741176 - Maximum: 2.589728 - Length: 6670 - Type: Float64 - Number Missing: 0 - % Missing: 0.000000 - - Trait7 - Summary Stats: - Mean: -0.001989 - Minimum: -4.272455 - 1st Quartile: -0.638923 - Median: 0.069801 - 3rd Quartile: 0.710423 - Maximum: 2.653779 - Length: 6670 - Type: Float64 - Number Missing: 0 - % Missing: 0.000000 - - Trait8 - Summary Stats: - Mean: 0.000614 - Minimum: -5.625488 - 1st Quartile: -0.601575 - Median: -0.038630 - 3rd Quartile: 0.527342 - Maximum: 5.805702 - Length: 6670 - Type: Float64 - Number Missing: 0 - % Missing: 0.000000 - - Trait9 - Summary Stats: - Mean: -0.001810 - Minimum: -5.381968 - 1st Quartile: -0.601429 - Median: 0.106571 - 3rd Quartile: 0.698567 - Maximum: 2.571936 - Length: 6670 - Type: Float64 - Number Missing: 0 - % Missing: 0.000000 - - Trait10 - Summary Stats: - Mean: -0.000437 - Minimum: -3.548506 - 1st Quartile: -0.633641 - Median: -0.096651 - 3rd Quartile: 0.498610 - Maximum: 6.537820 - Length: 6670 - Type: Float64 - Number Missing: 0 - % Missing: 0.000000 - - Trait11 - Summary Stats: - Mean: -0.000616 - Minimum: -3.264910 - 1st Quartile: -0.673685 - Median: -0.068044 - 3rd Quartile: 0.655486 - Maximum: 4.262410 - Length: 6670 - Type: Float64 - Number Missing: 0 - % Missing: 0.000000 - - Trait12 - Summary Stats: - Mean: -0.000589 - Minimum: -8.851909 - 1st Quartile: -0.539686 - Median: -0.141099 - 3rd Quartile: 0.350779 - Maximum: 13.211402 - Length: 6670 - Type: Float64 - Number Missing: 0 - % Missing: 0.000000 - - Trait13 - Summary Stats: - Mean: -0.000151 - Minimum: -5.592104 - 1st Quartile: -0.492289 - Median: -0.141022 - 3rd Quartile: 0.324804 - Maximum: 24.174436 - Length: 6670 - Type: Float64 - Number Missing: 0 - % Missing: 0.000000 - + + + +

13 rows × 8 columns

variablemeanminmedianmaxnuniquenmissingeltype
SymbolFloat64Float64Float64Float64NothingInt64DataType
1Trait10.00221138-3.204130.125013.47940Float64
2Trait20.00135253-3.511660.03351734.913420Float64
3Trait3-0.00129591-3.93844-0.0007821627.91630Float64
4Trait40.00230893-3.60840.2281653.127690Float64
5Trait5-0.00179039-4.148750.03103432.717180Float64
6Trait6-0.00119598-3.824790.0362422.589730Float64
7Trait7-0.00198906-4.272460.0698012.653780Float64
8Trait80.000614075-5.62549-0.03863015.80570Float64
9Trait9-0.00180965-5.381970.1065712.571940Float64
10Trait10-0.000437029-3.54851-0.09665076.537820Float64
11Trait11-0.000615918-3.26491-0.06804374.262410Float64
12Trait12-0.000588783-8.85191-0.14109913.21140Float64
13Trait13-0.000151238-5.5921-0.14102224.17440Float64
+ @@ -426,7 +344,7 @@ histogram(Y, layout = 13) - +![png](heritability_hist2.png) @@ -436,20 +354,17 @@ To prepare variance component model fitting, we form an instance of `VarianceCom ```julia -using VarianceComponentModels +using VarianceComponentModels, LinearAlgebra # form data as VarianceComponentVariate -cg10kdata = VarianceComponentVariate(Y, (2Φgrm, eye(size(Y, 1)))) -fieldnames(cg10kdata) +cg10kdata = VarianceComponentVariate(Y, (2Φgrm, Matrix(1.0I, size(Y, 1), size(Y, 1)))) +fieldnames(typeof(cg10kdata)) ``` - 3-element Array{Symbol,1}: - :Y - :X - :V + (:Y, :X, :V) @@ -461,7 +376,7 @@ cg10kdata - VarianceComponentModels.VarianceComponentVariate{Float64,2,Array{Float64,2},Array{Float64,2},Array{Float64,2}}([-1.81573 -0.94615 … -1.02853 -0.394049; -1.2444 0.10966 … 1.09065 0.0256616; … ; 0.886626 0.487408 … -0.636874 -0.439825; -1.24394 0.213697 … 0.299931 0.392809], Array{Float64}(6670,0), ([1.00605 0.00671009 … -0.000109037 -0.00556144; 0.00671009 0.997916 … 0.00173694 0.00685701; … ; -0.000109037 0.00173694 … 1.00166 0.000938955; -0.00556144 0.00685701 … 0.000938955 1.00125], [1.0 0.0 … 0.0 0.0; 0.0 1.0 … 0.0 0.0; … ; 0.0 0.0 … 1.0 0.0; 0.0 0.0 … 0.0 1.0])) + VarianceComponentVariate{Float64,2,Array{Float64,2},Array{Float64,2},Array{Float64,2}}([-1.81573 -0.94615 … -1.02853 -0.394049; -1.2444 0.10966 … 1.09065 0.0256616; … ; 0.886626 0.487408 … -0.636874 -0.439825; -1.24394 0.213697 … 0.299931 0.392809], Array{Float64}(6670,0), ([1.00547 0.00656224 … -0.000121918 -0.00555374; 0.00656224 0.99614 … 0.00176999 0.00683183; … ; -0.000121918 0.00176999 … 0.998578 0.000962983; -0.00555374 0.00683183 … 0.000962983 0.999599], [1.0 0.0 … 0.0 0.0; 0.0 1.0 … 0.0 0.0; … ; 0.0 0.0 … 1.0 0.0; 0.0 0.0 … 0.0 1.0])) @@ -471,21 +386,16 @@ Before fitting the variance component model, we pre-compute the eigen-decomposit ```julia # pre-compute eigen-decomposition (~50 secs on my laptop) @time cg10kdata_rotated = TwoVarCompVariateRotate(cg10kdata) -fieldnames(cg10kdata_rotated) +fieldnames(typeof(cg10kdata_rotated)) ``` - 48.837361 seconds (39 allocations: 1021.427 MiB, 0.57% gc time) + 51.732018 seconds (29 allocations: 1021.427 MiB, 0.98% gc time) - 5-element Array{Symbol,1}: - :Yrot - :Xrot - :eigval - :eigvec - :logdetV2 + (:Yrot, :Xrot, :eigval, :eigvec, :logdetV2) @@ -493,94 +403,73 @@ fieldnames(cg10kdata_rotated) We don't want to re-compute SnpArray and empirical kinship matrices again and again for heritibility analysis. +To load workspace + ```julia -# # Pkg.add("JLD") -# using JLD -# @save "cg10k.jld" -# whos() +#pkg"add FileIO JLD2" + +using JLD2, FileIO +save("cg10k.jld2") +varinfo() ``` -To load workspace + + + +| name | size | summary | +|:----------------- | ------------:|:-------------------------------------------------------------------------------------- | +| Base | | Module | +| Core | | Module | +| Main | | Module | +| Plots | 22.448 MiB | Module | +| Y | 677.461 KiB | 6670×13 Array{Float64,2} | +| cg10k | 1022.983 MiB | 6670×630860 SnpArray | +| cg10k_trait | 1.386 MiB | 6670×15 DataFrame | +| cg10kdata | 679.508 MiB | VarianceComponentVariate{Float64,2,Array{Float64,2},Array{Float64,2},Array{Float64,2}} | +| cg10kdata_rotated | 340.136 MiB | TwoVarCompVariateRotate{Float64,Array{Float64,2},Array{Float64,2}} | +| maf_cg10k | 4.813 MiB | 630860-element Array{Float64,1} | +| missings_by_snp | 4.813 MiB | 1×630860 Array{Int64,2} | +| mp | 28.748 MiB | 6670×630860 SparseMatrixCSC{Bool,Int32} | +| people | 8 bytes | Int64 | +| snps | 8 bytes | Int64 | +| startupfile | 57 bytes | String | +| Φgrm | 339.423 MiB | 6670×6670 Array{Float64,2} | + + + ```julia -using SnpArrays, JLD, DataFrames, VarianceComponentModels, Plots -pyplot() -@load "cg10k.jld" -whos() +#using SnpArrays, JLD2, DataFrames, VarianceComponentModels, Plots +#pyplot() +load("cg10k.jld") +varinfo() ``` - Base Module - BinDeps 41348 KB Module - Blosc 41202 KB Module - ColorTypes 41457 KB Module - Colors 41480 KB Module - Compat 41196 KB Module - Conda 41205 KB Module - Core Module - DataArrays 41456 KB Module - DataFrames 41684 KB Module - DataStructures 41356 KB Module - FileIO 41310 KB Module - FixedPointNumbers 41695 KB Module - GZip 41181 KB Module - HDF5 41403 KB Module - IJulia 4185781 KB Module - Ipopt 41172 KB Module - JLD 41376 KB Module - JSON 41245 KB Module - LaTeXStrings 4058 bytes Module - LegacyStrings 41212 KB Module - LinearMaps 22 KB Module - MacroTools 41606 KB Module - Main Module - MathProgBase 41353 KB Module - MbedTLS 41269 KB Module - Measures 41175 KB Module - NaNMath 41200 KB Module - PlotThemes 41167 KB Module - PlotUtils 41332 KB Module - Plots 42960 KB Module - PyCall 41711 KB Module - PyPlot 41771 KB Module - RecipesBase 41283 KB Module - Reexport 41160 KB Module - Requires 41172 KB Module - SHA 62 KB Module - Showoff 41163 KB Module - SnpArrays 41218 KB Module - SortingAlgorithms 41178 KB Module - SpecialFunctions 41252 KB Module - StaticArrays 41744 KB Module - StatsBase 41810 KB Module - URIParser 41171 KB Module - VarianceComponentModels 41278 KB Module - Y 677 KB 6670×13 Array{Float64,2} - ZMQ 41223 KB Module - _ 77 KB 630860-element BitArray{1} - cg10k 1027303 KB 6670×630860 SnpArrays.SnpArray{2} - cg10k_trait 978 KB 6670×15 DataFrames.DataFrame - cg10kdata 695816 KB VarianceComponentModels.VarianceCo… - cg10kdata_rotated 348299 KB VarianceComponentModels.TwoVarComp… - h 24 bytes 3-element Array{Float64,1} - hST 104 bytes 13-element Array{Float64,1} - hST_se 104 bytes 13-element Array{Float64,1} - hse 24 bytes 3-element Array{Float64,1} - maf 4928 KB 630860-element Array{Float64,1} - missings_by_snp 4928 KB 630860-element Array{Int64,1} - people 8 bytes Int64 - snps 8 bytes Int64 - trait57_data 347778 KB VarianceComponentModels.TwoVarComp… - trait57_model 232 bytes VarianceComponentModels.VarianceCo… - traitall_model 2792 bytes VarianceComponentModels.VarianceCo… - traitidx 16 bytes 3-element UnitRange{Int64} - Σa 3848 bytes 13×13 Array{Array{Float64,2},2} - Σcov 2592 bytes 18×18 Array{Float64,2} - Σe 3848 bytes 13×13 Array{Array{Float64,2},2} - Φgrm 347569 KB 6670×6670 Array{Float64,2} - σ2a 104 bytes 13-element Array{Float64,1} - σ2e 104 bytes 13-element Array{Float64,1} + + + +| name | size | summary | +|:----------------- | ------------:|:-------------------------------------------------------------------------------------- | +| Base | | Module | +| Core | | Module | +| Main | | Module | +| Plots | 22.448 MiB | Module | +| Y | 677.461 KiB | 6670×13 Array{Float64,2} | +| cg10k | 1022.983 MiB | 6670×630860 SnpArray | +| cg10k_trait | 1.449 MiB | 6670×15 DataFrame | +| cg10kdata | 679.508 MiB | VarianceComponentVariate{Float64,2,Array{Float64,2},Array{Float64,2},Array{Float64,2}} | +| cg10kdata_rotated | 340.136 MiB | TwoVarCompVariateRotate{Float64,Array{Float64,2},Array{Float64,2}} | +| maf_cg10k | 4.813 MiB | 630860-element Array{Float64,1} | +| missings_by_snp | 4.813 MiB | 1×630860 Array{Int64,2} | +| mp | 28.748 MiB | 6670×630860 SparseMatrixCSC{Bool,Int32} | +| people | 8 bytes | Int64 | +| snps | 8 bytes | Int64 | +| startupfile | 57 bytes | String | +| Φgrm | 339.423 MiB | 6670×6670 Array{Float64,2} | + + ## Heritability of single traits @@ -598,8 +487,7 @@ hST_se = zeros(13) # enviromental effects σ2e = zeros(13) -tic() -for trait in 1:13 +@time for trait in 1:13 println(names(cg10k_trait)[trait + 2]) # form data set for trait j traitj_data = TwoVarCompVariateRotate(cg10kdata_rotated.Yrot[:, trait], cg10kdata_rotated.Xrot, @@ -615,43 +503,42 @@ for trait in 1:13 hST[trait] = h[1] hST_se[trait] = hse[1] end -toc() ``` Trait1 - (σ2a[trait], σ2e[trait]) = (0.25978160614793233, 0.7369535197912689) + + ****************************************************************************** + This program contains Ipopt, a library for large-scale nonlinear optimization. + Ipopt is released as open source code under the Eclipse Public License (EPL). + For more information visit http://projects.coin-or.org/Ipopt + ****************************************************************************** + + (σ2a[trait], σ2e[trait]) = (0.25947604776534905, 0.7375938878381869) Trait2 - (σ2a[trait], σ2e[trait]) = (0.18647130348299173, 0.8129591079735827) + (σ2a[trait], σ2e[trait]) = (0.18588150448759697, 0.8137880281704691) Trait3 - (σ2a[trait], σ2e[trait]) = (0.3188368159422607, 0.6798809726936244) + (σ2a[trait], σ2e[trait]) = (0.3196093756535387, 0.6795300238388639) Trait4 - (σ2a[trait], σ2e[trait]) = (0.2651357653143703, 0.7308007669086968) + (σ2a[trait], σ2e[trait]) = (0.2657824469749975, 0.7304994606945179) Trait5 - (σ2a[trait], σ2e[trait]) = (0.28083388108246, 0.7172036435586534) + (σ2a[trait], σ2e[trait]) = (0.28143198005766734, 0.7169747061765026) Trait6 - (σ2a[trait], σ2e[trait]) = (0.2824159905728832, 0.7170988773569172) + (σ2a[trait], σ2e[trait]) = (0.28300551312417344, 0.7168800753377119) Trait7 - (σ2a[trait], σ2e[trait]) = (0.2155274336968625, 0.7815346282986375) + (σ2a[trait], σ2e[trait]) = (0.21565421413116753, 0.7816910320891622) Trait8 - (σ2a[trait], σ2e[trait]) = (0.194687807263945, 0.8049690651320599) + (σ2a[trait], σ2e[trait]) = (0.19408878271207622, 0.8058201577783586) Trait9 - (σ2a[trait], σ2e[trait]) = (0.24706855916591713, 0.7512942998567308) + (σ2a[trait], σ2e[trait]) = (0.24746236011764494, 0.7512222977091667) Trait10 - (σ2a[trait], σ2e[trait]) = (0.098712236297271, 0.9011756660217387) + (σ2a[trait], σ2e[trait]) = (0.0992417256213372, 0.9007769787053679) Trait11 - (σ2a[trait], σ2e[trait]) = (0.1664264642608195, 0.8322427413046204) + (σ2a[trait], σ2e[trait]) = (0.16457266481403124, 0.8343110221526249) Trait12 - (σ2a[trait], σ2e[trait]) = (0.0834296761650666, 0.9153609794266608) + (σ2a[trait], σ2e[trait]) = (0.08224956591863997, 0.9166483378364418) Trait13 - (σ2a[trait], σ2e[trait]) = (0.05893968504298988, 0.940270012443928) - elapsed time: 0.160999612 seconds - - - - - - 0.160999612 - + (σ2a[trait], σ2e[trait]) = (0.05687679106196685, 0.9424058676223442) + 6.559467 seconds (12.64 M allocations: 611.362 MiB, 11.95% gc time) @@ -664,8 +551,8 @@ toc() 2×13 Array{Float64,2}: - 0.260633 0.186578 0.319246 … 0.166648 0.0835307 0.0589863 - 0.0799732 0.0869002 0.0741007 0.08862 0.0944407 0.0953238 + 0.260239 0.185943 0.319885 … 0.164757 0.0823403 0.0569176 + 0.0799434 0.08689 0.0739664 0.0887138 0.0944375 0.0953072 @@ -676,12 +563,12 @@ Joint analysis of multiple traits is subject to intensive research recently. Fol ```julia # additive genetic effects (2x2 psd matrices) from bavariate trait analysis; -Σa = Array{Matrix{Float64}}(13, 13) +Σa = Array{Matrix{Float64}}(undef, 13, 13) # environmental effects (2x2 psd matrices) from bavariate trait analysis; -Σe = Array{Matrix{Float64}}(13, 13) +Σe = Array{Matrix{Float64}}(undef, 13, 13) -tic() -for i in 1:13 + +@time for i in 1:13 for j in (i+1):13 println(names(cg10k_trait)[i + 2], names(cg10k_trait)[j + 2]) # form data set for (trait1, trait2) @@ -696,173 +583,165 @@ for i in 1:13 @show Σa[i, j], Σe[i, j] end end -toc() ``` Trait1Trait2 - (Σa[i, j], Σe[i, j]) = ([0.258822 0.174358; 0.174358 0.185108], [0.737892 0.585751; 0.585751 0.814301]) + (Σa[i, j], Σe[i, j]) = ([0.258529 0.173943; 0.173943 0.184525], [0.738519 0.58639; 0.58639 0.815122]) Trait1Trait3 - (Σa[i, j], Σe[i, j]) = ([0.260236 -0.0144726; -0.0144726 0.319245], [0.736512 -0.11979; -0.11979 0.679488]) + (Σa[i, j], Σe[i, j]) = ([0.259934 -0.0139395; -0.0139395 0.320025], [0.737149 -0.120342; -0.120342 0.67913]) Trait1Trait4 - (Σa[i, j], Σe[i, j]) = ([0.259615 0.222203; 0.222203 0.265149], [0.737116 0.599854; 0.599854 0.730788]) + (Σa[i, j], Σe[i, j]) = ([0.259307 0.222267; 0.222267 0.265791], [0.737759 0.600077; 0.600077 0.730492]) Trait1Trait5 - (Σa[i, j], Σe[i, j]) = ([0.259574 -0.146827; -0.146827 0.28153], [0.737153 -0.254777; -0.254777 0.71653]) + (Σa[i, j], Σe[i, j]) = ([0.259264 -0.147125; -0.147125 0.282102], [0.737798 -0.254669; -0.254669 0.716327]) Trait1Trait6 - (Σa[i, j], Σe[i, j]) = ([0.259476 -0.129115; -0.129115 0.282688], [0.73725 -0.23161; -0.23161 0.716837]) + (Σa[i, j], Σe[i, j]) = ([0.259171 -0.129519; -0.129519 0.283267], [0.73789 -0.231373; -0.231373 0.716628]) Trait1Trait7 - (Σa[i, j], Σe[i, j]) = ([0.259115 -0.140455; -0.140455 0.215297], [0.737606 -0.197616; -0.197616 0.781774]) + (Σa[i, j], Σe[i, j]) = ([0.258784 -0.140357; -0.140357 0.21535], [0.738269 -0.197894; -0.197894 0.782002]) Trait1Trait8 - (Σa[i, j], Σe[i, j]) = ([0.259778 -0.0327756; -0.0327756 0.194698], [0.736957 -0.127026; -0.127026 0.804959]) + (Σa[i, j], Σe[i, j]) = ([0.259467 -0.0336174; -0.0336174 0.194107], [0.737603 -0.12623; -0.12623 0.805802]) Trait1Trait9 - (Σa[i, j], Σe[i, j]) = ([0.261858 -0.204589; -0.204589 0.246027], [0.734961 -0.307734; -0.307734 0.75232]) + (Σa[i, j], Σe[i, j]) = ([0.261524 -0.204467; -0.204467 0.246378], [0.735632 -0.308118; -0.308118 0.752288]) Trait1Trait10 - (Σa[i, j], Σe[i, j]) = ([0.259649 -0.0994858; -0.0994858 0.0956585], [0.737083 -0.303942; -0.303942 0.904218]) + (Σa[i, j], Σe[i, j]) = ([0.259326 -0.0987008; -0.0987008 0.0962217], [0.73774 -0.304854; -0.304854 0.90378]) Trait1Trait11 - (Σa[i, j], Σe[i, j]) = ([0.25947 -0.138603; -0.138603 0.164709], [0.737257 -0.359557; -0.359557 0.83395]) + (Σa[i, j], Σe[i, j]) = ([0.259194 -0.137578; -0.137578 0.162813], [0.737868 -0.36076; -0.36076 0.836058]) Trait1Trait12 - (Σa[i, j], Σe[i, j]) = ([0.261779 -0.145414; -0.145414 0.0807748], [0.735076 -0.041823; -0.041823 0.9181]) + (Σa[i, j], Σe[i, j]) = ([0.26168 -0.145305; -0.145305 0.0806851], [0.735514 -0.0421192; -0.0421192 0.918294]) Trait1Trait13 - (Σa[i, j], Σe[i, j]) = ([0.261125 -0.108774; -0.108774 0.0538214], [0.735674 -0.114123; -0.114123 0.945416]) + (Σa[i, j], Σe[i, j]) = ([0.260776 -0.108277; -0.108277 0.0513909], [0.736359 -0.114756; -0.114756 0.947913]) Trait2Trait3 - (Σa[i, j], Σe[i, j]) = ([0.186541 0.144056; 0.144056 0.320627], [0.812888 0.0995944; 0.0995944 0.678167]) + (Σa[i, j], Σe[i, j]) = ([0.185992 0.14505; 0.14505 0.321413], [0.813678 0.0987898; 0.0987898 0.677805]) Trait2Trait4 - (Σa[i, j], Σe[i, j]) = ([0.186131 0.0746032; 0.0746032 0.265122], [0.813293 0.221109; 0.221109 0.730814]) + (Σa[i, j], Σe[i, j]) = ([0.18554 0.074367; 0.074367 0.26577], [0.814123 0.22144; 0.22144 0.730512]) Trait2Trait5 - (Σa[i, j], Σe[i, j]) = ([0.186442 -0.0118093; -0.0118093 0.280842], [0.812987 -0.0365191; -0.0365191 0.717195]) + (Σa[i, j], Σe[i, j]) = ([0.185854 -0.0115021; -0.0115021 0.281443], [0.813815 -0.0368393; -0.0368393 0.716964]) Trait2Trait6 - (Σa[i, j], Σe[i, j]) = ([0.18649 -0.00366533; -0.00366533 0.282471], [0.812941 -0.0206271; -0.0206271 0.717046]) + (Σa[i, j], Σe[i, j]) = ([0.185903 -0.00350261; -0.00350261 0.283064], [0.813767 -0.0207933; -0.0207933 0.716823]) Trait2Trait7 - (Σa[i, j], Σe[i, j]) = ([0.186104 -0.030665; -0.030665 0.215304], [0.81332 -0.000667009; -0.000667009 0.781755]) + (Σa[i, j], Σe[i, j]) = ([0.185498 -0.0301226; -0.0301226 0.215413], [0.814164 -0.00124563; -0.00124563 0.781929]) Trait2Trait8 - (Σa[i, j], Σe[i, j]) = ([0.187023 0.0331783; 0.0331783 0.195259], [0.812421 -0.0326343; -0.0326343 0.804415]) + (Σa[i, j], Σe[i, j]) = ([0.186397 0.032688; 0.032688 0.194625], [0.813285 -0.0321045; -0.0321045 0.805301]) Trait2Trait9 - (Σa[i, j], Σe[i, j]) = ([0.185032 -0.085334; -0.085334 0.245909], [0.814386 -0.0809638; -0.0809638 0.752433]) + (Σa[i, j], Σe[i, j]) = ([0.184426 -0.084907; -0.084907 0.246288], [0.815228 -0.0814978; -0.0814978 0.752373]) Trait2Trait10 - (Σa[i, j], Σe[i, j]) = ([0.186587 -0.123303; -0.123303 0.0987387], [0.812872 -0.273083; -0.273083 0.901229]) + (Σa[i, j], Σe[i, j]) = ([0.18604 -0.123261; -0.123261 0.0992567], [0.813659 -0.273285; -0.273285 0.90084]) Trait2Trait11 - (Σa[i, j], Σe[i, j]) = ([0.185484 -0.117256; -0.117256 0.167776], [0.81393 -0.296772; -0.296772 0.830934]) + (Σa[i, j], Σe[i, j]) = ([0.184894 -0.116422; -0.116422 0.165892], [0.814757 -0.297755; -0.297755 0.833033]) Trait2Trait12 - (Σa[i, j], Σe[i, j]) = ([0.185907 -0.0909104; -0.0909104 0.0827171], [0.813555 0.0457924; 0.0457924 0.916135]) + (Σa[i, j], Σe[i, j]) = ([0.185297 -0.0908872; -0.0908872 0.0814927], [0.814404 0.0456529; 0.0456529 0.917465]) Trait2Trait13 - (Σa[i, j], Σe[i, j]) = ([0.185979 -0.0720811; -0.0720811 0.0568238], [0.8135 0.0751703; 0.0751703 0.942424]) + (Σa[i, j], Σe[i, j]) = ([0.185348 -0.07104; -0.07104 0.0546791], [0.814367 0.0740412; 0.0740412 0.944638]) Trait3Trait4 - (Σa[i, j], Σe[i, j]) = ([0.3188 -0.154562; -0.154562 0.264323], [0.679917 -0.303223; -0.303223 0.731591]) + (Σa[i, j], Σe[i, j]) = ([0.319575 -0.15468; -0.15468 0.265006], [0.679563 -0.303309; -0.303309 0.731254]) Trait3Trait5 - (Σa[i, j], Σe[i, j]) = ([0.319216 0.183527; 0.183527 0.282063], [0.679514 0.33724; 0.33724 0.716008]) + (Σa[i, j], Σe[i, j]) = ([0.320016 0.184736; 0.184736 0.282719], [0.679136 0.336276; 0.336276 0.715725]) Trait3Trait6 - (Σa[i, j], Σe[i, j]) = ([0.319776 0.165672; 0.165672 0.284448], [0.678972 0.298667; 0.298667 0.715124]) + (Σa[i, j], Σe[i, j]) = ([0.320608 0.166929; 0.166929 0.285124], [0.678565 0.297633; 0.297633 0.714823]) Trait3Trait7 - (Σa[i, j], Σe[i, j]) = ([0.318838 0.166283; 0.166283 0.215261], [0.67988 0.347706; 0.347706 0.781796]) + (Σa[i, j], Σe[i, j]) = ([0.319614 0.167317; 0.167317 0.215458], [0.679526 0.346897; 0.346897 0.781883]) Trait3Trait8 - (Σa[i, j], Σe[i, j]) = ([0.320718 0.0566397; 0.0566397 0.197764], [0.678063 0.0451569; 0.0451569 0.801956]) + (Σa[i, j], Σe[i, j]) = ([0.321532 0.0573397; 0.0573397 0.197221], [0.677674 0.0445311; 0.0445311 0.802757]) Trait3Trait9 - (Σa[i, j], Σe[i, j]) = ([0.319001 0.137699; 0.137699 0.246142], [0.679722 0.266704; 0.266704 0.752197]) + (Σa[i, j], Σe[i, j]) = ([0.319777 0.138268; 0.138268 0.246547], [0.679369 0.266317; 0.266317 0.752112]) Trait3Trait10 - (Σa[i, j], Σe[i, j]) = ([0.31908 -0.076513; -0.076513 0.0996001], [0.679646 -0.142905; -0.142905 0.900298]) + (Σa[i, j], Σe[i, j]) = ([0.319905 -0.0779601; -0.0779601 0.100192], [0.679245 -0.141562; -0.141562 0.899839]) Trait3Trait11 - (Σa[i, j], Σe[i, j]) = ([0.318094 -0.0177494; -0.0177494 0.16629], [0.6806 -0.1144; -0.1144 0.832376]) + (Σa[i, j], Σe[i, j]) = ([0.318882 -0.0177352; -0.0177352 0.164446], [0.680233 -0.114434; -0.114434 0.834435]) Trait3Trait12 - (Σa[i, j], Σe[i, j]) = ([0.321164 0.0843842; 0.0843842 0.0874609], [0.677639 0.0341558; 0.0341558 0.911368]) + (Σa[i, j], Σe[i, j]) = ([0.321958 0.0844644; 0.0844644 0.0863388], [0.67727 0.034186; 0.034186 0.912603]) Trait3Trait13 - (Σa[i, j], Σe[i, j]) = ([0.323273 0.109443; 0.109443 0.0634295], [0.675635 -0.0060525; -0.0060525 0.935819]) + (Σa[i, j], Σe[i, j]) = ([0.32405 0.109731; 0.109731 0.0613165], [0.675287 -0.00619989; -0.00619989 0.938011]) Trait4Trait5 - (Σa[i, j], Σe[i, j]) = ([0.26525 -0.215125; -0.215125 0.282572], [0.73068 -0.377406; -0.377406 0.715518]) + (Σa[i, j], Σe[i, j]) = ([0.26593 -0.216116; -0.216116 0.283173], [0.730347 -0.376697; -0.376697 0.715289]) Trait4Trait6 - (Σa[i, j], Σe[i, j]) = ([0.265715 -0.199714; -0.199714 0.283942], [0.730231 -0.347732; -0.347732 0.715619]) + (Σa[i, j], Σe[i, j]) = ([0.266395 -0.200793; -0.200793 0.284547], [0.729899 -0.346915; -0.346915 0.715387]) Trait4Trait7 - (Σa[i, j], Σe[i, j]) = ([0.26407 -0.18238; -0.18238 0.214324], [0.731843 -0.32655; -0.32655 0.782733]) + (Σa[i, j], Σe[i, j]) = ([0.264731 -0.182912; -0.182912 0.21441], [0.731526 -0.326257; -0.326257 0.782928]) Trait4Trait8 - (Σa[i, j], Σe[i, j]) = ([0.266229 -0.0965381; -0.0965381 0.196655], [0.729739 -0.151461; -0.151461 0.803044]) + (Σa[i, j], Σe[i, j]) = ([0.266914 -0.0976281; -0.0976281 0.196119], [0.729401 -0.150498; -0.150498 0.803836]) Trait4Trait9 - (Σa[i, j], Σe[i, j]) = ([0.269627 -0.226931; -0.226931 0.247265], [0.726443 -0.416085; -0.416085 0.751086]) + (Σa[i, j], Σe[i, j]) = ([0.270267 -0.227463; -0.227463 0.247665], [0.726155 -0.415849; -0.415849 0.751008]) Trait4Trait10 - (Σa[i, j], Σe[i, j]) = ([0.265098 -0.0352926; -0.0352926 0.0981462], [0.730847 -0.226248; -0.226248 0.901736]) + (Σa[i, j], Σe[i, j]) = ([0.265756 -0.0339156; -0.0339156 0.0987618], [0.730536 -0.227666; -0.227666 0.901251]) Trait4Trait11 - (Σa[i, j], Σe[i, j]) = ([0.265178 -0.0970634; -0.0970634 0.164885], [0.73076 -0.272291; -0.272291 0.833762]) + (Σa[i, j], Σe[i, j]) = ([0.265858 -0.0963336; -0.0963336 0.163013], [0.730427 -0.273145; -0.273145 0.835846]) Trait4Trait12 - (Σa[i, j], Σe[i, j]) = ([0.267732 -0.140985; -0.140985 0.081029], [0.728323 -0.0834791; -0.0834791 0.917815]) + (Σa[i, j], Σe[i, j]) = ([0.268394 -0.141452; -0.141452 0.0796851], [0.72801 -0.0831954; -0.0831954 0.919263]) Trait4Trait13 - (Σa[i, j], Σe[i, j]) = ([0.265695 -0.0970238; -0.0970238 0.0564809], [0.730259 -0.226115; -0.226115 0.942736]) + (Σa[i, j], Σe[i, j]) = ([0.266334 -0.0971798; -0.0971798 0.0541478], [0.729966 -0.226082; -0.226082 0.945138]) Trait5Trait6 - (Σa[i, j], Σe[i, j]) = ([0.281198 0.280259; 0.280259 0.281764], [0.716855 0.661013; 0.661013 0.717735]) + (Σa[i, j], Σe[i, j]) = ([0.281794 0.281033; 0.281033 0.282364], [0.716628 0.660607; 0.660607 0.717505]) Trait5Trait7 - (Σa[i, j], Σe[i, j]) = ([0.280442 0.231918; 0.231918 0.211837], [0.717597 0.674491; 0.674491 0.785172]) + (Σa[i, j], Σe[i, j]) = ([0.281023 0.232223; 0.232223 0.211943], [0.717383 0.67449; 0.67449 0.785343]) Trait5Trait8 - (Σa[i, j], Σe[i, j]) = ([0.280958 0.163168; 0.163168 0.193315], [0.717089 0.221817; 0.221817 0.806314]) + (Σa[i, j], Σe[i, j]) = ([0.281551 0.164011; 0.164011 0.192737], [0.716865 0.221187; 0.221187 0.807144]) Trait5Trait9 - (Σa[i, j], Σe[i, j]) = ([0.283544 0.243884; 0.243884 0.240564], [0.714585 0.509072; 0.509072 0.757631]) + (Σa[i, j], Σe[i, j]) = ([0.284085 0.244479; 0.244479 0.240901], [0.714415 0.508795; 0.508795 0.757606]) Trait5Trait10 - (Σa[i, j], Σe[i, j]) = ([0.281378 -0.0454427; -0.0454427 0.100081], [0.716678 -0.0579778; -0.0579778 0.899822]) + (Σa[i, j], Σe[i, j]) = ([0.281993 -0.0460624; -0.0460624 0.100628], [0.716433 -0.0574194; -0.0574194 0.899407]) Trait5Trait11 - (Σa[i, j], Σe[i, j]) = ([0.280066 0.0195669; 0.0195669 0.165607], [0.71795 -0.0345589; -0.0345589 0.833047]) + (Σa[i, j], Σe[i, j]) = ([0.280652 0.0199425; 0.0199425 0.163738], [0.717732 -0.0349079; -0.0349079 0.83513]) Trait5Trait12 - (Σa[i, j], Σe[i, j]) = ([0.28101 0.0592641; 0.0592641 0.0831831], [0.717036 0.0552788; 0.0552788 0.915608]) + (Σa[i, j], Σe[i, j]) = ([0.281652 0.0612882; 0.0612882 0.0820476], [0.716765 0.0533362; 0.0533362 0.916851]) Trait5Trait13 - (Σa[i, j], Σe[i, j]) = ([0.281854 0.0680641; 0.0680641 0.0591899], [0.716223 0.0551992; 0.0551992 0.940027]) + (Σa[i, j], Σe[i, j]) = ([0.282494 0.0696137; 0.0696137 0.0570685], [0.715955 0.0537392; 0.0537392 0.942222]) Trait6Trait7 - (Σa[i, j], Σe[i, j]) = ([0.282435 0.220236; 0.220236 0.213997], [0.71708 0.581507; 0.581507 0.783041]) + (Σa[i, j], Σe[i, j]) = ([0.283018 0.220794; 0.220794 0.214128], [0.716868 0.58124; 0.58124 0.783191]) Trait6Trait8 - (Σa[i, j], Σe[i, j]) = ([0.282435 0.18375; 0.18375 0.192999], [0.717081 0.436932; 0.436932 0.80663]) + (Σa[i, j], Σe[i, j]) = ([0.283018 0.183995; 0.183995 0.192405], [0.716869 0.436926; 0.436926 0.807474]) Trait6Trait9 - (Σa[i, j], Σe[i, j]) = ([0.284516 0.233768; 0.233768 0.242478], [0.715071 0.477502; 0.477502 0.755765]) + (Σa[i, j], Σe[i, j]) = ([0.285057 0.234311; 0.234311 0.24283], [0.714902 0.477263; 0.477263 0.755728]) Trait6Trait10 - (Σa[i, j], Σe[i, j]) = ([0.283087 -0.0427658; -0.0427658 0.100634], [0.716449 -0.0599491; -0.0599491 0.899275]) + (Σa[i, j], Σe[i, j]) = ([0.283699 -0.0433873; -0.0433873 0.101183], [0.716209 -0.0593852; -0.0593852 0.89886]) Trait6Trait11 - (Σa[i, j], Σe[i, j]) = ([0.281046 0.0272144; 0.0272144 0.165044], [0.71843 -0.0516242; -0.0516242 0.833601]) + (Σa[i, j], Σe[i, j]) = ([0.281626 0.027639; 0.027639 0.163178], [0.718219 -0.0520113; -0.0520113 0.83568]) Trait6Trait12 - (Σa[i, j], Σe[i, j]) = ([0.28256 0.0548537; 0.0548537 0.083133], [0.716961 0.0502064; 0.0502064 0.915658]) + (Σa[i, j], Σe[i, j]) = ([0.283196 0.0569533; 0.0569533 0.0819944], [0.716699 0.0481832; 0.0481832 0.916905]) Trait6Trait13 - (Σa[i, j], Σe[i, j]) = ([0.283231 0.0585667; 0.0585667 0.0592752], [0.716314 0.055827; 0.055827 0.939942]) + (Σa[i, j], Σe[i, j]) = ([0.283862 0.0600111; 0.0600111 0.0571748], [0.716057 0.0544601; 0.0544601 0.942116]) Trait7Trait8 - (Σa[i, j], Σe[i, j]) = ([0.213998 0.0875641; 0.0875641 0.192993], [0.78304 -0.055939; -0.055939 0.806635]) + (Σa[i, j], Σe[i, j]) = ([0.214129 0.0883281; 0.0883281 0.192399], [0.78319 -0.0565873; -0.0565873 0.80748]) Trait7Trait9 - (Σa[i, j], Σe[i, j]) = ([0.219039 0.216925; 0.216925 0.243338], [0.778156 0.463024; 0.463024 0.754935]) + (Σa[i, j], Σe[i, j]) = ([0.219059 0.217228; 0.217228 0.243722], [0.778422 0.463004; 0.463004 0.754868]) Trait7Trait10 - (Σa[i, j], Σe[i, j]) = ([0.216296 -0.0412106; -0.0412106 0.100663], [0.780785 -0.0868086; -0.0868086 0.899246]) + (Σa[i, j], Σe[i, j]) = ([0.216477 -0.042081; -0.042081 0.101258], [0.78089 -0.0859952; -0.0859952 0.898786]) Trait7Trait11 - (Σa[i, j], Σe[i, j]) = ([0.2142 0.0204227; 0.0204227 0.165077], [0.782833 -0.0478727; -0.0478727 0.833569]) + (Σa[i, j], Σe[i, j]) = ([0.214308 0.0205244; 0.0205244 0.163203], [0.783006 -0.0479474; -0.0479474 0.835656]) Trait7Trait12 - (Σa[i, j], Σe[i, j]) = ([0.215054 0.0738562; 0.0738562 0.0814228], [0.782012 0.0366272; 0.0366272 0.917365]) + (Σa[i, j], Σe[i, j]) = ([0.215192 0.0752708; 0.0752708 0.0802075], [0.782157 0.0353123; 0.0353123 0.918686]) Trait7Trait13 - (Σa[i, j], Σe[i, j]) = ([0.216093 0.0728515; 0.0728515 0.0570272], [0.781006 0.0409945; 0.0409945 0.942189]) + (Σa[i, j], Σe[i, j]) = ([0.216198 0.0740251; 0.0740251 0.0547527], [0.781186 0.0399163; 0.0399163 0.944534]) Trait8Trait9 - (Σa[i, j], Σe[i, j]) = ([0.195154 0.111756; 0.111756 0.246453], [0.804528 0.185842; 0.185842 0.751896]) + (Σa[i, j], Σe[i, j]) = ([0.194551 0.112282; 0.112282 0.246832], [0.805384 0.185461; 0.185461 0.751837]) Trait8Trait10 - (Σa[i, j], Σe[i, j]) = ([0.195015 -0.015506; -0.015506 0.0990776], [0.804651 0.0118538; 0.0118538 0.900815]) + (Σa[i, j], Σe[i, j]) = ([0.194399 -0.0154035; -0.0154035 0.0995883], [0.805519 0.0117323; 0.0117323 0.900435]) Trait8Trait11 - (Σa[i, j], Σe[i, j]) = ([0.194421 0.0215044; 0.0215044 0.166226], [0.805231 -0.026247; -0.026247 0.83244]) + (Σa[i, j], Σe[i, j]) = ([0.193837 0.022072; 0.022072 0.164393], [0.806067 -0.0267843; -0.0267843 0.834489]) Trait8Trait12 - (Σa[i, j], Σe[i, j]) = ([0.194491 -0.00425152; -0.00425152 0.0832711], [0.805162 0.0349872; 0.0349872 0.915518]) + (Σa[i, j], Σe[i, j]) = ([0.193914 -0.00259164; -0.00259164 0.0821237], [0.805992 0.0333257; 0.0333257 0.916773]) Trait8Trait13 - (Σa[i, j], Σe[i, j]) = ([0.19448 0.00235501; 0.00235501 0.0589351], [0.805173 0.0396048; 0.0396048 0.940275]) + (Σa[i, j], Σe[i, j]) = ([0.193913 0.00330925; 0.00330925 0.0569126], [0.805993 0.0386552; 0.0386552 0.94237]) Trait9Trait10 - (Σa[i, j], Σe[i, j]) = ([0.246455 -0.00257997; -0.00257997 0.0984563], [0.751895 0.0743439; 0.0743439 0.901429]) + (Σa[i, j], Σe[i, j]) = ([0.246837 -0.00323396; -0.00323396 0.0989732], [0.751834 0.074992; 0.074992 0.901043]) Trait9Trait11 - (Σa[i, j], Σe[i, j]) = ([0.247001 0.0303415; 0.0303415 0.166421], [0.75136 0.153765; 0.153765 0.832248]) + (Σa[i, j], Σe[i, j]) = ([0.247395 0.0305136; 0.0305136 0.164575], [0.751288 0.153633; 0.153633 0.834308]) Trait9Trait12 - (Σa[i, j], Σe[i, j]) = ([0.249421 0.0829968; 0.0829968 0.0890874], [0.749007 0.109331; 0.109331 0.909778]) + (Σa[i, j], Σe[i, j]) = ([0.249928 0.0844216; 0.0844216 0.0880992], [0.748829 0.108018; 0.108018 0.910884]) Trait9Trait13 - (Σa[i, j], Σe[i, j]) = ([0.24861 0.0916799; 0.0916799 0.0602352], [0.749811 0.100027; 0.100027 0.939032]) + (Σa[i, j], Σe[i, j]) = ([0.248998 0.0925603; 0.0925603 0.0580327], [0.749747 0.0992651; 0.0992651 0.94131]) Trait10Trait11 - (Σa[i, j], Σe[i, j]) = ([0.0914658 0.100613; 0.100613 0.166501], [0.908376 0.473847; 0.473847 0.83217]) + (Σa[i, j], Σe[i, j]) = ([0.0923059 0.0990858; 0.0990858 0.164638], [0.907657 0.475503; 0.475503 0.834248]) Trait10Trait12 - (Σa[i, j], Σe[i, j]) = ([0.0951392 0.0588424; 0.0588424 0.0796744], [0.904735 0.0828862; 0.0828862 0.919115]) + (Σa[i, j], Σe[i, j]) = ([0.0957712 0.0574496; 0.0574496 0.0785646], [0.904228 0.0843534; 0.0843534 0.920326]) Trait10Trait13 - (Σa[i, j], Σe[i, j]) = ([0.0995192 -0.0257171; -0.0257171 0.0598595], [0.900397 0.163778; 0.163778 0.939368]) + (Σa[i, j], Σe[i, j]) = ([0.100109 -0.0266483; -0.0266483 0.0578511], [0.899941 0.164674; 0.164674 0.941451]) Trait11Trait12 - (Σa[i, j], Σe[i, j]) = ([0.165386 0.0579914; 0.0579914 0.0796005], [0.83327 0.144637; 0.144637 0.919166]) + (Σa[i, j], Σe[i, j]) = ([0.163544 0.0571549; 0.0571549 0.0784378], [0.835325 0.145549; 0.145549 0.920432]) Trait11Trait13 - (Σa[i, j], Σe[i, j]) = ([0.166417 -0.000985185; -0.000985185 0.0595681], [0.832265 0.200012; 0.200012 0.939646]) + (Σa[i, j], Σe[i, j]) = ([0.164571 -0.00169939; -0.00169939 0.0575331], [0.834326 0.200725; 0.200725 0.941755]) Trait12Trait13 - (Σa[i, j], Σe[i, j]) = ([0.085082 0.0696185; 0.0696185 0.0569655], [0.913729 0.572041; 0.572041 0.942247]) - elapsed time: 3.587337102 seconds - - - - - - 3.587337102 - + (Σa[i, j], Σe[i, j]) = ([0.08398 0.0685417; 0.0685417 0.0559416], [0.91494 0.573206; 0.573206 0.943343]) + 4.258555 seconds (4.55 M allocations: 306.219 MiB, 3.55% gc time) ## 3-trait analysis @@ -882,7 +761,7 @@ trait57_model = VarianceComponentModel(trait57_data) trait57_model ``` - This is Ipopt version 3.12.4, running with linear solver mumps. + This is Ipopt version 3.12.10, running with linear solver mumps. NOTE: Other linear solvers might be more efficient (see Ipopt documentation). Number of nonzeros in equality constraint Jacobian...: 0 @@ -900,26 +779,26 @@ trait57_model inequality constraints with only upper bounds: 0 iter objective inf_pr inf_du lg(mu) ||d|| lg(rg) alpha_du alpha_pr ls - 0 3.0247565e+04 0.00e+00 1.00e+02 0.0 0.00e+00 - 0.00e+00 0.00e+00 0 - 5 1.6835078e+04 0.00e+00 4.08e+02 -11.0 3.64e-01 - 1.00e+00 1.00e+00f 1 MaxS - 10 1.4742941e+04 0.00e+00 1.10e+02 -11.0 2.35e-01 - 1.00e+00 1.00e+00f 1 MaxS - 15 1.4701394e+04 0.00e+00 1.16e+01 -11.0 7.78e-02 -4.5 1.00e+00 1.00e+00f 1 MaxS - 20 1.4701019e+04 0.00e+00 5.75e-01 -11.0 1.51e-04 -6.9 1.00e+00 1.00e+00f 1 MaxS - 25 1.4701018e+04 0.00e+00 2.40e-02 -11.0 6.38e-06 -9.2 1.00e+00 1.00e+00f 1 MaxS - 30 1.4701018e+04 0.00e+00 9.98e-04 -11.0 2.66e-07 -11.6 1.00e+00 1.00e+00f 1 MaxS - 35 1.4701018e+04 0.00e+00 4.15e-05 -11.0 1.10e-08 -14.0 1.00e+00 1.00e+00h 1 MaxS - 40 1.4701018e+04 0.00e+00 1.72e-06 -11.0 4.59e-10 -16.4 1.00e+00 1.00e+00f 1 MaxSA - 45 1.4701018e+04 0.00e+00 7.17e-08 -11.0 1.91e-11 -18.8 1.00e+00 1.00e+00h 1 MaxSA + 0 3.0244169e+04 0.00e+00 1.00e+02 0.0 0.00e+00 - 0.00e+00 0.00e+00 0 + 5 1.6834042e+04 0.00e+00 4.07e+02 -11.0 3.66e-01 - 1.00e+00 1.00e+00f 1 MaxS + 10 1.4744248e+04 0.00e+00 1.12e+02 -11.0 2.36e-01 - 1.00e+00 1.00e+00f 1 MaxS + 15 1.4701415e+04 0.00e+00 1.25e+01 -11.0 1.08e-01 -4.5 1.00e+00 1.00e+00f 1 MaxS + 20 1.4700955e+04 0.00e+00 6.34e-01 -11.0 1.66e-04 -6.9 1.00e+00 1.00e+00f 1 MaxS + 25 1.4700954e+04 0.00e+00 2.63e-02 -11.0 7.01e-06 -9.2 1.00e+00 1.00e+00f 1 MaxS + 30 1.4700954e+04 0.00e+00 1.09e-03 -11.0 2.90e-07 -11.6 1.00e+00 1.00e+00f 1 MaxS + 35 1.4700954e+04 0.00e+00 4.49e-05 -11.0 1.20e-08 -14.0 1.00e+00 1.00e+00f 1 MaxS + 40 1.4700954e+04 0.00e+00 1.86e-06 -11.0 4.96e-10 -16.4 1.00e+00 1.00e+00h 1 MaxSA + 45 1.4700954e+04 0.00e+00 7.67e-08 -11.0 2.05e-11 -18.8 1.00e+00 1.00e+00h 1 MaxSA iter objective inf_pr inf_du lg(mu) ||d|| lg(rg) alpha_du alpha_pr ls Number of Iterations....: 49 (scaled) (unscaled) - Objective...............: 4.4720359684330265e+02 1.4701017692082147e+04 - Dual infeasibility......: 5.6081357364421780e-09 1.8435742302386474e-07 + Objective...............: 4.4662368766169078e+02 1.4700954216526392e+04 + Dual infeasibility......: 5.8683114125458765e-09 1.9315987885869595e-07 Constraint violation....: 0.0000000000000000e+00 0.0000000000000000e+00 Complementarity.........: 0.0000000000000000e+00 0.0000000000000000e+00 - Overall NLP error.......: 5.6081357364421780e-09 1.8435742302386474e-07 + Overall NLP error.......: 5.8683114125458765e-09 1.9315987885869595e-07 Number of objective function evaluations = 50 @@ -929,17 +808,17 @@ trait57_model Number of equality constraint Jacobian evaluations = 0 Number of inequality constraint Jacobian evaluations = 0 Number of Lagrangian Hessian evaluations = 49 - Total CPU secs in IPOPT (w/o function evaluations) = 0.014 - Total CPU secs in NLP function evaluations = 0.076 + Total CPU secs in IPOPT (w/o function evaluations) = 0.022 + Total CPU secs in NLP function evaluations = 0.059 EXIT: Optimal Solution Found. - 0.097955 seconds (55.15 k allocations: 5.632 MiB) + 0.098447 seconds (46.49 k allocations: 5.053 MiB) - VarianceComponentModels.VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}(Array{Float64}(0,3), ([0.280777 0.279441 0.232208; 0.279441 0.28422 0.219831; 0.232208 0.219831 0.212832], [0.717266 0.66183 0.674206; 0.66183 0.715287 0.581891; 0.674206 0.581891 0.784183]), Array{Float64}(0,0), Char[], Float64[], -Inf, Inf) + VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}(Array{Float64}(0,3), ([0.28137 0.280153 0.232512; 0.280153 0.284916 0.220363; 0.232512 0.220363 0.212921], [0.717042 0.661484 0.674207; 0.661484 0.714964 0.581648; 0.674207 0.581648 0.784373]), Array{Float64}(0,0), Char[], Float64[], -Inf, Inf) @@ -956,16 +835,16 @@ trait57_model MM Algorithm Iter Objective -------- ------------- - 0 -1.470102e+04 - 1 -1.470102e+04 + 0 -1.470095e+04 + 1 -1.470095e+04 - 0.003006 seconds (21.01 k allocations: 1.551 MiB) + 0.548238 seconds (1.04 M allocations: 51.897 MiB, 5.51% gc time) - VarianceComponentModels.VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}(Array{Float64}(0,3), ([0.280777 0.279441 0.232208; 0.279441 0.28422 0.219831; 0.232208 0.219831 0.212832], [0.717266 0.66183 0.674206; 0.66183 0.715287 0.581891; 0.674206 0.581891 0.784183]), Array{Float64}(0,0), Char[], Float64[], -Inf, Inf) + VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}(Array{Float64}(0,3), ([0.28137 0.280153 0.232512; 0.280153 0.284916 0.220363; 0.232512 0.220363 0.212921], [0.717042 0.661484 0.674207; 0.661484 0.714964 0.581648; 0.674207 0.581648 0.784373]), Array{Float64}(0,0), Char[], Float64[], -Inf, Inf) @@ -983,136 +862,135 @@ trait57_model MM Algorithm Iter Objective -------- ------------- - 0 -3.024757e+04 - 1 -2.040300e+04 - 2 -1.656070e+04 - 3 -1.528529e+04 - 4 -1.490986e+04 - 5 -1.480638e+04 - 6 -1.477811e+04 - 7 -1.476968e+04 - 8 -1.476639e+04 - 9 -1.476444e+04 - 10 -1.476286e+04 - 20 -1.475000e+04 - 30 -1.474011e+04 - 40 -1.473248e+04 - 50 -1.472658e+04 - 60 -1.472200e+04 - 70 -1.471840e+04 - 80 -1.471555e+04 - 90 -1.471328e+04 - 100 -1.471145e+04 - 110 -1.470997e+04 - 120 -1.470875e+04 - 130 -1.470775e+04 - 140 -1.470691e+04 - 150 -1.470621e+04 - 160 -1.470562e+04 - 170 -1.470511e+04 - 180 -1.470469e+04 - 190 -1.470432e+04 - 200 -1.470400e+04 - 210 -1.470372e+04 - 220 -1.470348e+04 - 230 -1.470326e+04 - 240 -1.470308e+04 - 250 -1.470291e+04 - 260 -1.470276e+04 - 270 -1.470263e+04 - 280 -1.470251e+04 - 290 -1.470241e+04 - 300 -1.470231e+04 - 310 -1.470223e+04 - 320 -1.470215e+04 - 330 -1.470208e+04 - 340 -1.470201e+04 - 350 -1.470195e+04 - 360 -1.470190e+04 - 370 -1.470185e+04 - 380 -1.470180e+04 - 390 -1.470176e+04 - 400 -1.470172e+04 - 410 -1.470168e+04 - 420 -1.470165e+04 - 430 -1.470162e+04 - 440 -1.470159e+04 - 450 -1.470156e+04 - 460 -1.470153e+04 - 470 -1.470151e+04 - 480 -1.470149e+04 - 490 -1.470147e+04 - 500 -1.470145e+04 - 510 -1.470143e+04 - 520 -1.470141e+04 - 530 -1.470139e+04 - 540 -1.470138e+04 - 550 -1.470136e+04 - 560 -1.470135e+04 - 570 -1.470133e+04 - 580 -1.470132e+04 - 590 -1.470131e+04 - 600 -1.470130e+04 - 610 -1.470129e+04 - 620 -1.470128e+04 - 630 -1.470127e+04 - 640 -1.470126e+04 - 650 -1.470125e+04 - 660 -1.470124e+04 - 670 -1.470123e+04 - 680 -1.470122e+04 - 690 -1.470122e+04 - 700 -1.470121e+04 - 710 -1.470120e+04 - 720 -1.470120e+04 - 730 -1.470119e+04 - 740 -1.470118e+04 - 750 -1.470118e+04 - 760 -1.470117e+04 - 770 -1.470117e+04 - 780 -1.470116e+04 - 790 -1.470116e+04 - 800 -1.470115e+04 - 810 -1.470115e+04 - 820 -1.470114e+04 - 830 -1.470114e+04 - 840 -1.470114e+04 - 850 -1.470113e+04 - 860 -1.470113e+04 - 870 -1.470112e+04 - 880 -1.470112e+04 - 890 -1.470112e+04 - 900 -1.470111e+04 - 910 -1.470111e+04 - 920 -1.470111e+04 - 930 -1.470111e+04 - 940 -1.470110e+04 - 950 -1.470110e+04 - 960 -1.470110e+04 - 970 -1.470109e+04 - 980 -1.470109e+04 - 990 -1.470109e+04 - 1000 -1.470109e+04 - 1010 -1.470109e+04 - 1020 -1.470108e+04 - 1030 -1.470108e+04 - 1040 -1.470108e+04 - 1050 -1.470108e+04 - 1060 -1.470108e+04 - 1070 -1.470107e+04 - 1080 -1.470107e+04 - 1090 -1.470107e+04 - 1100 -1.470107e+04 - 1110 -1.470107e+04 - 1120 -1.470107e+04 + 0 -3.024417e+04 + 1 -2.040172e+04 + 2 -1.656053e+04 + 3 -1.528554e+04 + 4 -1.491023e+04 + 5 -1.480677e+04 + 6 -1.477849e+04 + 7 -1.477005e+04 + 8 -1.476675e+04 + 9 -1.476478e+04 + 10 -1.476318e+04 + 20 -1.475020e+04 + 30 -1.474023e+04 + 40 -1.473255e+04 + 50 -1.472662e+04 + 60 -1.472201e+04 + 70 -1.471839e+04 + 80 -1.471554e+04 + 90 -1.471326e+04 + 100 -1.471142e+04 + 110 -1.470993e+04 + 120 -1.470871e+04 + 130 -1.470770e+04 + 140 -1.470686e+04 + 150 -1.470616e+04 + 160 -1.470557e+04 + 170 -1.470506e+04 + 180 -1.470463e+04 + 190 -1.470426e+04 + 200 -1.470394e+04 + 210 -1.470366e+04 + 220 -1.470342e+04 + 230 -1.470321e+04 + 240 -1.470302e+04 + 250 -1.470285e+04 + 260 -1.470270e+04 + 270 -1.470257e+04 + 280 -1.470245e+04 + 290 -1.470234e+04 + 300 -1.470225e+04 + 310 -1.470216e+04 + 320 -1.470208e+04 + 330 -1.470201e+04 + 340 -1.470195e+04 + 350 -1.470189e+04 + 360 -1.470183e+04 + 370 -1.470178e+04 + 380 -1.470173e+04 + 390 -1.470169e+04 + 400 -1.470165e+04 + 410 -1.470162e+04 + 420 -1.470158e+04 + 430 -1.470155e+04 + 440 -1.470152e+04 + 450 -1.470149e+04 + 460 -1.470147e+04 + 470 -1.470144e+04 + 480 -1.470142e+04 + 490 -1.470140e+04 + 500 -1.470138e+04 + 510 -1.470136e+04 + 520 -1.470134e+04 + 530 -1.470132e+04 + 540 -1.470131e+04 + 550 -1.470129e+04 + 560 -1.470128e+04 + 570 -1.470127e+04 + 580 -1.470125e+04 + 590 -1.470124e+04 + 600 -1.470123e+04 + 610 -1.470122e+04 + 620 -1.470121e+04 + 630 -1.470120e+04 + 640 -1.470119e+04 + 650 -1.470118e+04 + 660 -1.470117e+04 + 670 -1.470116e+04 + 680 -1.470116e+04 + 690 -1.470115e+04 + 700 -1.470114e+04 + 710 -1.470113e+04 + 720 -1.470113e+04 + 730 -1.470112e+04 + 740 -1.470112e+04 + 750 -1.470111e+04 + 760 -1.470110e+04 + 770 -1.470110e+04 + 780 -1.470109e+04 + 790 -1.470109e+04 + 800 -1.470108e+04 + 810 -1.470108e+04 + 820 -1.470108e+04 + 830 -1.470107e+04 + 840 -1.470107e+04 + 850 -1.470106e+04 + 860 -1.470106e+04 + 870 -1.470106e+04 + 880 -1.470105e+04 + 890 -1.470105e+04 + 900 -1.470105e+04 + 910 -1.470104e+04 + 920 -1.470104e+04 + 930 -1.470104e+04 + 940 -1.470103e+04 + 950 -1.470103e+04 + 960 -1.470103e+04 + 970 -1.470103e+04 + 980 -1.470102e+04 + 990 -1.470102e+04 + 1000 -1.470102e+04 + 1010 -1.470102e+04 + 1020 -1.470102e+04 + 1030 -1.470101e+04 + 1040 -1.470101e+04 + 1050 -1.470101e+04 + 1060 -1.470101e+04 + 1070 -1.470101e+04 + 1080 -1.470101e+04 + 1090 -1.470100e+04 + 1100 -1.470100e+04 + 1110 -1.470100e+04 - 0.794377 seconds (168.12 k allocations: 15.640 MiB, 0.80% gc time) + 0.794444 seconds (135.15 k allocations: 13.553 MiB, 1.62% gc time) - VarianceComponentModels.VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}(Array{Float64}(0,3), ([0.2808 0.279454 0.232256; 0.279454 0.284312 0.219977; 0.232256 0.219977 0.213052], [0.717243 0.661816 0.674158; 0.661816 0.715193 0.581746; 0.674158 0.581746 0.783965]), Array{Float64}(0,0), Char[], Float64[], -Inf, Inf) + VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}(Array{Float64}(0,3), ([0.281394 0.280169 0.232564; 0.280169 0.285001 0.220511; 0.232564 0.220511 0.213147], [0.717018 0.661467 0.674155; 0.661467 0.714877 0.581502; 0.674155 0.581502 0.784149]), Array{Float64}(0,0), Char[], Float64[], -Inf, Inf) @@ -1128,8 +1006,8 @@ h, hse = heritability(trait57_model.Σ, Σcov) 2×3 Array{Float64,2}: - 0.281351 0.284453 0.213689 - 0.0778252 0.077378 0.084084 + 0.281842 0.285036 0.213725 + 0.0777056 0.0772501 0.0840114 @@ -1145,7 +1023,7 @@ traitall_model = VarianceComponentModel(cg10kdata_rotated) @time mle_fs!(traitall_model, cg10kdata_rotated; solver=:Ipopt, verbose=true) ``` - This is Ipopt version 3.12.4, running with linear solver mumps. + This is Ipopt version 3.12.10, running with linear solver mumps. NOTE: Other linear solvers might be more efficient (see Ipopt documentation). Number of nonzeros in equality constraint Jacobian...: 0 @@ -1163,43 +1041,49 @@ traitall_model = VarianceComponentModel(cg10kdata_rotated) inequality constraints with only upper bounds: 0 iter objective inf_pr inf_du lg(mu) ||d|| lg(rg) alpha_du alpha_pr ls - 0 1.3113371e+05 0.00e+00 1.00e+02 0.0 0.00e+00 - 0.00e+00 0.00e+00 0 - 5 8.2233766e+04 0.00e+00 6.03e+02 -11.0 2.32e+00 - 1.00e+00 1.00e+00f 1 MaxS - 10 1.1960260e+05 0.00e+00 8.76e+02 -11.0 6.20e+01 -5.4 1.00e+00 1.00e+00h 1 MaxS - 15 2.4416551e+05 0.00e+00 2.50e+02 -11.0 8.69e+02 -7.8 1.00e+00 1.00e+00f 1 MaxS + 0 1.3111983e+05 0.00e+00 1.00e+02 0.0 0.00e+00 - 0.00e+00 0.00e+00 0 + 5 8.2228529e+04 0.00e+00 6.03e+02 -11.0 2.42e+00 - 1.00e+00 1.00e+00f 1 MaxS + 10 1.2570490e+05 0.00e+00 9.38e+02 -11.0 6.72e+01 -5.4 1.00e+00 1.00e+00h 1 MaxS - DomainError: - log will only return a complex result if called with a complex argument. Try log(complex(x)). + PosDefException: matrix is not positive definite; Cholesky factorization failed. Stacktrace: - [1] nan_dom_err at ./math.jl:300 [inlined] + [1] chkposdef at /Users/osx/buildbot/slave/package_osx64/build/usr/share/julia/stdlib/v0.7/LinearAlgebra/src/lapack.jl:50 [inlined] + + [2] sygvd!(::Int64, ::Char, ::Char, ::Array{Float64,2}, ::Array{Float64,2}) at /Users/osx/buildbot/slave/package_osx64/build/usr/share/julia/stdlib/v0.7/LinearAlgebra/src/lapack.jl:5075 + + [3] eigen! at /Users/osx/buildbot/slave/package_osx64/build/usr/share/julia/stdlib/v0.7/LinearAlgebra/src/symmetric.jl:638 [inlined] - [2] log at ./math.jl:419 [inlined] + [4] eigen(::Symmetric{Float64,Array{Float64,2}}, ::Symmetric{Float64,Array{Float64,2}}) at /Users/osx/buildbot/slave/package_osx64/build/usr/share/julia/stdlib/v0.7/LinearAlgebra/src/eigen.jl:383 - [3] logdet(::Array{Float64,2}) at ./linalg/generic.jl:1244 + [5] TwoVarCompModelRotate(::VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}) at /Users/juhyun-kim/.julia/packages/VarianceComponentModels/Mh6LK/src/VarianceComponentModels.jl:119 - [4] VarianceComponentModels.TwoVarCompModelRotate(::VarianceComponentModels.VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}) at /Users/huazhou/.julia/v0.6/VarianceComponentModels/src/VarianceComponentModels.jl:127 + [6] logpdf at /Users/juhyun-kim/.julia/packages/VarianceComponentModels/Mh6LK/src/two_variance_component.jl:62 [inlined] - [5] eval_f(::VarianceComponentModels.TwoVarCompOptProb{VarianceComponentModels.VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}},VarianceComponentModels.TwoVarCompVariateRotate{Float64,Array{Float64,2},Array{Float64,2}},Array{Float64,2},Array{Float64,1},VarianceComponentModels.VarianceComponentAuxData{Array{Float64,2},Array{Float64,1}}}, ::Array{Float64,1}) at /Users/huazhou/.julia/v0.6/VarianceComponentModels/src/two_variance_component.jl:683 + [7] eval_f(::VarianceComponentModels.TwoVarCompOptProb{VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}},TwoVarCompVariateRotate{Float64,Array{Float64,2},Array{Float64,2}},Array{Float64,2},Array{Float64,1},VarianceComponentAuxData{Array{Float64,2},Array{Float64,1}}}, ::Array{Float64,1}) at /Users/juhyun-kim/.julia/packages/VarianceComponentModels/Mh6LK/src/two_variance_component.jl:731 - [6] (::Ipopt.#eval_f_cb#4{VarianceComponentModels.TwoVarCompOptProb{VarianceComponentModels.VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}},VarianceComponentModels.TwoVarCompVariateRotate{Float64,Array{Float64,2},Array{Float64,2}},Array{Float64,2},Array{Float64,1},VarianceComponentModels.VarianceComponentAuxData{Array{Float64,2},Array{Float64,1}}}})(::Array{Float64,1}) at /Users/huazhou/.julia/v0.6/Ipopt/src/IpoptSolverInterface.jl:53 + [8] (::getfield(Ipopt, Symbol("#eval_f_cb#6")){VarianceComponentModels.TwoVarCompOptProb{VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}},TwoVarCompVariateRotate{Float64,Array{Float64,2},Array{Float64,2}},Array{Float64,2},Array{Float64,1},VarianceComponentAuxData{Array{Float64,2},Array{Float64,1}}}})(::Array{Float64,1}) at /Users/juhyun-kim/.julia/packages/Ipopt/vosfb/src/MPB_wrapper.jl:64 - [7] eval_f_wrapper(::Int32, ::Ptr{Float64}, ::Int32, ::Ptr{Float64}, ::Ptr{Void}) at /Users/huazhou/.julia/v0.6/Ipopt/src/Ipopt.jl:89 + [9] eval_f_wrapper(::Int32, ::Ptr{Float64}, ::Int32, ::Ptr{Float64}, ::Ptr{Nothing}) at /Users/juhyun-kim/.julia/packages/Ipopt/vosfb/src/Ipopt.jl:128 - [8] solveProblem(::Ipopt.IpoptProblem) at /Users/huazhou/.julia/v0.6/Ipopt/src/Ipopt.jl:304 + [10] solveProblem(::Ipopt.IpoptProblem) at /Users/juhyun-kim/.julia/packages/Ipopt/vosfb/src/Ipopt.jl:346 - [9] optimize!(::Ipopt.IpoptMathProgModel) at /Users/huazhou/.julia/v0.6/Ipopt/src/IpoptSolverInterface.jl:120 + [11] optimize!(::Ipopt.IpoptMathProgModel) at /Users/juhyun-kim/.julia/packages/Ipopt/vosfb/src/MPB_wrapper.jl:141 - [10] #mle_fs!#29(::Int64, ::Symbol, ::Symbol, ::Bool, ::Function, ::VarianceComponentModels.VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}, ::VarianceComponentModels.TwoVarCompVariateRotate{Float64,Array{Float64,2},Array{Float64,2}}) at /Users/huazhou/.julia/v0.6/VarianceComponentModels/src/two_variance_component.jl:893 + [12] macro expansion at ./simdloop.jl:67 [inlined] - [11] (::VarianceComponentModels.#kw##mle_fs!)(::Array{Any,1}, ::VarianceComponentModels.#mle_fs!, ::VarianceComponentModels.VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}, ::VarianceComponentModels.TwoVarCompVariateRotate{Float64,Array{Float64,2},Array{Float64,2}}) at ./:0 + [13] #mle_fs!#27(::Int64, ::Symbol, ::Symbol, ::Bool, ::Function, ::VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}, ::TwoVarCompVariateRotate{Float64,Array{Float64,2},Array{Float64,2}}) at /Users/juhyun-kim/.julia/packages/VarianceComponentModels/Mh6LK/src/two_variance_component.jl:942 - [12] include_string(::String, ::String) at ./loading.jl:515 + [14] (::getfield(VarianceComponentModels, Symbol("#kw##mle_fs!")))(::NamedTuple{(:solver, :verbose),Tuple{Symbol,Bool}}, ::typeof(mle_fs!), ::VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}, ::TwoVarCompVariateRotate{Float64,Array{Float64,2},Array{Float64,2}}) at ./none:0 + + [15] top-level scope at util.jl:156 + + [16] top-level scope at In[92]:3 From the output we can see the Fisher scoring algorithm ran into some numerical issues. Let's try the **MM algorithm**. @@ -1216,133 +1100,132 @@ traitall_model = VarianceComponentModel(cg10kdata_rotated) MM Algorithm Iter Objective -------- ------------- - 0 -1.311337e+05 - 1 -8.002108e+04 - 2 -5.806935e+04 - 3 -4.926111e+04 - 4 -4.611059e+04 - 5 -4.511606e+04 - 6 -4.482679e+04 - 7 -4.474294e+04 - 8 -4.471496e+04 - 9 -4.470174e+04 - 10 -4.469246e+04 - 20 -4.462243e+04 - 30 -4.456888e+04 - 40 -4.452774e+04 - 50 -4.449601e+04 - 60 -4.447134e+04 - 70 -4.445199e+04 - 80 -4.443665e+04 - 90 -4.442436e+04 - 100 -4.441442e+04 - 110 -4.440630e+04 - 120 -4.439961e+04 - 130 -4.439405e+04 - 140 -4.438938e+04 - 150 -4.438544e+04 - 160 -4.438210e+04 - 170 -4.437923e+04 - 180 -4.437676e+04 - 190 -4.437463e+04 - 200 -4.437277e+04 - 210 -4.437115e+04 - 220 -4.436972e+04 - 230 -4.436846e+04 - 240 -4.436735e+04 - 250 -4.436636e+04 - 260 -4.436548e+04 - 270 -4.436469e+04 - 280 -4.436399e+04 - 290 -4.436335e+04 - 300 -4.436278e+04 - 310 -4.436226e+04 - 320 -4.436179e+04 - 330 -4.436137e+04 - 340 -4.436098e+04 - 350 -4.436063e+04 - 360 -4.436030e+04 - 370 -4.436001e+04 - 380 -4.435974e+04 - 390 -4.435949e+04 - 400 -4.435926e+04 - 410 -4.435905e+04 - 420 -4.435886e+04 - 430 -4.435868e+04 - 440 -4.435851e+04 - 450 -4.435836e+04 - 460 -4.435822e+04 - 470 -4.435809e+04 - 480 -4.435797e+04 - 490 -4.435785e+04 - 500 -4.435775e+04 - 510 -4.435765e+04 - 520 -4.435756e+04 - 530 -4.435747e+04 - 540 -4.435739e+04 - 550 -4.435732e+04 - 560 -4.435725e+04 - 570 -4.435718e+04 - 580 -4.435712e+04 - 590 -4.435706e+04 - 600 -4.435701e+04 - 610 -4.435696e+04 - 620 -4.435691e+04 - 630 -4.435687e+04 - 640 -4.435683e+04 - 650 -4.435679e+04 - 660 -4.435675e+04 - 670 -4.435671e+04 - 680 -4.435668e+04 - 690 -4.435665e+04 - 700 -4.435662e+04 - 710 -4.435659e+04 - 720 -4.435657e+04 - 730 -4.435654e+04 - 740 -4.435652e+04 - 750 -4.435649e+04 - 760 -4.435647e+04 - 770 -4.435645e+04 - 780 -4.435643e+04 - 790 -4.435642e+04 - 800 -4.435640e+04 - 810 -4.435638e+04 - 820 -4.435637e+04 - 830 -4.435635e+04 - 840 -4.435634e+04 - 850 -4.435633e+04 - 860 -4.435631e+04 - 870 -4.435630e+04 - 880 -4.435629e+04 - 890 -4.435628e+04 - 900 -4.435627e+04 - 910 -4.435626e+04 - 920 -4.435625e+04 - 930 -4.435624e+04 - 940 -4.435623e+04 - 950 -4.435622e+04 - 960 -4.435621e+04 - 970 -4.435621e+04 - 980 -4.435620e+04 - 990 -4.435619e+04 - 1000 -4.435619e+04 - 1010 -4.435618e+04 - 1020 -4.435617e+04 - 1030 -4.435617e+04 - 1040 -4.435616e+04 - 1050 -4.435616e+04 - 1060 -4.435615e+04 - 1070 -4.435615e+04 - 1080 -4.435614e+04 - 1090 -4.435614e+04 + 0 -1.311198e+05 + 1 -8.001595e+04 + 2 -5.806836e+04 + 3 -4.926167e+04 + 4 -4.611163e+04 + 5 -4.511722e+04 + 6 -4.482795e+04 + 7 -4.474405e+04 + 8 -4.471603e+04 + 9 -4.470277e+04 + 10 -4.469345e+04 + 20 -4.462308e+04 + 30 -4.456930e+04 + 40 -4.452802e+04 + 50 -4.449620e+04 + 60 -4.447148e+04 + 70 -4.445209e+04 + 80 -4.443672e+04 + 90 -4.442442e+04 + 100 -4.441448e+04 + 110 -4.440635e+04 + 120 -4.439966e+04 + 130 -4.439409e+04 + 140 -4.438943e+04 + 150 -4.438549e+04 + 160 -4.438215e+04 + 170 -4.437928e+04 + 180 -4.437682e+04 + 190 -4.437469e+04 + 200 -4.437283e+04 + 210 -4.437121e+04 + 220 -4.436978e+04 + 230 -4.436853e+04 + 240 -4.436742e+04 + 250 -4.436643e+04 + 260 -4.436555e+04 + 270 -4.436476e+04 + 280 -4.436406e+04 + 290 -4.436342e+04 + 300 -4.436285e+04 + 310 -4.436234e+04 + 320 -4.436187e+04 + 330 -4.436144e+04 + 340 -4.436106e+04 + 350 -4.436071e+04 + 360 -4.436039e+04 + 370 -4.436009e+04 + 380 -4.435982e+04 + 390 -4.435957e+04 + 400 -4.435935e+04 + 410 -4.435914e+04 + 420 -4.435895e+04 + 430 -4.435877e+04 + 440 -4.435860e+04 + 450 -4.435845e+04 + 460 -4.435831e+04 + 470 -4.435818e+04 + 480 -4.435806e+04 + 490 -4.435794e+04 + 500 -4.435784e+04 + 510 -4.435774e+04 + 520 -4.435765e+04 + 530 -4.435757e+04 + 540 -4.435749e+04 + 550 -4.435741e+04 + 560 -4.435734e+04 + 570 -4.435728e+04 + 580 -4.435722e+04 + 590 -4.435716e+04 + 600 -4.435711e+04 + 610 -4.435706e+04 + 620 -4.435701e+04 + 630 -4.435696e+04 + 640 -4.435692e+04 + 650 -4.435688e+04 + 660 -4.435685e+04 + 670 -4.435681e+04 + 680 -4.435678e+04 + 690 -4.435675e+04 + 700 -4.435672e+04 + 710 -4.435669e+04 + 720 -4.435666e+04 + 730 -4.435664e+04 + 740 -4.435662e+04 + 750 -4.435659e+04 + 760 -4.435657e+04 + 770 -4.435655e+04 + 780 -4.435653e+04 + 790 -4.435652e+04 + 800 -4.435650e+04 + 810 -4.435648e+04 + 820 -4.435647e+04 + 830 -4.435645e+04 + 840 -4.435644e+04 + 850 -4.435643e+04 + 860 -4.435641e+04 + 870 -4.435640e+04 + 880 -4.435639e+04 + 890 -4.435638e+04 + 900 -4.435637e+04 + 910 -4.435636e+04 + 920 -4.435635e+04 + 930 -4.435634e+04 + 940 -4.435633e+04 + 950 -4.435633e+04 + 960 -4.435632e+04 + 970 -4.435631e+04 + 980 -4.435630e+04 + 990 -4.435630e+04 + 1000 -4.435629e+04 + 1010 -4.435628e+04 + 1020 -4.435628e+04 + 1030 -4.435627e+04 + 1040 -4.435627e+04 + 1050 -4.435626e+04 + 1060 -4.435626e+04 + 1070 -4.435625e+04 + 1080 -4.435625e+04 - 3.551301 seconds (178.42 k allocations: 70.115 MiB, 0.42% gc time) + 3.423491 seconds (131.35 k allocations: 65.559 MiB, 0.86% gc time) - (-44356.138529861186, VarianceComponentModels.VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}(Array{Float64}(0,13), ([0.272384 0.190358 … -0.128222 -0.0980655; 0.190358 0.21692 … -0.0689912 -0.0444349; … ; -0.128222 -0.0689912 … 0.118227 0.0909188; -0.0980655 -0.0444349 … 0.0909188 0.107456], [0.724562 0.56992 … -0.0590518 -0.124939; 0.56992 0.782639 … 0.0238629 0.0475408; … ; -0.0590518 0.0238629 … 0.880671 0.550889; -0.124939 0.0475408 … 0.550889 0.891929]), Array{Float64}(0,0), Char[], Float64[], -Inf, Inf), ([0.0111619 0.0131088 … 0.0128956 0.0127641; 0.0131091 0.0151759 … 0.017162 0.0171466; … ; 0.0128956 0.017162 … 0.0173994 0.0182002; 0.0127643 0.0171461 … 0.0182003 0.0187848], [0.0112235 0.0133094 … 0.0130111 0.0127861; 0.01331 0.0158262 … 0.017867 0.017798; … ; 0.013011 0.0178666 … 0.0179487 0.0187579; 0.012786 0.0177975 … 0.0187578 0.0193328]), [0.000124587 7.24074e-5 … -3.35716e-7 -1.40982e-5; 7.24411e-5 0.000171849 … -2.05381e-5 -3.17975e-6; … ; -3.60221e-7 -2.05683e-5 … 0.000351859 -1.5168e-5; -1.40799e-5 -3.16738e-6 … -1.51641e-5 0.000373756], Array{Float64}(0,13), Array{Float64}(0,0)) + (-44356.24416253091, VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}(Array{Float64}(0,13), ([0.272112 0.190023 … -0.128464 -0.0976418; 0.190023 0.216814 … -0.0687833 -0.04341; … ; -0.128464 -0.0687833 … 0.116994 0.0900933; -0.0976418 -0.04341 … 0.0900933 0.105876], [0.725183 0.570497 … -0.0589748 -0.125486; 0.570497 0.783023 … 0.0235685 0.0464638; … ; -0.0589748 0.0235685 … 0.882056 0.551829; -0.125486 0.0464638 … 0.551829 0.893642]), Array{Float64}(0,0), Char[], Float64[], -Inf, Inf), ([0.0111615 0.0131072 … 0.0128913 0.0127584; 0.0131071 0.0151656 … 0.017153 0.0171365; … ; 0.0128911 0.017153 … 0.0174174 0.018212; 0.0127593 0.0171355 … 0.0182119 0.0188], [0.0112229 0.0133031 … 0.013004 0.0127777; 0.0133033 0.015806 … 0.0178516 0.0177826; … ; 0.0130042 0.0178517 … 0.0179556 0.0187634; 0.0127776 0.0177822 … 0.0187633 0.0193475]), [0.00012458 7.242e-5 … -3.70019e-7 -1.40553e-5; 7.23929e-5 0.000171795 … -2.04581e-5 -3.19854e-6; … ; -3.8356e-7 -2.04665e-5 … 0.000352066 -1.46068e-5; -1.40476e-5 -3.19013e-6 … -1.46022e-5 0.000374325], Array{Float64}(0,13), Array{Float64}(0,0)) @@ -1352,7 +1235,46 @@ It converges after ~1000 iterations. ```julia -#using JLD -#@save "copd.jld" -#whos() +#using JLD2, FileIO +#save("copd.jld2") +#varinfo() ``` + + + + +| name | size | summary | +|:----------------- | ------------:|:-------------------------------------------------------------------------------------- | +| Base | | Module | +| Core | | Module | +| Main | | Module | +| Plots | 22.448 MiB | Module | +| Y | 677.461 KiB | 6670×13 Array{Float64,2} | +| _ | 240 bytes | Tuple{Array{Float64,2},Array{Float64,2}} | +| cg10k | 1022.983 MiB | 6670×630860 SnpArray | +| cg10k_trait | 1.443 MiB | 6670×15 DataFrame | +| cg10kdata | 679.508 MiB | VarianceComponentVariate{Float64,2,Array{Float64,2},Array{Float64,2},Array{Float64,2}} | +| cg10kdata_rotated | 340.136 MiB | TwoVarCompVariateRotate{Float64,Array{Float64,2},Array{Float64,2}} | +| h | 64 bytes | 3-element Array{Float64,1} | +| hST | 144 bytes | 13-element Array{Float64,1} | +| hST_se | 144 bytes | 13-element Array{Float64,1} | +| hse | 64 bytes | 3-element Array{Float64,1} | +| maf_cg10k | 4.813 MiB | 630860-element Array{Float64,1} | +| missings_by_snp | 4.813 MiB | 1×630860 Array{Int64,2} | +| mp | 28.748 MiB | 6670×630860 SparseMatrixCSC{Bool,Int32} | +| people | 8 bytes | Int64 | +| snps | 8 bytes | Int64 | +| startupfile | 57 bytes | String | +| trait57_data | 339.627 MiB | TwoVarCompVariateRotate{Float64,Array{Float64,2},Array{Float64,2}} | +| trait57_model | 472 bytes | VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}} | +| traitall_model | 2.961 KiB | VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}} | +| traitidx | 16 bytes | 3-element UnitRange{Int64} | +| Σa | 6.844 KiB | 13×13 Array{Array{Float64,2},2} | +| Σcov | 2.570 KiB | 18×18 Array{Float64,2} | +| Σe | 6.844 KiB | 13×13 Array{Array{Float64,2},2} | +| Φgrm | 339.423 MiB | 6670×6670 Array{Float64,2} | +| σ2a | 144 bytes | 13-element Array{Float64,1} | +| σ2e | 144 bytes | 13-element Array{Float64,1} | + + + diff --git a/docs/src/man/heritability_hist1.png b/docs/src/man/heritability_hist1.png new file mode 100644 index 0000000000000000000000000000000000000000..9c74baaea874dd0d098f26342ccb28093d5ba9d3 GIT binary patch literal 12699 zcmeHtcT`jTwrxO^C@4WGN-v+HRHaHU0)A4YgLDCafy6M74s&_j_fBAw8S z)X+ka7Fu{ezH{C^_q=<@J$H=v@7a#AiP=f^{*|@XTyxG3s?VQZzeaNn0)bqA_NS~m z1VZ=%0wLIg5`uT^dD{ZPpDU({Ph}zJ7yr^4a$_J6Cde~cX-$u0?6jwkhHK;bj>F*H z>9qbIdg-K0U*u#7gj8jR3tm159lb;JiQGcf;9Ur7R456lYP=djXE#|kt4N>}NA`Wv z@cwH7v|(vbO^U29vDq#T$du$+#@c1K&oTXX#h)GfiZh;)H>M^#OB^jH*C)mFq#+Ox zh?I7Rlo0sCL>>fzLScRt#hgrcl!;AE+_R+txyF7 zWZb_I!V3NmMeXd4+ZtZ)akw1-naS#rpXr324ta8nuz`AGA>Jn4C)(MtVF z=rMONj-q5>V4$TfaF&Fmu?477S@b4%_4OUn2_^NAi3?^fZrthnyrz1^z;z$rcfjxO z4uL%+O6Pn(`_-+N%7irBkyfX$@@w$r`RhfgtGFEv#Tf(f#7l3#F+*Rg890`&1qWtm zi5|9$`#jpJlX4+~NWT;6hCs69VHuAa&kiPUJTr_i;xw$|@4rqc8s6_}d!(GoHjYQMDm3NK!~ zV5QoaMjcL6+MLZqlQA;}5#OM@Eol3~)zvkP*6IU7;dk^o#pnTVU)Z#%_La!dCf2le zj{Em(){9a2lcsag{zxKuKRr$Mt-#%K0xGwzLLz+w50fx{fc}u~hT;-%H+wpQKp~260B1~8|c_aPj za-^!SK=9-((zv!;DnH{NW!j7uzq2QTz*=N*!w(kW6mKGNN^lX0laQo`!FO&15f+=l!=jiI+ zoN~VSUM#D_T_^z&<#{TN7RBAWcjb`$E-R|7B@CX6pWYYeR#fl{2naAUn}Em2&em&+ zeakdp5md9T0zX&6qAu6*K^!p{iy*}~ZbPJbPnlV#m?8qfAnvudw>q@5vm+C_?P#1d zVxZ#X<@M&x8*S~8LR};kk5RqjoXlvVfQ~70+-+lQ&TD?<4YqRB$aCrIev_IjdSpai z792I_@oFExPb0T_zB*XN{xnCX3XO!a%^|GIUg(XXd~9rN^vAfYlP4zh3 zSyI3#dby#dDZ`4p6W6|G>m5fx-_E0{E3`HJ1P$$F25?ncPu zcN@jHGd_-7D_?Y+S|~N#(;7_bvF-C11R^s2TWDzL%d0KqtnwZll+@H@cX$`kI>y+E z%Jqmld~mO|{KHv)W9TEhi3+t$`A9}d332gh)3To_qN9^u*GNcWVq$_}U0q#ra&mFD z_t@Fj$oz4wm&nBWagz(p=E(NWPGT&djM}M zD{g~z=GveVh(aLl#XJxZ5dk|q9&If^6RA9YmxYyeTK)#T#BpSPNzK}3PNjBJ5vIZE zc+ZMnaPi44wo>vvZ{*K!?=HDxCzk5#EeA6E&W?q7cy!AbdquZ?c6B{?@L;^uBrsYa z4`KcAb}}`EoYm%g0;U%(8+VVV5Up-4H_UCKXi48+y>$=lkPZP}r};Jkn~|F5?@K|zduxaK1%ticbguAUtIRn9p-lXa7kX{m8q7%9>h zdi{f9G%cq$MUqD&xf6^)GG?WRsfReD99o}R7+sXo8EUdI`_fdtVS zLN-cYea<#rX9p99lP@nVO~2{pLZi`#Gu{qHMn=&L;>$frLT(!`EL_{eC~IqLkB*Ma zI-?>ZBk2XKzf|vM<>Z*Zdd2yolnFc$7ngEK@0{_CBu}*D%y>c)MwgeSZNnmUiH;Rk z@%}n(75FA&XVuiA z7qlG<=V^>jOw=b7Mt7Z;8LPZ zvFBF9q00&#UQ5Hr+Bb=zC1ZzIees}4*?&(h00z}NoO13XX7u?*+On~+;k3|E8_;i# zw418gNpfgB+UaCu^xYeXGxF+kb>shoUlCT<5Ik{Y9N7HO_ZWnDF3QkzZ)KpDBt^t^ zZ8$$BdcVSI$b5IX2k$`bMe`2F{^^u}*Y96<* zZrBNdm*3ukC61yOjg5fH=~LVEj6;4*bz zwH4*zP#NQ%!&NE6BE3o)eoIxbI7&)N=lHXAc$03E-&rT4|HhB{I*X@I<}C&${8m!4(HPN%D_nQ$F<&E%8FBkBNHe^2})v*n)& zw{G3q&6Pas5~u;|TWpA`^MhLAw&!x6ry#uD_;KVx&^HLx;G@P-0d}@Vbs@BIWL+u#UeysxXv5C(_%TdY76nQ`0&6~HBP&1LZC1?7o*XKtZ%vV-Q z)6-;NEUyeXABvJTO~s|FiQwj<*`BXWll6;H)$g^S&uETB39h@3f@EAUS`~0vI#iX) z|ZC3NEp`;k0hgUm2HB# z)k=SQ;FarxwTuwya3)#Bz1=wC=kSxwOrO561)RmMwarx-2saV49!tCVpN$i5W^LeK zx3Bw;G|@wvBbbQZGTo%7Pf1Ad+{^1YT>tT_C79&c0nEf(CSHI1Y4wa3N53IX`t|GA zwzh@K4h2X3NQ=I@Spx)F01OiFr;E#8jEcma74FktQ?Aw($4_-s_cb}{k_FJj34}DFeeVfv&|`NJF>BH-K-;G zpos%QAPajBAH_G$2gfbTwRL#MA{6?1oi_LCh&h}?6JVLQ=WG9n8v-Wt)jjIPPyCd>;B3P zvyBaSLjp^CBIfO;pY3c1m9$(e#h_%{`QX3S_T37S3YUcuy6)9#8Mszs9B~J7$A^Sc zjP(HO{AXNOLQWW$!+ua-Uw`ToX?)DZ!GSu*pX@taecJ>J`j(AO>``f|5)otX&CJkd zn0f{yWR7Ga;#RIqC}3;6J8|g9b6;@5jK=u*8dDG~NJ=9`!dK5(jFg1%ce4UN)yDp9P4vu>!_29*@j-+_S=K-5|P$J2sykQC&; zo42QDt@#6X_8+f!SEtCB_&_jD2gis?&J0N87g)%Z=`Hsj?Sfq(u1HBqIa%#+SoK46 zvnJDZeFSfN^Wzn1=t^<@_kQmh*JEOu?2TIeDt`lY?K$0X_~B}o!qdgqhGL4#TN3!? zaNo1z{ax?ex1>-RrufsE!|?AbJl7^XwIkR_=nl_%#60RP^7l{Q^~~PsHzfxnTXkx{ zl_bDhJ5+Rp_Z+T#gly4VhWrWU7@RaRrM@F4Y;WaA(9qLkb;GXR@)WJ2A-!^jB9yr- z4Gj`gl@y3^&R2}(93X9JYikQ(@X8xcx)i`54XuERJ5Z#YSVz!CIKS!h zx>10po}~7ZhYu}m#h@P{;@cgYs*NdHO{24?_Q4V&6$E=g1t+HvKPRLe2!=De* zuTCEHHmh)X0@OtnX0S7jGN`Gx zekg>NH)(`Nm4}G=?YXAD)2|c8z^ctLFZwt*LDe&_URNQ{Ppfk3?zd*TB zK2wluIYi&+=G>>vg(Zt{A@Rx5s&^gW&ZoH$;zV0 zVo&EWMUWZLFHYStod|^sSyN;zxNcpxbOSg3Wmfc{r1GB2T83`D*^^jugzS3nzVHlK7TryQ&99=xXtVIqQkA(aPJs>9=GJX~-A zZlocgFuQE9e>S(gNy0@Y4P|P(J%6)JOOFx#IEklCcY0F#`+{@fKeewrG|Uj(5GctA z1cF}??0VGPds>%G;y93X-U_0o?aN;9E6Qg+5}8;aaSFn1Ap3G8#PvZsGP4VQdM}zz z7IuX>0QQywVBlq3r>D1eeS-ot(86j9V;q~xvB#N*pkZY6pR@ghVWA4YTW?q$_TgN` zOc6CB=4m9_Cm^76J13i)|?f0uW2?~cCLyAlAC>aI3LudVtg zTR*hSN@=BFlb7J0unF!@;M}PuBp>SVt*`%Hl{eQU=<|DhDmB`-Ol%TqD!Q|8oSHgU z!c>@_U;BH)gqqs`3}Daw=i`T%KHfFf4lHps~oSqJs}!Dm1v~ zz*gY{Iz^#bFI12Ex~@9q>#oo3d=|aKO&q?kAc)?_6_0^p6sDB@z29q@_ifVL}I@7sCH>nmy>g_nKMc;K69ifyLTTdnjt~2lu*JXy_YSogOX0 zB*RpO-c-rdn3zmmL}8)NN_xor!joT}$)K1+AEM3iqsL^8_)X$I^?2!1tK}gb1E*k% z{PGRc{mhbZT-0Xpu4MR=QIE-3(`sr;D<8u0`ueGE+Sfh_xRB0+oH7-<9 zRJ_RjapsaoSPQI|pZ549k|L%06o0abCT;1sFY0`X+ku}6m=5Y0a$kvbSUF~AXaC6x zhn<~wBPn(RNBEZ0E(Q);agDjXlZPo?MP0{FDWK&l?bLgwN(2<{G#$ngM}_BF^m(2f zQ5)}Y>dMvJ{?&Ko5=fqJe`QbDq1>?S+}sD8;~NvkG?;S+h3Dg2J|k}3tnlno&%piF zbX^dUuzG}ISe#F(DDUCZK6Sey-6jc^4E{#z=g^)l%j?2mMu~XP_GJ-?X3uKnGOLXF z68c}BKI8%UPK=c_D=aL`N<~s%)$s>XQ`1rXS)J~AEJR9>$b`vvr=6P7@3-#$N}PZ- z+fXabPC5fb>JoVn0oBYJ@1vJrKpA1U;|f8`U^@1UTdz~97d1SKaY zA1Kh_NjV{a3=>PQ2t9FVI{6vQ8#r+GtO_y4YNtgVMsjJ867o$bpx@#7GGi5|7;}a` ziuy-yh3*?OCJ{o20S&+v7qrYjojQ_U@1>2|G5T!3!{C|&i7DHQ{Usp~>VVQCt*@GB zJqEqsJbhtHN{!PLx&BZ8cHx1b3ADh&BSn67Rbe(bCeca6=&J>=l8fH8nUuuN3F!Uv*)Hyj7)ak^axj3Lyk)!b}&WBDT{8dg(eh zEX|5M1j5W26!?&j&$%s>{7RrS)a2pqpXSbKqJ)qjI1FDrP+rc9xg6M>hOmB_L;wl8 ze=f1NQYm7s51Kb$=A574uZCoPSi~anFxo&larIb zeg$t70``r^pr%xMeW$9XIybPgveKb`r{gK3@1mfD-zg3dIt>KXh$wK^kXEoj>1sAp z$iaSJ_R09(eN^Sl=5k&~G#{uv3`;NjgHIli{q(=8f($=5cKD7VS^5=Bc zRusj{D-Iz;*|KtS0>{l&=@A_rslWD|VNtgqwLGmXEqy2~tPA~=oNS<}xk#5RXg86k zme~=_m*x4&JAtYMysSIWuDyN_l1f|V}$6wT1gk(YE! zrtb=ES+==lZ)$<<2CFc4*U3*B(q3XE`XsAA3}-nr5r|zL5|F{@W0d%`X`qlP7zC5@Wlb;UOu%E(<$OO;>yE4|+ z*7EZ25r{i??pRq_^)^#-XfG}=FJmw~FG^o_$8t%GWY#{0K$O#;!s?8+lSJJ&K?7gs zvib&edcw}jkBLz(tL5PL7Z(?~^s4}B!@Y=Jo`*pA<&gFAD+fmUqyCc>R!GlX^J)&j zDol8$Yk&q(#PxR>Lr*&a#4-$q2Q305G3qa<`0vx-G4sDC;{SglxPk)v=Ny$1cwcY- zY6iSqLPEkjdtwOev&>_t`^K0Fa{ORxHZ>tZ7%)6i)t4a9XCQg%7`mb}M;ri``st z16lV$P~dit(5%nFxJ7GAi`U+Y+685PaiyzI`te`Xvsh~O*Vx#coSb`q`~ebnPDRD= zA%a2`k0pZQM@{1IllBh~MC z^;yh4fsLxOwfXjNfKFLhWHNIQGjY2{zspoxkF3+3IBZ|_q;~U1Rp)QJvK=*vme^?0 zA|KIPl*~<9zh6GBRY={X#~JE2p+?vo>~K?tzSu=Ttvfk6DQ_U)^9u{g_cRt__bp(c zF&VM0%*oEC7j?5;{QikUr#K=k4B;INsJP^0x|ps#+(LA2cD8U}zkM}|Wysa@+gk!a zuK%g5+^{$=0hrrf@Dm$jW%dhKuU_rVvnSHhT3A@94D5fo-I#p%IXal8SBE50--Oi$ zZ;<0uuN>Um`ELDuv%YwsKOUvfIX>j1KXZg6PTigUZa5yjN=Z7zRbM^l?7m+aQ*AdX z_~=pD2Saqsz!>gUGM=<$etw?$$@}`_wE{zwTgoG|4hO-d1@W#Bhxl$-@Xm^W?=J6zzjVB=swAB$e_h_!R^1#^gF%?4QD5iE?ZI^Vvm-S9Kbfj)YIL$ zv+7w_<&5dQeEG6WXiG~=R@NVwm;Xc&&sum0{8b`O?*sv!ccCLv^vz-@+B!wZ0lwZ| zTVF4R2_7CEj)))yTZESvl>6jB4(-D4Jv}{rea59Pn*rtOQ)vA6n@c~o9V-Qh`VrvL zA3Vqh#TKyi!kb>r4x3Z8AvfsP2D7_Cr;PcTut2u~UUx*&g8f--JAU)ltp&5Q#J@@q z-Y2sEr=pO(d-*6FyyzRd=Q9^ERVS!m(1XvKz_+ngKc0ihOEkGb*zkEp+as6SJ zkHg_=YHH-|)wSf7E~~l8qx6h8IVs=T0wJDU_QDIGwHf#o6YOZoV_Hn%mgLr@&B2MD zxEsP^{exc-!7i)yVf28#_gecxQ4u937-7w$u3yY%Tp(~0#~a;R0NAv?!S_?B}q_s)z_wsp0KgNk>xCVg1mgZ&Ak zCC`%Vau#y(C5acVMtKrZGzD>)9qSwUGto5ZB0ls2OYJRs3lc9@7xMC9Gf_19MWu=L z6_|xQwT7zPYidr~c{dJs!jMMMi4$*1u(~!{na$m5X1i{~xW{=#Zk=zXM`=iDRI2sb zJ1CM*pPAVyzvQR6uD69LAqzwx@K|{jP?|7(snJnU#YII}keY#vIK2jhv^UPqnVHj! z_2#|FN52Be1nt8YHj}o`<`{!UY<09W+bXTGy6W^%i3r5lNRc?Tnyq#Lnu?xwUAH{J zhQ7rzjMp=kv$C?IbUXAr6m-YRcCTK&df^dokzZHNdyl)<`Kg-LQ)NKluX{uuYzP*# zuN8^Q3CPNMi}_tO$uD~JUe2=0qBuWtdjv2PN=k*zE8}oILp28Kk{wvdm>m`JuKcgIyDCG99WB<23HFUi?Y!KNi;ONAGc7MQwQ|ZAGdRMLN&CsX{cvMoU2{#ht=lX z%hBS%9&9LRDn-b9R|>S;-YjY;-QEs`MxIRTcrPtE6)gx%INz^Cqsxl;qi9e)*45*c zlYFCjEK0k1JK&j=m9rkOD-|@Xg@wH+A+7`~Y+X6wB9>>iP!UCwN?bWkL!XO=w{NHh zKTz1s%NOvl>N7Bl3XRwvG}2Nk?2zUP3Js4E*+x^t7nI2v869}7e%kp49q<9*lM=mw zK(L3+Ye$BM>o@uw0p<5sC)e_;qVVYGQL*GKbqx(OK*_?iIJ{u zLf5O`rzj0U?Se|sdxF^ls_#s#GX@l-))v6HjRhUp1f__&bKk!YlcAuZx_9^P&-d@DS^|G*om~A`2t-E3l6Vdf?PT;K!`X^)Ku_&U;A8vyl_vSI9OWdS zpAy%RK%}4~Sc{AQF>2CF`AD{*Z`{o=SpSM^6O%R?s z{~8M=2J?!2i)8@Tv!PX7NRAoC@4x5q0E42 zzYwT{f{1(t-GK;uu!b&h>VAI^XMf^`KRwbh_N^S4eOlZ#=XhJ(pIKU>ceC_(wa0x&uprr+%>`QE{VBSQF z!S4?CFfc0_{$^gk#jgqr3!j}FIshyI(06ibD&xl+t_$AmRrb?BahwI+`5t!C{p0{k zu&TJSZ8Kt=4!9x-NNKHQ4q zG4|V;Z`U#M%zmxXewUHxM|XFwQUchnnf?wUyN@XU!%Xt^ucM?1@0FIgrx5Jk;I%IT zkQxHUE&U7!<);ELFyMS)6kAsV-DJ(sx0EF^R|7sBXfB;mek08|CY@zAH}{L&Yk zNkR^8kLiFE1SHfoAP2qm8V=lAXqj`2pcfV7<3qMH(jfZs@3XUm<);PWvVnmCu!{iS z$fe;-9j^0DtFW=LvBX3gEiD^sYrRTqbec7ZMk?*(eDuMwQoyvm|Kg{oIV!2e4mMt> zMNLg@?Q6`WS6XKpyj{1ROEzLmKsf_u8TGr5`888DoB+%Pt`EA$j%it0uYNCfHJwav z5rP{f)9lM>X<>w3=OGaH6)wi7DKwI#_`Kz!D8Bms5fniXVrw%g#oWS_Ac2D>W*5`v zzm}it0&XL*42JIxRCW?-Zc#=HdVOSHA(k~jMwb7nUIjE_r6X|Ni)@Noc-8#peP4=iKM*hSno z>#flUa3@(UG|=%3efQDtY%3Cu;zK~7qk8MFGXG_lcmdJ3vzN%psH^jus&N8)rA)5~ z*b`_$zr{q0&T3s9?bp)N69HBV;6MN&bV2*2*$na*aDCm%?>}|uUNF14+?UFO{JrIz`x5GZ*T>6O5)S0Jl6)ADd|@FghqGK#tyVOIt+d!{d}57)1>BD4y` zl3eT>;3@nWCV|w~^`2RR0Y7<9D(t?QoOcNSHNVjNTQCyjOL{SnQna;Q6>U)3NQ?hd zTIL?>O8c?Wx%3bQglK$9O40765zwlTF-p+U(k1~m0N549g|Ov5N56s3O^hk&Mkp|! z%(_k5f&Ao-uf}@^Y>9SH^1H3{&yDCi_%1{W0no78MEkCve1%b#sNyPfapUpYC)THR zk+Xg$*j`3)@U8N4b+s?%*u`LKqtNK+b5IO!Qc(dX&gJTRhLU3Xhvj7Z^?J9cuUZKv z6#ZC6wuhv-wBP*E2A5?=b z9DK0^mDW1v0Tj%W{dH<;Y9Y)1H1PR48{Aq!2}o@vs7;YRh8qlG@u{hjCM`kWw$~ZN z9)rXqky3!DGi?u(@I3~Up#j*_1v(|#T9pGjyFEqzjO(Xh?!g`&2T!#*jXIwLCNE$H z6)?62oPgyt^^8wpe`gT*31BD$wkVL>k8_gjkN1XDf?=R+9rZZ)E$E#N=W77rj*YDo zO9c{u9Uv@SpI!{QV@8)7Eck4ruXiAoGzob;TgJC+D038@@b6AZywmrO%*D+nt?!jn zME1XbVl6h*ImL8!0Xw8Ua6*B?ktF8%2B14Ft*^+6S6!b0J_nq)!+81YgN<>(4gmL7 zx=lWi1ysQIjn9wcENDU5-UQsyZp1iXmOyt7M5V2fEXHl_HF@$q@Wk*>LipR3y^ zgLr^dR6@cJJcF*TE@agEq93myB_`I9P- zq-~(h76Yf8t~$k1F>2Eb;0uuO9=`kr7_;P2z?BUo9|3O_b{bW^*;CR7S4|b|?&L=x z>bws(L0a`b2&9T;lKuEb;`Bk-=PzIOz|I022Q1tTdXX_(K}P8s|8u`;VAu0H*wAH` zefI1_K?V3~Tbt}C+HGs5XuK8pSyzwxJx{RJ-u5lnH^785sSX@fmi=h4+?GyRG3;>9F!&{1qF4AcHwHt)9pZvYfEb@K#q%5 z6N;Nxuh${@L6UkXCN|Yc*{{B{bCpOSbKo(E7oexqZ?}+ulGFg)TP>}=*a~>`#Ef>q z{jl{W;Fuh$p8(4SMC8LRw$vIUK7D`{KHpWm*j{$zx$*Jw;@$_cPoJLH1NRmxtFFgG zKNoH7dh*>6l)vQ(!2GYxW-SLzV%;3p&!@NiE(~&D8in}ycHh(5uMFizV_YXHtTeQ= zmIOVz#M1}dV-WBuQvH++CT+cks zc)h1H$k)C+OOB66N{mxuV@GmTQok6B<9-G1dy1L@vJ3bXQ^NN)r@fDUFM&X?-$T>1 zO^TG%|Nh}VNt!=~cX6QH~&j=chV7*r*cGv+zQpZ-SYujW8%&KFEUDSV9O&1sNd z78hTIw5H=RwW4!Ni&>NY)I3I0pt>~S_e&BTVh6;`s@%7XnA-p`wSGf{E(*|)GoThU zR>NarhQQ}Wjl8);MZLkkm4Wv39MvQ)YvSPdXV<67%tB!EaPLiBl$#ykg#iWxgr(t7 zd~&kbo5k-yj%nQO6}4yVEx_10oooT54V-Z^9t)A+iUgRRPoF*&>nz0b7~AvHn3Dd%>w zF!r$DAQy{BUV7;GOR7)ku9x8UVvI@^Kopy(Ybk&&e$7wQmm-SsUN6utQUII(!-qTk zmi-q#NjvA6;$=JbHI4nMUhq&x{%89De1W2n#BVhSromvS22`CdU%r402&x?(w7l<~i#(VDGGR-)aOAnPgx0Z1);4RSF9qfyEu(m0il$ z$Snr@&e-qhg1fGcWchO&Xv8UF*n=NGet;C3os~sOEBJE~&BDS0a4P;_(qVmk60917 g_r@c2)#vAo;(xg960(`4LBR1$?zwES%*%lP1L(D|j{pDw literal 0 HcmV?d00001 diff --git a/docs/src/man/heritability_hist2.png b/docs/src/man/heritability_hist2.png new file mode 100644 index 0000000000000000000000000000000000000000..bd78c583509a9acc846342caa376168a3447d705 GIT binary patch literal 44302 zcmb@uWmuJ6xGp+Dz)7cwR-&o_NVqEBOr@(QX*0nQ^JC> zsBmx$9=&+uiGX_ z+{5)I@yDrLo*C`g)gxNF#$vsTq2FyelC|xh6aE-t} z@^t;lDF68s`~Thx_+?52>^3uOxhmY|BN<6a#Oej=#l7LrVxyy(opv>9wDyF9I}9}H zc}3=+EJR+K)us^&`NahVef|BK_4bquIlecyXl zES?cIc^o8Z2j`6#WyCh{dVYTX;JSl8lW!KCuaFRqSYMh+-K;oOjAA}g?ET%1_vxsl z^P*>+UO5vJQ`<(GI|RnS0LDkh%X{-HSJu_l)x^XEyaOq()9&@z&T4OzU5!MbA&v;5 zZ@Vm}CqFr**w{BEI*j`xL|<6Rtbv3?pxVo`E!+B)ojZ&nJg)Y6T$Cc*^Mgh2)t-pE zn~RMMe~IiEDVn(yx-j+PCm~`E4JjW{c~TW%IAY@BSE_y*ufEFUrpPYEgTa;SwYV*{ z`?eB`l&TNDXVqtFE-a+y;n8|Uc;l!pDk|#X(Oh0yTvnz~s1|7bkdTnTWis&PY-iGW z>(_dAltAN7SrbGkQ;Cg@4NOh-bjs(?Sz}vTcszXr12T$=iwz&mirJHfOq=G(@$px4 z0|~>vTh{dS^tib;Pgt@i${igY>5g^nQec4YNg z?bE&aCNOu3a&l`rk@L2bmNm8qiX$=^bTXyt3~@3)pWFL1O_mlET&{#M1!9u)aEPdD zsH%>EcZL`&cDdm2jBGhlON7@uZs9#9`=(VYMs?I1u<6JHW+=A*XDY8*jaGJ6mfQL6 zmMW@SiyjQ;a#iNctwp5}Ltm2Uy(5pVv~-9RP2+o%9)uKjrc z%PE~&q*CJ*UPvn#O1_>;KzOKpY;63`)bm255V4hwU?nPWNq9f=>p1ni&e8>MwTw!d z4JtJ}$ev0N$>cixF445Guy|woL!;UhlR-{VvAn2g;Kz@W!osbQO#jD*+9!XqOp4!? zd)ap%@{8-Ee}PeqH^rmRWxw98GkI&)T2IeAFSb~W(W+}GDKQ&%!o$PE|L)i8t$gss z;rdgr^H}W0^cB{%u?3{%<>%YH+hUm+8TXUzTEPJRv*ZZ>@z@IJXBsXjC|T3S#X!MV zUYag6)i~uON&2W$z zxCbLdj1~MTS%3@1p`xid?Y^|6*fvFcTg2j};+$=Ib-eEFZS1wNEAViC(RQW&xa*PQ z-I->!nXJiiKa;|;_R|X__B3s%5l8aFN%}(y&!^D5hM+)HNlPoMgO-bhx5et8w#O~b zcc}F}gh3!ay?3r2HP`0-VCNVv{)e;^c z!`$nmpj{4u{Q(Vjt^D4DX=J@WM+KZ#mK7FTJ+aAn#_Q}B;cz(Ek6st^&Lt_L)-XR9 z0UGk?R|teYj8e=yyv_L6TtTU#jCAtVwv~-mQ#>(BB93?4R#ET|e3 zuL;XD54n%l9Lb8HSWo;qqKfIedG#y->TisWy@Y|^#mvsm?l_WoHbbI6TW!0YlGI%) z;eNI~!SJpuK`w8J4*yoXNFw_v`OFiRyVU4u4F^R4a|w$)EHz(BsQ48=e*8qnVh&hCBT+&D&DV3|l;=Ayz_ z!ieoKaJgEKA5P)!62|ndQXFA>)Ive4Fpu{5Hi(uXWM6QNvbbn}XI{T1i6V4JVEOJ% z;Vd3FaBfk_e)`VM%>|%8jePb-qIe>bn0?zHTfktQi$kfty>-r1kV&`k+q-hTT{GM1 z7r)DNNv3x3k%S=?{+Ejnas4@IRu*>CFX%$Adj;<+J#WtU=bP+iYa=}8@u;es_s}9E znSCOdX=zPoYi;`b`$@+Q7Pmfpq^#5PpYvKg4|;gbvzjzB^;UFcbyfSV+{?J@L{E{I zaVUaeh{8BydGg5|Hc7F3q{fftOA7Cg2bpdz4!k!~osUap51~tgdp;j84-E%w?Ji`G zw9JN**m+XRva*bp!IAQm@%^Koz)>XC?)IaJ&8@L~vcU-*63WZZo>cD*t-n5$=<6#f z9gZmq9+jf!nY{Go_xiKay;Z(9TNh>b)=#Fdb&ipq{@EL^s_&L}tL}v1Bmxh&CzJZ# zx{UEp2?&~x2N+72+3J|ZIY`r|!zEVhKdeTQa{sEadN(Nn?yu3L#3l`fXL!+dR9SRq zw!5l|3JMl!UmZ^ZVFhvKje!dB_|w5mA@DA>TBV%{G$U*^7O!K3GkzuSoL%RqdtKig zv~dz)+`rh77>WcMSqcH6!|5*@3fSP4l_0~;@gBkBA=XVYH9`q1PE9FWYlfM zQukcYWBGkYcefZgwTmg6*(l`4z&17c1{=V_Lr$o{NyV5+#wkULC5Ed&U{X{pGI(V3 zD+=sMdD5RxCA$6)mJ}DOzKqLk@a$BquMvjhi1eLcR%(^jU`Ak=JXV1qp(3F=F+|fq zzkhHD*?3F7@MUrPh7^Z;mgcoRUij_Xx0#t40A!wFpg{V5{J1@z zwR`jC4U>NB?UZp;Tzq`flSVD2zew;xM;OY2u^J`$;fs>8CNd(fah-@Y8c~uEF{Y51 z@AHP5OR zvB`7J^Tl03*a!1Xlk7_PLHu&KfkH5;h6t5Hxg#vw z8}D1BsI7TFoT^%kfGetCNxVb5LDjsHVq@%Z2r&4u5bAJ_-9?5=0GMxjb6M z(G)(WBInCk-47dl!LHiz{$Hepxhw@^j!4o&Yazn@jq#ma@+WgTbc^ z&=!m`!TvXxYD4$J#tHTlGK5GOfno<`m>yrl>N;U zab+aoZ>Ki(_$-7N0;FVy?Ig>i#=Y#WgTgU=@)YUOwn>|cziEb}X|{Iu5q*Wgj+l|6 z&|fB{VKRC}#K_fix|@z94BC`eoSg9=&rz*A>>q1Kzxn3}Q9^|3NQ1F^tTwGPB{dz1L&v zG|S)GVB=8x_leL!fl2^zh6Nv*OG?7+#F=#pN=j0=KCHBQ-dvyWHG!r6RL^A@U?C5N z30fp;ISM0Yj5h%I+052rl5kjqE|IDEP()rHb5X|7qvw;iZ~?9l=2=-pMPsAe#r}LG zAF*vfX3rOTRrPYcAuq3^?@!;AY9T{BuTNc1hB#d(&{toV5jKM%fSY>I_TZH$$KiVX z{#FQorCuvKt3Xlk&hhea8SHkdnC|c2kBTXK-q^Y;*WI9tV3KP7AygE&S}W6Sg0x%K z5I%VlK_N&E0UZs8+o^f$+b)-YV&ot!vPQ!$6i9|eV@AeOqFLIqwq*DVo!4}PVUiKV zCTzr@=ztxkVPzyjSl3c8dxxBf;smcF67aYLM*+Zt=XfSPdiyHY+lK`8Ej{_KNr63+eVVS!ly3Go&AI_em0u$nNI!6~d%sCLkN z@)Mn@xqYCEM=C8AIV>#9d2gl$3oMpKZO&|kr~1xUO<*IgUERRN_9x3DJ+6WODhoPcg)MwS6trr9;cGc^l(F>JsU@!)Ze_jPVU-ziSGQfq9h(mzK<1B zd2&?>L&L(T?Ii8i#KUX~pCaq?s012rd-SGbFq4|eldhurcPz1`(E+$dp$%^C6t_y9 z$P1$tyj@;Xd?W4+h9mOgg(3XPd0Rp7Zg1aC^un;bnYBAJKPditm?83VzUFri z(>FJwUk2A-qw+d#D!qNXZZ<*L2iEO=;|^m>6sYE+<&%#!-xg00Q&4!X_2C8@0_>KY zoIE%HZdhu1`Y^Mcm)E^jdCTw5%zm$@@)aq1(`#yK*yK+NqBxg0rbu6)^BC66?(FRN z7e>Xi9fW=$*##$6G{gGlCvC0Y;nmZ)NSTK*1qB65<-2xZ4Z+2W!VT<6!qx0a{wcDu zgGtm(n@&ipPR~RxQTnaymN9f#<){zP&aOZO6yWQts<)bz*v zDEKQ&9Kjt(a3viqaKX$(awF}LIDl7U-Xr$v#2I{F<>!YOAc6qlNS3F;t8^EC6g;oM zf`JJULPKDIKxj}gFcDdl)Ny2ds{DkIFp%LH0gtRee`&0m_fb7<$T%&gS4tTUV+EDQcD_mdPp@)iVX*e{8$;uuh)cw&79~&zDTIwVd^+e)U1}$u58rN(MI%RYU05r zQgkRGeiez)%CBYiu76Y@Tr9Ykt8!bfJ(U#lSI@MVF{)%!oQIIl{)6(8M8`klbWivJ zx~I&!>&4uHBDP2CYB5C=8Hz8_^}6e9m{WZi*9|){fBMw<^}xludI1a;4C>f^J+CX1 zpYJ*XURW^G6O&pZ=MC-oPIdig<5KYpiOp@IiQ^^>9wB zFY#6Lfu@E_(Z}Dq3KJGJTA+x_drPA3nr^^OCf@1)Z5cuPcevC6*wGTDT$KoNejWh< zuOA64;12qa2)B9NfmUP1GFV-?P_69g{q>xO%ZjeB5lVC8V>q5mZgjA0P^N0Y1s`Mm z3AxK3PgX|EKja90@OS~rINhcD5Wf`EP6bbc6&_Xz)RV|Z!tEces;}<~T`SMd&YYU& z-(hwciMBkrETTk6WJ_d!h_kBEs^m)wm-YW*JF+go%8uXdAnfq8Zg&^`Uimw9Lc4bnH@K) zP3QaA=n815?xSUCZli?(ImZf92Gjvm#S`0Fv`ic;_XXN&YU54L2le&!l%j!vZ>|c!FHjU+i7>I4?2Czn|gCnD$EQW0<{EA27p?h zQ+6<>`j-nrVHB@x%Fp%x1*)u7Rk5o7q(sK(V&=44$a#d+g|w~9kM>1o!N*VFJ5H6& z2ye7rdG9kS7D~k4@2ruw1Q1%qv}VUG74@Io$NaQ?7-T$-n+ZLJqX7Y9LUGEt*DK9xGiPXDj!ZOg% z1Jfj5y%;n+rjg7X9A8h+$ZU}+Fn_V#4KlTzZch|X0P?qZ-qv|&VTt8YB*}?#QqQx8 zgocO1i-3o_sY24P>gJO&2TPnN|x6{pzGS79~ zwpvA`9Fn;H_hb2r8@0dsZR=!d@XpOV_vaeS^pg}ETZZp=i@si~WW^n%Vm(Lglzy9( zhm7c(&QRk=;YdQs2J{mA7YYP~giZ$w^OPGYh&|piGBO2`o-rk@5d%?Dh`&&tpndO& zAknSft~X*$1jyz@Bfjm{&FL39linzTNI`(a_{vzg28~nExMhtD4W1pFp7uCynqzXfBN-S5{VZo19YlT!C;T~$p$9MqC@#TLYN-s#?6#B zKM_}MeOKj!Nz>?jlQWQaNH9=(b2uYCC>Tk(9db%aUI_>YTyvC=23cc06)C_RuxCYz zgda36-k@N6y6P%>io)ScOPkPH3r@>7?B-`JkN;rR3&v;BwbmSkfp_EG%Q0N%LYAu^ zQJ6JA!foC0!WEH_kG;QAL`4v+>_^1v9ZUdV?)K_LcO94fN>A6gRzo(3iE7b1*pR5T zCWNl2xR~MPOPtnXy|I6OLi?vSC?ybZ<^ro_O1~Pg5$CsF4>QCMa=%-~K+GrR)(;TG zms{Q&w%XZNf7p3IMe4*Aee5iyh%FKUmkMpf>^#x%%>}LQLxv$w>eru}PZRLulXl`J zjta^0<*#Wopf4MyA*7>kZs~7DcifTkC@(bJASD2w!hUp0cgOdm0Qxr6+rE4 zT%4Mk0#wzD7cT%J0*uPffvG*GLbaqXU&eLJ*@!_6!+-#m9HYxe1B;*B(hIVD+F_C~ zvJ2UG-i;(pLqn63lfy+rARh941=j12X(tJVjOV#gPo!}`kpE-;zXZrg~*x0cX{2LG3Z;JIATtIT@U zDYK&C;+5*M`6K4_;|w^er-T4Yt7^UiZFLCfO$DkCmNMU%4Lk59W`>OKd(gC@J`pjW zw?i+{YWdTt%d~Bd;CWE(*y}a&DZ_IKNzpg0^J}m@93m!%M%Q#)Dt;+E=IG?)^ildS z-uA&|7kcaYPOP>p*lccWSg*KnT|3PF`IpH%4t93kR*%H2-RRNBV#{ABBT2ltJC!yi zSW+~boNN>nBB|#%WH*`Sd)R_5IHW=wdeso&yE&Jwk0o@PU0Cdi128ii0=B?PygA?7 zaUJ%&z#w_6t2*ZNB!FO!jIwRMKqi;R^*BeX6jjP%FIK?wy7}r)vVsWK^b3*1>&6)% zW^(Gs5=z7GvN1$eC2acbIO|LCx{*=hSfB1UB>9W(aQZ8M< zv(agf$Mi?SpUw=EbQh16Ff&G7c8+nOp4j{NKMc@d@1OcG0%)G1`*TJ{#!ECLxM5dd z0~+_Mc@E+TWMpJOwgHa#1u`2kW^&>frG?=@yr z7TT6Y zpXWDl2GXy#VPaVw{*PsIRrISB&DoNE{Zg43t9@YyI;Gwm;k#oorxu{;_W#{G?$Q4ZSrKx@{SPm|=4iHdX$&B~FYzhIL(^|kzn(P0-ZK+T2~#G^ zi@**!5(_{_MTD`C9G6AcqIbYCC?+R|@lycTgQ&@-H^!pGAIsk5)`MS@J)j!RFaEWM!Y{3gZ>wvf|a($+A8eL`A$XDR0O z0MP#(BqKQz<@KU~tAG<0SS!dC)<@h~PK@9L&#Zog(iNek(HLT70t7$jAI}hJ9Jd?MgFcA3&80hE{m<>e$rS))sCwOJO4SBrHw;iWF%ZhW38CPy)UqSFDGmLIhhL7xC*@4ERxi;;A?@w@MxlRKj1pNQh zEV6>E%6hu$io_|ir_n&IM+wwy-HP`n_;j10)u!1Lg<_(3dgLZ$It?oy;_zv$r@pPn z?j*X}=W0Yxp|b7=9-nVQyDXkSAXC=0*y^vi`42IhQXd?BOiL}d=jsBZh?{I?Ia)uV zlF)@}eRKRCnW{g^PfpX_TFSZ}c-FIyUAtG>e4l(Y%y8(aE_4Z0jP&H&cd`AX?W|E0 zg7>466v!1q;n*}zmK)P?T|aOUn^eB_9{!#TtL6Xp{k@kLI)+D}op@&musAU3)m_E! zMBtRr#rB!j@}Ck5yERc!jJ*V{f@9EcPO@vEt_e}Ias^KtFBvz<2o%YFC|MOi9I*1)nAn zTN=?rd|dv}JIl*-(=4m<=Cg%gKj*6+IQ9;j>2NqAD_UDy_4k8Md7H)@`)cruCyD^| z5mEeW!Q9$9h3)Dp%OQ2>j}2)w#Ggn8_Ke$U&M&~lm&*CK_wn~oc8YJ(qdAa+TvSA8 zruacQ8Ml*;s4?}LfZ_mPrJ&KRzFxPwvu}#`_|EnXSD}EL_b64!c59#;KYz_~*7l_N z%t=J$co`DrIS%-mxsYErRO!#Mqxp^u(irQt}!Sg3tMk&z^{d&9LQ!ZN%ifL<`oXk2LM9>F8ENiZZhvmJNVp#ksh|f06 z(yXkl*EB0RXkj)L&CR#7soEAWVLH?jH!N!y(N&+Z@%`<1FNOCniMMci-)-??U`ZGl z@PRF-3F3?OVZzOmScx$H5|)lzKwXGes!+#6T9vqbb_HhqKC?MLZ^(-}XOUm;ZdoPU zWPP~NRaV`3Gs)A;)Je)}hx%UQkjXpw{%2jTWE5%H=8ch*+`}e`ER&9$^ugs@i`66j zpQMuW=N@&7=4`cIMqH~=$QDVAZ5}3S3+Xo-d_tp+$>(W(Mg}XtBERT7948-OQ3y0U ztPjj+r5xbEgswyf1_uvX?WFb#K49Q~=Keua?tRksK(9J>KylyK9usx3J3VHNwKnh` z8;QDBvpD8}Nw2w2qScJ(59l9=BYMo)K>JmbDP8mNg45)w>@XXpL?+Epd`2DtBxCy( zB2vwIaFwv?^MsmUp^PGwCgh@@oPv;D4yAYg(vO^23W|KHM25J}0hPYZNZs^goZUc; z`ZGLFCZj~@6CotyniPig>0R_ZS`vwY3S|#g#FOBvd4VN97|M2;&paY{)tYMM7wq}i zdG%ILdVF+pnm#Z}+1S_skxYb+g99Lkn#xKRetyq`%xnyQIL+;iz7-7~Dr9Q!90!|q zvIu!_U;r36UJ#%?3!JVr?gckUj#gD&T^(@sfFYyYq0e#@#DhX8M}GTFK;e+nv-XjZ zOA?q*&rbVDhMkZpzlMAUp#1}{tsqch2A^90Ox(npK2wAtOd*VMePoEzTi6T=Ejy&q zL=v8{&kC8lW=W`#f9*z+V=Z+Ll0C>`rB^IxZX(LQNk_H-xx&?Cy{>|1Vg zIM?v;!-o$J4Y%W0Vfr2o#kEC$G$isTLvL=hjNirIEs z!~ql^98uqSKdFw=aypVJIU!AiN;A!NS_s_75b$pkNKOn8h^X%?+lLO(huIo`g3uqP zCuGiZnJR^#qiX!L^Lu>!OH2%l`3Ow?ZK{Iv+4fub2(YHuFcT3I69a1$I4Pb53PWRP zq;Zf$Knvgpllq)PCj+`*n$G}wQo(y7$#%@$a1A;@Ytl3i*alEg%O-cqb(_8dsjs@$ z>tS@2&E)1xhu*kuTUbJV1~M2_r^^(&%MsaP-t~6Vjbh?P*=X3|4*lWzyKtBDD|(dD z#vQU$F)C$=UaaZqX{V+Z(I70qipGTBuMguSPqm{w~jHcYIul z0{^Mq_>LIfvkoOElztPHREgw-6di5VxL=f_6aRdcV>1AczC(!vO}yWPID&_sJGVCP z<#T?~`SFb4kT)cbh?Q!P@QAGJtTp1Azn?7wpKZx5s-)!M_-QmQ+fDTu_KMI-&f413 zSXD))C;=|I=sD<=?bady@ zu|V$k)+uX90wSWaQ>C2kh{Y-%o*L%a5o!nu(3uFBbE0dfbCl~G$HpgVX=%5@|6F#9 zHR(N%Wsr)|D1Xba##$ZmS(4ton46LDw`EPzE}J|wSsO=0;GgYa3d(^_^U!-3jk0Bg z)~nArNWPE~(`=t+bJW}Zb4<^VHB2n{+pzuUM0Gw|o}vmG0vWB^5 z#}>#g?Azar5=oUgLJEvH8P2xwVbngxJ&(u3R!q)e_M{re$PrsFmL(}IPD?zWWJZ7NSBOCgkm;P zI*gpjLr}`>QM&lS)K;KPEdoDkecUZTKETD)Nilzav-t3LnCWt^hm&}n%M|iY z&Ps4-lxz;cQIqa@o_KL~e>$c}!fAVYe|fSo?0wkrc!9!ueIV`$gSw0NZ3GGDg4d1B zK@>JwbEN^WgcZFlzAapzsj(XT`uZdv9vT`7Y;%{YKpP$6J!oE4L?wmxkZ=!6d;A(3 zdpL^nwjbO8b!#Fafhbp{U$f3?vJ8m1xj8xZjLYDRP$^_%XOH`c3ZOsGKu>|9*Vh2A z&3lV(J6C@MRJG#?901*9a(~^V@V?5;%>`E-92t4ORZ?v>yuaACI2j(2ub3_X8coiq zM7x$VbpiM@rE*}>H_H%}9j zbszyZ^DS6&#K*@|2zt$(6+jusLZ;m!^FgFyT7*5-oaTp-J9j) z=1#PGt%amr@wNiFiwwx*05hR<;aajKq2kk~@VkGF@EA5d>xSb_`7LAB8r1v0p2|b& z`o|@Wf?oyScQE|NL+uA~0-i+_e~U>2Nnkf*PH>W+;_b}J%35e~@BRKA^BHRnaIfTl zyu)7HP~5`;X9*Q`2Y(Ry(HV-vn`dTk#KSFaJ5J_F*vxT0Bm?HN!~c_x&4y<`pddam z9O(!w`)wRkqj-DO;f`cGQ1;yrK9Us|4-mp9w&h=E;?Bm8#MKTS zh1!$C_tn$8FTtCNNG*nQE_!?fG$i=|_L}bMlm8e1bu-D;a#xVP+t%~*kDi{MMn+kR z`OXImcR(zLW`;dw0z4#tUkx>!DA4uxs2BM2$oE@b*P$=lKjKaZPfn!jVe?wK+MjQB zRn1hI`#8*x7C^iUx zOKZEqt-#|SLQwCJIZDf?k?u5t1JJ<*o+JetM+bnw(|hY;)<-VuO}uZ z3YU2T@=@WW1~mMuN>~T>tvFbm(f<8h<9OgEQw%#Fj2Eg#7FDR9kH&xB=ax;iJ0Xzd^?VBuov7h|4bN+9Yk)G+DE$qj-Se%n-U*)F zn7;2$0&#qww$`|nK^FLy8b~|GM_VvD`a)0vwuAw>et^G<0qCm|$TiwYr-Pq5c!JSmo z)ZDTp>!tr^A(yJ}{9VWl9{UX&DaVX8YF^3nB3?`9O%Vy?6YuOxl>cttZ&saNqWs_= z2F&+9|7W>}zbT6U&8WoxwQrdtTh0*n51y4v<%xSzG_lQM+ymIsa57%?QgwG69y%r_ zaFi*4Ta6=<@+m9;+7WB_7jNCaJv6 zt-u)$Xs54C`uoD02Dg_-Pnq-_@qkPu$fTV;dTE*L6Q40*N!^MaAP5X7~r8OFxP38t~x{$nKnH0a#vSP&p2QYf0T0{cO%AQ~K*{1vp-*`#%W zEh$e|k}J)^D$XMyn;O{ds;8|%N(0ELKoD_s!*%y zec-$S9uWR;8BtQoy0QO18#M%gmOLO)L)x(rru{+KVAHV?jDYKt1`rtu#8CrCCmuoK zWK623#i($DJ|2+q%~8p+zWeIYa25tccZc7?rRv(H>UNI7VPUaSG${-#KE`qsBqZpN zNbMo4Md#fq7PFrzARxn}S(WqLknW_XQ#@I45ew;q{=xp^3y+~X9-{rVi1cwW)YlX7 zkz@j!ipJgF$bjKDiti{~`K|Z(+^7L`lUlP@qA}P?-+y>P+!Frt>!=1ks1$;` z@CeeN^Ya?x07NH7M@Ip+%dhtHHh+vfb~>a8JE{%cyQ3zGfmr& z?jvZY#*ER0VR>bqopp>vZ3U%`wq6%1yKRyurO-TaIa+-2(479%OLaIeDgO|#$i60& zRZzf!1b;W`HK*n0Zz(Ez`Q}YEn0s(ND%;%=C?*99L z+wq%d=<#vD$Nv~)R9M(-tEan`7O^`-`t!XYgH*0cqFm2(jnowW+li4guge?T1md`) zq=75sN5VMt^z><S8Qph$}nm=gM{sUff!-UzeFQYZh$jdzEBN|n5`f$FYrm= zs9()unQkr+^>z2rFH9y#Z=ilKV4rpZC@wHCFiP;AiZ;orWb`J<;9##ZIAbpcPwgo)*K;Hha+wC>D|ML{80|Koy}rvs+|pD*G9VcJpdqs>&Je6e(&^L1D-fc!6n8{ z!FjnWyb<+CSWH>%@RR-0umIcsAnPph`)&cc3gc^wu{`en>E$NQPJ0nz&xc@15C}8% zI(=P{7~1B^N%>k-Ok~3Do#;pTa!#@HMtuly;sJBH>$@;(^&8`IVO{PhE#$ADI{>CW zqp1o*-A2c&&Fm-*${=Jb(`wg=qPGA$f6_)L8Ee0JekzfTOSu-IZ-gUG2*RdQ{d~^C>j=NYZ?__C3cA#& z-=Zf78V6tx@2?>Q^{z&X`-178WSK+o2;d_2N$x8Y?5Z?gV^Lb3+ZY>XbFKpz?FYJf z*<#7-u?UC2vs198c7B&8vznmY{7UQcV-FCi1Un1JfnQ%9TQDW6y%dLvOV+hsijP<$ z5DQ0x-AL{Tjg*;dj~sc55y8%BN9(-#{>}F778x_&$VVi~<{DTqjx=Q^b=&r(B4qMz zVEx$jzTX!kHOBk9)H{{7xZ->*!=yunuo&yg;H#V>U@brSG`` zW}ZC#%a=1%CbC4>M;z_*ri=-4SeFji$}q1 z0AawmK(<8~32hes(2ylNmcej%N^IJdDKqQ*q{hW`}=I5dy561Fs?br*dPxE$7 zMj&JbHnohjG`Gv)5>Wm%4HFnOsx@A<9BfxRtjn}MSw{3?M}yd({E^?}o*TCzYPhih z=bL?pIU5@9eCf@g#ARzsx7x>&n+eOBY>9~LC4cOHiUS-K9~6h{i-U)){B$=QBbOKK z?4JPZrO?T&TOpC?C{`YH`(vu>>3t+rRz?PNK&0LhYvi;Tv`8UKEzM6Eb+}xQX+&PX z!u>`pIDI}LYMiFia<1Pofj&lTNYdQN)B-}HVCEM2Ms&LO4MF%yI1?~s8y&qp!yaR% zR+QLA-wUmj6CeNBlJvght}imABHWJzX4c!2;f;+A&ZiDjXvC2)3%U3-=f%;M4X~#! ze}C6MOd!WsF|W}IR`~UB_EGV1u8F8l(82j$l(gKlqdwyf6-@j}duKT4pm%0!c0e>D z37vb_5SZ728Ajy+7x;?Kv6*D+0p9;HT~25_E1|mUHcA+;#il6j8@`={`e2hK_UY57 z`<)DR!2dt5>NR$gtN&#mvCStW_@OHT<;h}^ef}L%Kl%qQp1=02e;nxo$iR_K6=Qz7 zIm()raPd3Ko2y#vSg%3HQLjX3AmX=mX+%Qivb?$U%VYHJqH@9YkvHZ8W31&85 zHsW1%2+|52b0MsinoTdL`)5uW$8-7)`nO7eSvEAJylDUd)%W-(i0wcT03AO_@Ot`P z-kD|mS0>UnVCc2R+qdFljuH|mH!tMJ?hoWmPaa$!9x*4%y=(~yQD`;c{e;=K>QjYa zVr$E0xz>_L12Y)GBD}9;dy2Bf6?JGfL$Cb8 zLDciGu(T9R*bmU~A@u{Kf#OqAQUccW?(VK^JYyy0=^R?iH`wJuYE{e+KJ5s@gwO#Q zm=yShEHsNii6f0qpkk+P?5bYafUF6~VLFMgn!x^_B_1|4Jk+Gjbb|Z!%NOgJZy_~q zG|GBUrCc~&!uaBB-H1d%5)-ayX;h8HfbUHuWDW&BG~9W_xx{m91LDHc$*nZ@Yn z>95Ap@87c#PYVJHCPz9J)T)@|sk`NmvRvqRlcYT-`d6f|xN6 z#sw^fwG;S6ZxjwyW0qj9nUAn413$dUbLN4acJbCIBGv@ZcLJY$VPVMKk37RFtrz5FhuWlT}c68+76c#`B6ZgC@2TE=C(0uv;{G9!JXit)pCmC846baxUb| z9wZ(Fy+{u+@KIKI{|46$QT6uW9Y~={uI$&Bs;kA8{scKfa-EaGEXbPY9kgPj! zd`~d&{~>nz5|uu{C8NkD*IqvsH^SNeY@1|WxrEe=fTA5OiC8rUW^b;8a80g&O9bL$ z%Z)=3K8>zII$mSA2mLo*fz1^pX8))b_M(0m7i7xnfMyE>3Q(Wt>g*^O!Yk^xY>m5& zJE5Uj0OsVWy?l&Gtfs2k1)zF5rOuLeHU$5O5;u~Ze+J<3&CN}0Y76!M;ROHef`dfB&vZfkT$LdpOssT=ETnV>J~?~Bq0Yt0#RK|9fxFW&!19npNQ8*tJ$>3c z&@~BH&6Cq?abv%^`uEPg?tgLJ@NOvX&WxmA``8umnsvhQyq9Tg~Nhf5gC$;s*IEdU>-hU;b0VgHNtxf-jd5U|<7 z7Cl<+t*%_=w4KucJq8I0@5euKb_2rDL{G3ZpV^8nWQP|cBV=S{fy`?w2t*TcSSMV> zY&~1F=t8&RtgnHEfxP2ZZv6dOx%U;m2KPABKz$OYqZC>2{&E$h$w73q!qr(Z@U{We2}Hxc!P={hbjlQYE|1GYFzLT1Ccw-p-d&%` zD*pX&CLvjPLnF=er}9}K>a+)ZH#@w;AUGuC!|wM$d#^7aYG4pB0dOUt)d2x2PcG5n zc&%T3Fq5umIciG+@K;*$8-$=oU4!Vx1ZksJmFP4$fae5&b+@M6;h+k39wsB+VY(Cq_}#(p9`3<$3B2peX8#PoOod+d>+|a!QWSU$=$6% zng-Z(cEfhR8g)tH?PmV~^^0_M{?PNEq5ZEIC_5*oJJ|sVQFwqt`DGk^X~*C&b*HK8 zJXitdN%x0c*exE<@g>Ccq>$qKAqFV zf&7h{$Ha`5YIBl5L8moLMd+FE11X^CZx$ecdI`M0`>i*&UK3WR`yb(eE^s@Wcn8!C zbaZsGk8neq&LDEPO+cZ6Gdn6I#MiWP8Dtm0As_!*yQj|2mxhYMB;^$pBK!3fH^?-| z$P57U4_I`4FRuAu|*Ydz3oSIe~)kzGc(XVCdLJQ zLL+*CjwoblS!CrE2xjBo_gezFoRdNvrz8HGY1=w{q~*)8N8ot_;AshJxIItap_`hS zrE`7olZZkM6Ow(0 z7Ko(KNcSd$=&lnaStKATbZanQ!b7R~PKV0=X1t~1T5ioj{~x>iKjS4`BRXr(t_{#p z691P}$*TjfzBIWv*n|F|)M3sz6n}llMMa_UMYCVPZaD><;gK2W1tu)lmD6jf{a(9F z>N<0yf@|()#$dX30hz0w-d+lB2e@GZpkx00`BQZ~U2O*7-y4kB^z?@rtMVF}a07G> zJAY6&$|pfQ#0=!n#thXslnxpVf)TWwb{!w~vp{ zEomczNF$2aTH~TmJ9D^9t|cQ2aYVtikwG0D0ou=xu~WDxMB~?#pw;;pCWWTo|eQqBa!m98U+Vp_P z`;Y7)Kq2gf>_J$&@7UR^*nuk{!1_D{xJ1B%OMvyW+-|WIG;*_`Mw8r83JNY7F#j!8 zfTaf<(QZm}@rXY%*?~PGcyfpt6b}axE(B{dL7qX$f#HatAg$yzk^ssZlrbwkJ79eP z5(<1>Vj~xmqt~>WQU!{EoC6^u2HyvP!K8%PpkhMPwl%7mM07W$wZCUN+*(?{eLLNq zuC{&+RN*h2;}raE-&`H@0SN{fF=qolRWA4QS0sHQm{?0M*S`CKr?L1zYRTbAo+uC` zEM%M&(!f^0dHsXq9*NNL@FW$ll(vuJ@?==J0p6ZQx=gzc3qpV<4(zjXv~QA!0QX|e z`v36umSI(H-M{dnyAc8DMnDh==}zgEl9p}}P^1Bn{rG`v@6C3td)@1vbB^(=VPd!pWHnQA=cqi=zfUF1% zZ)QtUVM-K;&wwJ}aO9+dhuN-cQiSitjbHjG{@>l!G{e-UWI%uZv%hJTd^zjQb$s4wFt@XnFaeJ)M-(y9Ak~=|L#y9iTnApd-b%wEoa@c zE1K%^XnXFteNo{Y3&!GHtw{MigF1p2r1VnswwWgG5F!Cf!)}fbv32Z!6c-)+;KV>*ZJL&q83l`){Ws-pmn%9bHmU=YQugt)qkjJEa&$XCnC%Fd6rYoN!6A_^_YxI9O|VHD0yPcJVxe658z)5|KGFbv-t z{o1V=0?u`$4i|XZ6jhNPx|+E?as?1qPE}A;7@4I>q+a-Ojaj3cp5we~3HdB-{6x*4 zcV*O4;1BC`4Qh@t?%byAqO8>rfATrg-_=dEibVHgEajh=cC-${z~OhV?`sd*Ot)d? zm@-X^?6*E$1Sj?w?5_Qjq$_d5^dF)0ca;ZKD#G~RZlS>l*D%G%!p2*S>P@1B|P!n6EWI`6!V5(>*{b06LMak{^>1j;GE-2j+3 zVET_!PPP{wlXXubSu98>ka+%w@85N12juANVL_wmyzvz4`er!W1^$W0Kv;yv#%Onj zB>mSV(-;`S_n1lTZ?f7p?k#6&;sg47R9UEpad@JqVVje(gEq?nL!INYI)3cVl_)Zu zS5&=?0=-hV8}p@N9%0CtlP-!8E=tbjIuR+wLPG@y)-9KRRUvO6+UYtgWUQzvP*Fhv zka%RmSDR?yei^)n0m?1Zd1&~bKPD#6fRYkz_wT()B@cQj(~Yr-F6G=$Ii@YF z_c)aIFnI$BnIGewi+et3Tx3$-Aq+g{6&F_;ezxWknu+f4@L-hvVCz`?HtZ-3nM z@O^=nkwfs3H`s{qmNUbv&P-!|A&iY|;%bU-)Tq`=NJd@csU%L|U#V(qUf)d`5iSeaR<~B_X55$>45k7fPH7`gm9Zpf#A;)S@6<&V~a6Dn#mw z#D?XfoH^2uTm$1~X;2B}AIY?M-aKG5aN+9s~@hXzjp`K*!zJ zCl`2g7J%%)Kg(F*6myG`)l$4B2O+Nzioy4vQqV3QQ1Y)-z%SZ9) zwasJfqlPSg8~TAZW)~jd^x#9{Knx3*LAb@Ydqb6;ge}9slU8g#-ccBpvzQwu6G+?Cp*aLs5av6sE`ooi05~;?AAu3 z_f=+KNNW|m0^96{N%ka8Z@M`LIn)`=*Fh>>#-ye7t!p77;Z;KzDT^2K(1YhW_(7$JeC>)J5#;rB59el6J9 zTW>nLd~H=0BRuzfdElRnLmi`H!tWcr`~Ao4tnsHO4q=G6ZD@Hw4xVi;Ujv5VcY1v@ zB-Mn|O=}yx8UOs2)dbFO0zr|xHp2cUB^sT+;_9iL9ozh%($>kSEJ|Ri*st=Io5Yrf z&Ll$7XHn9ASl*u^+<0Uouk9`BH*0=Z4#v?^Hm_q_>Haaudoq3ZAMCqhV`BuEXkzap zPFnF|g?_x)6I-2M-!U#vbu%-@+W$gGJNpuvawhzq^lNv`$JWPhda0R`dk)HxhWAs& zzS>lOO=7PxNkLcuF$+Of7KTE5^G#!+!zk)wfR~=sgB(J9Z}4aDxrmt$>dm_@rTSi( zLj>ozae~n}XyI}GE1P6W<&j7nO^kfWa7pM8uB+tNuf;9egDl3k#Fhmmf<0I(`?CSy zx~h4)@k^V01}$*Gi@u2r73VE6sgWMxXFOt*c>Og!Js(>VGqZ;_3WPT>P)NFtb!BXA zQ*0NWv}#F98>}hbQ9wfv2}X$YW`CVGJU!C>Ph_47+n7X>2>{2J z^=t7KgwO@PB$(OrL0~`UOdwvK?Wwo`;Bjz`PRpa}U8QjLgz(60B&YHcH1{a7NB6p@XcxP79ilMP%3z`ZM3Bm zYHY-aDc_R*Y}5D1!Sol_vI7ZO#CIOG5wsj;d`=&^){gPOt%X+qW6u_A>jgf{&?xL* zmGfVnot-7~slqIjD0Mi*bR20P4E85$H=EuHjbZ=^5X66>EsczAV<`7L`nkrJ)2ej> zJ^%&d_J{ zv-Mhl?&Ib;kq;QFxsOa6*Oq(euAl{+tIK8B{$rAUKfL$MtRMv&dpph@2xX{0PwxD^ z5m`^m=l0&;E1r3krf@yP=;2&&)NiE>NR4>@v+p$R^>=xUl}>qf{I2N%#hv-CJ&CCY z83gAy&_4a$Hiamc79r#|pYl^AwAzPXv(aDqwAYF)V9BaBxC>L`b$92-RL>pr6@{sz zQU-GbCjY9277PNu!!ilMv<(GK*vNLoJEhG$@oKUrFF zzrWkTINn}+!?|JRrDB;IT}fEQMQagu%@T@Ze>DkZ6Tw72{?H-Z#g4UajU{23yr?P)}yHdQpb-t+L_{IXMYSHUG*^S^zbN4N(I#- zrz6=vzslU}Z;`pvJY9G)1(ew!S5&7_KDh#|gulRZ-Xlx)?l`SZLdq1?<3FqM=LHGi ze}ot>c9|v)j_A>yW6!Y8D=`_wuie&1wZuObH3z_$$(FQzd}hIP8bR-n<<|z`f5RKh z??1c?*vUTaQ|0w=*4_6Md}`6M&U_?dkgMhWW1Lg;kVJNi&emCI&g|*J#F6>#XJ;W0 z%YM}qlBI^!804SS^V|~QZRt@y!N0E>jLYGCMVEoQBE$tw9Kx&oYn0;}zM@##Zae!? zCgQaLoraFDqk}f3&_>(Im&~VABn#&%TrU9jdj0yfz4>$pifzsShr8@E&c?HMsq`k)W3Byb<0@iXsmz2Ejv7cT zfl#`Qxf$HdU&#OGAzm>ealEh%bM!jNsx^h_t5vYAGZVZ8jz5Alx{U)VdNkV;e`3_7~H zCP(ON`URZVdM>g3%hoZgQ@$tnYX2l`5pSPwSC3T{hvx# zCUo_R*0ebLN05IA!jbZIdWvc;te-5%&yN;oLGXkz#r4-N+Gq$uS0EAyA|pN|l#DZV zi2v;VB6p|gUu3;f2JffY=ik4d-n?TLlC$hk`9%EA*6YX59P=;qjs{%>u%!CBMM)R8 z!3hgOOOt*nZW}|w`JGDcm&Cs(PtJeE>rISH`)hjSRDNx&Yx%G%HE~mdArK5yvUxha zQKKYiYR@~Xcj@T@e6w>+TJK_#mMP-!NcPFgnZ5$)qk*B};eE;w zHnj|v>@R7(rIogR`O*+#E%n_NR2!FTUl<}$4YX!R-+Bxso-hGzlkd#chyX1KhVSgZ zc!H}JCx1nN>yC>=eHE2sln>l&cY2gY+C6k(-98Z5Y`98zhE%+h*kX{&Hz-Tl^lLKx zFt&ZR^kTDc<(?Y(YihZcoB+!gI#w$WhSambzf>^MVbK}9;CcmE*Qb9cCPNAAB(`Zv z_Dy7FUY9{{UPVkeh62VLLC9e*M;TYx{QsL_x@s zBaN|W=dI;jxd{Fi^Q4T0R^50>&WxE&4Z6pRcawhiKC+*kcfa#kHrPzp+?>wAjBj-L zMyA;Qt!Fz2&%i)&!@Vdm!MW^p}dK0LVcOZK0emrFF*Y&dGnP-|fuRh1V6j1`z`fHU*y z{g0~H-w+DjHWq7d*<;6)d^)S9XK1KooJQHc?C$IOk8G;xX~7K?M9|qwx3(6cvS;zd zSE9_LrQMX+xoB8~Nd7ZsS%#!{{LOH_muBBfh-Ce}8j*`kmea8-NNJC>{yjGIv%Eli z)s_MK#k6kFC7DC4ao&*oOa@P)0h2!`FR#>-ALI!9TSDBU%@>$9?w*|heHK!zbEfx6b{rlZt4ns}4&vXjey`HU)PfT@5JXJL2jgj}w zd2}_>^K#M1_33@Bv`+TUlA6u`4o1U4O$zxK_ZTegB zbWxc5g|&iwwJ(c++c>iViS9YSA@t|LR+|T0R`KHsT{%#{SI$Vy9gC% zI<*-{znQf6GS8ZO3^W+K#_MU&Z$d}_b6q=S=PI_`E7UK0K106CDd+Uhy}u!&K$(0R z6KK|;`3xu?=Wrq-uPMf2pOBA z@y!ur0TC0D;*RlA-vvgzX13_6+t8E!fGz@IqNL&#~14yPbaD=KqBc%J` zx6p19G5-tx_x{u#^Yp9}jRFO2J@67jHt9K0a|&ZJIwzhVSzCvkpZK?R(XP?DD{7CW zEu}Y?bMrSJ-R9z&gO3kP#azR7Dt6Cs$(4H*P0x*+H<#SI2oUXu7jXMy6|DmT{sFTK zfqcF>Z9v_=#(1zxku;-n*dT!!art*(DoZ*EJ3f%XiTGrUxYq$y_XZ!iUJeSPgj+u7 z?FV#9qV%$MTNFi{B2^2A5=kB!1h&jSjP7BJ517SNi7b@7XbXwORVlYJF=31We%Gw_`Jo|LHYmGpU%8>VYq~~|G z_vA`|a^R?yxBW9Sf7;AhyA=mDVMfcR-AYx1u`$sk+iJ(gNphe5pkW`(Q55p$x(9*)T7U|XWgNQ5feP6e0 zArLfZbi$}~h@i*8xy0*;Zsu;Z2WabvI1FUaF=7w(3jsE9KkBL?ISwLnv)!u@8(TqK z1A)kqLw-{%Rj$LY`nHcbuXeCoC(4|-b>!WLAD){Z7GlH0yVuCCfBP=jnu^Xr z&@t=0al*4>)yTkjve^lZ)8#rvl1$P+V{TV-H&nBIUByMq*oGmu-@q%)v?WMhwnqYL6)ihNN5gv5o<#L%!nNY3a%1TDmD23j@y1<6C6JJHLDz3)Tvn zGfM|&7QV6NE0)R8$UU20OSb8FZfBiuT*mI3`kU`7Cuhi4ZpD1#Vzi$PvPs(I6DUJE zwH3r3+zMly?;Ju}o$c~<@|lS@i?))-$D4ojeS7~ws7B@8$gB32d2dxm$ABmL#qW2^ zWY;{KbVo;DJX^bAsZ{rN7)u0wLO&(D#fFMHcDCE5&%v;yPF#*r=K}*yU)PqmG2F1s zoJB3eq>PEW+NnwrlxiB0 zO}W{hZx6FcN0DK+>qo6|)v1?%oyoY}>#oC>rO3gY}G3 zqfC3w`2?>1OL((}88Woyo1&bRXY`n>uG5p4ISkLj*V1&UWA@`dEfTEjeI2y3A#KGR zndp)d5)xu!b9+99`udDW2nhFU-gc5)!aS&swvS7*mR7udphsxSa7hh&vY)-baKV%fXZW94O*o`bQ=vExx~2+T)2rRjqJ( zZdtmz{=80}hB#i3l8PU+qe15nn97G- zIIw}GBVgNXha?=xpq+xnA`}lA&R>wY$tpa)i0&$rv#I&|J~%DkGMK>UQvr#=13bs( zMF7cp-nXau3d?%S{>(@lRV8;BtXcCOY`zqr*vw7v4oOm?L3QsI$Jg*#?2R z-2ESn>q@s9FhiFC_1r6$OTGsH)c2a^2Kwm`mk=Jx9SgiKq8&zjG;zaLRpKYp*1(j@ z;-*((wDllyojZ)MO!y`{D4Y#Dmm!$|%mz`$c~6@FdjYS3B|=R0+J4aezl;NHAEF$U zjP&%#zg#uX1C95w)9;trH0GHU+yW$*DNnrtjg2KY6+c{}zhNh-5#Nd}3?C*y+y{mrkzu&Z5RL1yT z$O2!Ei=l~0+0>3(^ETUmRi$TV{%_Ep-+6!Kw61Fc`6%@H0>T+3eUSh6`8MxAenCc6 z+GWr*Lo9VDG45q{itP-L7wT^$&Gb04H8wq)h-L4jaWOJ7ek<=0V63IZT|o`+3y~=$ z*+KSYlI(_i*tYd$gqspx^;v&h{_ggx;T1wWPt2O#~*vgTgd(D@w&=$xG&QjL5Y~>z4`B zgkj^)Ipitu&12JF$q9n|E*ddVEYOM@5ErWR)Aq*Fkg@Q>K zMosah<(%}TANE*{X)JX&hTiPoOZ3|sW~@p%5SId`TpROHz3mkne?t#&`{WMVEJ=LfV*GB zjLzHS-QJ&XsxeweYhiIAj*YA2e1&BL9{c&0r!W%2xh=Y61vVI9DRL}2fp7_LI*ya! z3w7vt)Xu;|d0S8vfAl?7hb!e!{rTnjzSxr=M-UD+EFb|73w$LwzhMdD(LcZA>@=l% zf3dh8WR`-$s!oFWlLRAH>+qeC{pUG~c!Goyfxs@{{fsl6prf9T&q?}SYmf@3Lj*s% z*W~~2-3BH8^ThNVUckEd^7I7jW3gI>Rh= zH)E64{YjwuJ1kZ*OBS`_VWm>4J^NqtP*$jS|Fe$C{>SFmi1~jm;*S4jKY9@S*AN)x zbS++kmoWW*`VvtenPJcLXj!m&;6c-K_z}55EX&=m%7Kl*?^5RKt>%_hHlEpHm&Q~5 zg#8LnwLWbv?{;db&n0Sz`N5`KpDml_rsz9r_AU2pWdRZ-e!mcUvWsl zl$Ak1)JC$wFu)q(bPl%pZxB2jf0@z;>>SKGZTM9ht(lN9iuuTmPEDJ0hW={6H8VG0 zi`n*8<+`Qj_Ua@l@&&lOj=)V3Eg_f&!(5u=Z!G&Q+VDr+Mn%9kxXTPOF~SHrXSr7Z z+=3z!#w>uAACrjgBS;oy8m&G@JAG&>s;|G!#f7B%Bjy+dNFD&$-5EryD;Os>zOAad zCR^ZSNHk#akII$w1(Xm7a`R?y&I@&GS!Iy6DrhH8=Fv*)P=OB>oWaDhLRott>!(11 z7Uc8i&)B&Ta{~;Alq@ei+Djk(?s1XAN~$ydoND!Oa@!kHIg~kitDTKc1f%>yNp(iM zDV*)?-9@us5PtVFuV!Q#{BI2~EB?)1K!^N>Z=EpFRjX+JTzUkZi@m)CrDBK`u)s7u zQ9PdypS!q}U*E5iGm9XKqmI>IFg!y#Zoc(d!=0Lzq6b4gMVAi~?$Z#KP28?mfWXW( zpO(Uul2F)SqgrkWxJZa{_n{Rv{=tK=dCR2MUf6AP{lLq|oo)_J$cVRRk)E zcVl`%3i3m6cbLIfTB*;d;6yzLE&4lj5t@{Aki_Q0{# zSrgT>>d|DXeDRu0w%5oE#f^UK&L{KT;f&8#xekzXUhgBVqw`9&khDyvXTx)bwRCAZ zdnq7uqHf3Qo*m#R&JYcoXmGn3Tc5sm#u>!uq5)^MC6iR)@N5R-_W(#u=o!)cjL@r@ z2;4}Bp#Jrh`x>(?VV|vlJRCiHU~SDT8_8lqiPIHOsScA~=BPd`_qj7Ikl@CXf&>k+ z==R{N{|D1s5QuB%({kuj^J0bEG%&}4S0WFzSb4^VWfSmGipQe#vx*#w$1*ZAZ`Z$_ zdP<7UXkhXm0M1>QEVhnteSR>F?osQWk);Zi4^k3YO7!FSOweia`kZsjwxnz_rBmjF z2r6P0M2Vj2vTvR7fQKpAXGP($>>s!LF&UhRO!zZF93Il)mwNMTsBv*h?+sYRB^e7;UkCT|7DUnb&U^ z#nGo{QK^QFv)ZR?#~M4txy>L%-dC&r&+^ypAqC^G6WM0wi+?ci1HA1c0{=R&=<%rq zl!;-qf*lke*tw8u?fYiQg#62Gx`r%;uMSiJANJ`T^g~#~Q;jW*Sv1ZqBJx2vxc%K= zTq4H2aChZ%WbG}?0wIaC%M9}+@gof9Bb;x z1fN>noo)yET_aF^!a@{U_(EJqlrON3sh;;#sF|5DAg&TjY5=F6kg&0{Ust!pjBG1N z$9G{9j$qERffo8|1LjoB?VZ$Em2iUZ!b2oG;n&Xi`<^aXM?so|2x+4+7N4J3j3Ak| z<G|mm8-g*fKpF>o>1~o`Ug>?eB#Y>6KQEwUDB) z>*PO}d2UspAB77eJD$Ppph6u^-}@m9P>0};d^(|Pa!yrNkA~8mL^Ui~&eH(sMBI<# zsALA*b$ssJhZd4^BR}&@ybNvqI0UEX^w|b%c>1iyl!FVU#iC#Mjp85W|}RASO&SU{#I!5ut5v z^(m1yF>K_-r9cclR%!|MGzYT8+Qkuff)H=68z8Bi;(E zFDMD5NNFtt0$O`ksf1lkVU#n*pi_1-F%utdCyOJ@a3+G+?ii!qU_(hzghOrD!0PXc45vZkxB+u~+Fc+@F?%OVO=4T%;Zq*}eZ3NU?Oz%{^oCMCMX44)W zA>M=W`#CUE;&kzEveAH`GwKNIK#x4NJc5exn)bt@f6jnhO0L9nRCgTeIlAo(^K`z6ICo*BUW*2B^;&B zn8&e7vWS^+G=pqg`5clXgaEUpj7ZYU2u+Z82s(=!u(p7--z~ zWHvysVbR}ddX_--v09Hp9<#W|3c7+eEImv_q0V;e=Lgeapt*FLuco1!AkY-nx7_0{|@8gb7YHi<8^4s(RzaBO;1nUU2*NAp!ANRcxAO0_?JE1#tIC%kWeiK6=EfcdhWx$+>glqB1?gx%^dwl9I%v@?#!qp*O~=t@2j@i&?x;m6+|5 zI&4u271a;SL?1=UAp5Me*4QPhB8sLbk%g-F{B_qdpD2fXi}X7bWKb^r z>`oG&YSQ)p{)-dUt%?)g0_r*PTUP3YI#C3g)GQz^$^=>LwV&G4)BO1_h0G%O`~iq` ziD&hyg?&Rqfslf4lxhYI@bC5hunM3^)!C8`avk(X&lqGQH~&fx&MvzP%q|%23t(fD z`V!jkGpt8>N~+sU?w>5l7`OHNSP&T8|M6kvX8?Qh2q^#QN-Qx4*3DT>q#nL z7a$qA_YDNzudeCoBew}AY%2(kggoNY#D;WxrVred4mV~ooI6fj1TNIA@=&zzsR5ob z7TgiXr$JIT4}%YCv923O;K<=-!(c8L`M`>u&w;NsA4F^Unja+I3NI;Qp+$gV#aXDr zs7f|L>4CBweo=nc;RU$SP9T9ZQ>_T1Ys5j`A0qzyW$h{vjS8a%iC`D9!$Ov@zZy&e zzhLRn5=N*Ns%mIZtX*~?EN4S4masvG4Hl8pC3~VjFFU;Nqi%<$bI3=su6T=rl)oqT z)+q?Na6PXqgH}*-HsSAgEmyvjF&>u)eoK=(;Rq|I6wJ7)dER0|m}4?Hl|)u~6cDKC zen0h&nJ5@XWLBi`wZy*<=y69|7$-{#L=#^pkAQTa?!_O=3N6|8N5Q{{G)v##Q#Mc} zS0$}?J7WSbdiD>u%enOrqsAmS=bxB+=bRS*?i-qPjZxqn{J^^hc^$WMHmqK@pT8xV z4jGa5^ZN@AJp}g^j%{DUo(pPrJkP7UoEW0@V8j~UImYkWIzBt7YKFNezzFe*GmY{WKitO*yVh=c_qA80?_&JzzK@&P66})mY2}g5Im0Uy4oW z=;#!D=1oyEFqo9O`a1-BBGE0O-Rp6>A*R=5f zHnD4)F!8ODiQn+omX%hBtf24&4;(~(Eq2U{c52rBU4}-hEG9U0yy-2$0di6?Ic12k49D=1gL%7mO2`i zgVGlTky!*)KFq%>kO{@)dGu)>rXD~NrTNGE_2%)jEJaw8nUH0ncrB}{-JeDI&s$=5 z7%Gs=I1~$=Thx*?VwjLyF=+%*&`3UStdx4S+6Y2=$+N9|l977#Jgd?B)RZEgYXBbW zJbKgyI#bx>?QCrWV18?7(lLDgMbKpEB|aR>d!0^aualE&0k*1i;E1-T8j8BFTKGE{i*6c}vhaFPg5e`Xvt7Zy)0q=-tw{U1AT}x* z9x844ULkqoi({;Gy^Uq3=68y2d3KcRfjWI$6%abU2KHr$w7@G4&ue!>w()zcbZbif zwSn~jE(1i+wZjU=0YT(FWb^^Z3R;^@rgDpCBwlDv!{3a`NyvB`f2jn)i}C~;9RLsk z_j9J+#l*lMl~|DHH8m!o53np6FO;t zde3^-1!zPAp#g!XNwm7z^7+Bt)U+bE24XSZ#?i=aXCD;f;do-Qhp1K8uQuX_QxO-; zICm-6vp4DGu zPI?tnT;$Q;O>Ta^o+N=hAcM$2FCB)pxwopQUH>z}Oc`*cM;`lDQHlZ4H` zlc-FhG=z1=3oAvMv5I7uCD<@jCtE7;g}%N%*zG{<$@{*nJAm&3z0QcOUCsDcUD``T ztZ!!K|;+_bn)$K#qLMW^K+cY1s&()~8@!$7-? zionjOg<28APYJh~hX7T20Y61`!gQJ(NO(JmRx~G#;<3jTJU*eW5$GL(k*pVG2>IiV zM%iAwDJBSrsH^C_5}h{wb9WxAD_(x?WnaM!Wq%&EbR0X2a_g?ck3gr0>Cj%ryy~#2 z>5@l-ZIoW}`F#OAsqk~44ynRP1PgMfc33ryRNpY;izgYVp>suPB7^Gy_4xK$-*(R&Ah%!*HEr6ncGlvNIreiWp5qqSH_YGNYv z?Nc;VM40K~hK|WTvF>*PA))9giCGfvE%qCx_U8;khxq*{G6-1&deCV_8k=lLqt)c2 zE8GQD%%C+EMjzUT_z#C~z4*pOl4`256Z#wBkF+6{9njTm7~3(H=T??y*%*LbvHI~4 z-SQzM1703PNWnp;uSY6mQ$Fuo^$;w-!ppn$on)4jiVX=)OR36DRmV@b%< zT+t8&yt@AfrTHnv^A^%CcS3k7I?;H$p1O4ovq#cPDNSAFG#*I&n2i@m{(Xl3%Hxpc zL6Nu2)Z)b~&yOOU+fYAi6yqIQMQbeszz}0qPVyE5=1=Opb9SM*Dl{}NeASYqDp)ES z%73V=6aHf`f6$L*h*6I7$9$H}e6L;@|FfiI{V?&sY;kQPt?yVZ3lNR^d2tEU=t^-k zGcYJxyD%?9>)|j~+$u-3qvT63c#PuPBz7llU)39%zmaMF_VISG&Ad1#JI;5JRJ{_v zSrx}5--txzZ}J<|U^5Bl@BPO5V=#Ajr?>Rp^HUVuG>3VD?ES`7!Z&Ado!29*GC7%< zSBV1u$h~|tF*OB)#N~zj3eaJv1Hl3a7ZkFo)yIg_rMNgbAGZ8Nliup9!An4QFI)2G zY7c*{QCw2@3s4zAfUGImoGnS14;7*Bg?jiL_l*=BUoZxD!FM$_&k?ei0Z<&*D@CY} zX%yG^*<`@LXF9LKBsD72r$ipV`0y#^UeAWRzMzbIj^1#E^XVB`xCDlYdDt9%sKkDI zP?fu**b$Y?c#eXEE1O8>P%rBOMNV#1t*bV*y5?=5NCP)w5jpElj76R6m0Cl*Dq5tz}DN*q8AC0DhZT>e2D$P6}P1T99DT#7b5G{$y zUs2_#5+|@oSWLyRDXIMzQIjz|UDiXeibuv$O1? zR`rMU_*x1|wHRP`t}Ysy6vEg!Bt=G>E%=xcB;_li(Y)@UMMV4{^9C^e5Wb=sO$#Pe z*{#?MTLEAePp9=b?rkiy$I4MHY>J9xOmT|EUc|) zqvEvm+!?M&Lhr|k1zw7iIry57rrUEJ2GjI}+!YG{f`#lqpJ(4?7-c-5#1u!S9zg? z@7ZLEN5S0)v+LzKEgI%*`KkJ=tprbf#o-~_PWH64Tt+I<=i5^&w%snGzw&n;`^o#f zz+u+0&Ob~jD|Yq1l-K8|s3^-pJEo7CyU>weKQMaN}b|E&}3k%Uz#mx|@%yjF5pCGZSstA6p9_YW^xzNi0LPX_zCx3^jzW zDC|#bUhNIYh(!#qoV|m}nbK^EmQl;n(h@>sn9CLIu^p?_g8uVSRF##D+Fk@4);+!i zJNYZaSV*@kDpQ2>$GL4gzS`>?A^lDE*~*kuw_5?WblHTE1oJDFe3x_NX#gsM4w^+s z|HAXcpzX;*l5lO(virHz%duJ<9Hn}HIs%BvgkIAIEPs$!qV#otJBg&FLTW9wti?4e?<^BtNRsSy zKcJWIbru304h)zVBEk;qK%z%kuS1TlQky~oUgAFF#zT3%5gRv_2L+@&#>Hptp!|fA zAEBVJm`psq^7x_qQ>#ss@3;^z_2bCBhEoVHk~8s2Zuc;SX(O4cdG5mXAT3E`K-Va4 z^gFW0)THj=Ts)0JmSO*f79j^Fl0c<(U8Pl81cQrvbsaF2&-$@ncIems5NpyfMkEH< zhy6&6$}JaCH3HHa1SbJ2xk}5fyZ0VPKUWZM*q__8G|2x4UT%%3($gplVB0$HP^wEv*F~{7<5WSBY!bicF|qj?k_&JMW8-XaVDFpQUvl$QHsB*K9Ah>*=^W?3clA< z?|<4{G{GeLw>%uIyc5z9X$M=}Kp3fkt4aAJ^wDXlsnJlisYRlbb=+w`I0EVvWVb4m zqcjb31AO=jQIQ`hziA9N`XC`_w~?d&WNS;Ai?j_aj`U(aF`I&;(KQ;ksB3~8?-T13 z@}(;?xv+16`2ixDp*DBpgzTE6)0W%|ba-}8|-fzN^nRAM$7S*XvxgWemu$X-w1AW;GAm$fKw z|Mt1i=iUrq7!l-f82Ydkqxcj7xD?oCL;qApbRzxc)>|ZvSpt z6FOt5t~cqgxsowE%bN4B%Fae6cyVBD6eWHa+g{D?`ZQQy5)Zm&HrV{#$6Wjkx$1R~ zxU^+!%jYk@Ke}@8ORHe>SZI^ND`wk^@K&Pa8vDqjV`P+`nIZA`=e*BxSi%c}cc%jg z8VUI>7Xg;^TenhqObj6Y!$F}_UEoM2t)KP5w(OD+EAoj@gab!;+vRdU2Wuuhmz=`4 z_w#XQTMvSn1LFAfz@J!r2$3C2%}>foCd1y&(Zxs3OQ2rE=pP^|j%;cntfVMZ#40xW z#lJCs4ok+1ba*0t#b2;_LLO#G##+L4GY!5HNPHByp&BVjZ5eYzAM^0l#8u;ZstNMV z!wce4$t}+&7~;1+B;`h-*v|_VCTF2HjdQ45Q-jNg3NPuiXpghr&Kat-r3#o4`;)1U zjixFDM81JCEe^$ay7G}o zlRF)JF_4;Qf{RdoC6CRS()U(fsrKOALk`Oo#~2F`Wft#XinQ~@YAqx3k5tx=4vy)1~*E~hK_->~}+fzulObkxg4ZAFJ<*g5Xo}u34c_G)bGVs)Z zrv>6-6&$!APs8M87hR$xC7BV=MydqkkQs-ligq&c+`X2>N{NF}8Y;+-=RaFMe|ZHb z8PrX(=>8!L_v3&XkGd>%3!OkHfh~>Y8s{ega}6SFeDZJs#@TvDEnC z@5UaFO725g#ae~%!<8vh`jVMt?md5(^*QFai%Q48w-&3(6`(#)aYxF^Xcx+tPdXVE z9CyI5gy=nW{*0I+heDm1&i$*Yaqe8C!+TGn48^qFTn<&^6P#Rjt2gYpeG7e;xJc;_ zkPj1o^*gN%X_!Auwn8sot9~fd<5(wo*ZL&yB84f6??7?aImw+Jx2{)Pb2BpT5-6$A&9PEe+~Wc;T^8IEQPkUZSC{{7uUx4Qo{xNh=C0U?EX31Cq$v!S34hd%K~DDmrkj!TOlcT!B`!K-qYjw&kslP)|M|aJ5(^P}?m0xP7$`-e{tIB!(s@yn- zUa?SC^P_y~43x{zYRhCJ7S83g(@hx=zSL!GnwMmVGe}TSjHSL&?}Nr3PG>u-m{g#= zjgJ7;9Zb9x6BBFiy-DeifXj*u2qn&=Ig;Z_!o9q;;j>DX){hbCcI@;DUG{j!@DXp| z>hV?vTbgvOnsY4|Rv zE3D;btJ~DO(Gz#%&o4ioULC(PDbhcoxgIa=_USL%nz{u{UpN>ciVgTfbSF(rN0ec* z20Z`JKX+R)gjLFDkN*}zvu8jT2XeIuK+j5!i96mGxg+q8ZvrnfdPcbO!@((xB=+ue zi0|{89XP%vmm{_Hf%kl?<;TJjeam)H*68SHL3w?Tz$SFJ;OXqrWXCgWkWi=l_q1T# zeIoU8hrQ`yCtK?3!V3hQFf{u^>&dJ#=a@F+pRHpnAUYI>iyvK8Kw1*uj=PZN2K~Pv z?cZ2P&d7mC%_7FGtBcVc*Sp)|vuLTd5_FpHxicK3AoR3_&M%;=@+n3Y{@mt=(#%<> zXD6NeVW3g*wHtaQu=e5HwedKv?bC(m=Ys^A5fi#0kEi0FNvp5SEz-&t++mW8^6fF1 z>%K2Y$#~!kz@6jcNVmTV_BavD_?jm;Spz=c6@%{xj6Q9(e_}5imNFK~_GyXK1cZg| zvwd(aRzyQq3&aE-$kZ7JKx|U4BU7x_!-FWQ8v4JhZ_*hxrG2gV@46PK7#vfH25$FZ zebJbPpdlz6yGC4(NIqh4ad81>M6V=>CV}h}**H#c;vhkHpQG2${vZM_ZjgpUG-A9X#5P%#YqC%JACF^TP`BV$;-s;`{2fvFY z?}ak24IkF~jPg;M7M+PaSZe;4gEh)uQ;QRIJz)UMx)1`V4e0(S?c*-xY1E}GHMXpQ zdJu*d7pp4;Kh=-JDD?RkDCyymFjAhzf7iAn1Ow)b{5!(uSx#p@qOZq=)M5L#Y@#3A7i+N&%|(Ni&uum+ha}t z2+nOc4ZV-4mKMC(u7wG<76oZU-6jIzIEacE&p|`Jt-`;L`AxrF%c5K<8SS;AyjajN z!!u^z*YNM$1RIXsRTl51#fido(kA$Qc8!MFidos;-LM&XM+Uh;joADd-2Spj0!qmk zZE#Ng{_Po^H<-igasRbq5>*x5_on?9@6eDzn~4Jm?4W99C1ym>93d_%Dx!ga7z?}I zR?d?Oftq&X7e17#TzPvzzU~jfdVy?rz?_XD25-pk#^B8kOVn`G9JB}=BWe`X2oeuE zbWg1knGZI~$B&n2Q8$HTi`KH){B+o&y778p#zkW=;Icuh(RV05beJXN0;>0&`KEgpK7q;ZA+p`_NG%x|8Gu#I%@_AyoQfe} z{JQ`4`d_;&G{&_edxVB*x_A!SD@4w!U)*M|t=or(hl@)9_;TOT-2a(fF4QuyJ^1sr zi0kea9xLni_O{8>=jUg6h7byJU75zwY)>R@`h_rXM1yiKOW+Tjhya=@#nYMJ0^Qej zxZAd7;O}q@!Y?86ly>Du{@~JqZPB$s?B2aGyBfiNLc+o~5plF)m6rO- zI3Z`IDYw2^H_chjOx?d&P1oAm)3J2Gabo9*MzKl0Hds(N*&os=cL$q(y+@a!Gcf8} zsHoILV;m6DWv8yDQI#IU=|H6@XQpUp5O8gN@v#mtbM$Ag`3h5Rooi+w3-{J&B4kye%M9WamA~VRQcJ{Ee`Y%MsGql|-B3+tEeJAzGz+ z$#d#`Hl9r(Mf08+rnx%J*&1Q8jE1GAge_1Y{+20x#g%-gOr>~?F|5zB&auuh!_+}o zn2nWwF{^H#oSZ0p zY$GH@tur>J9vjQ);6RbYDZ)Bob(fuu6+AB81uQKKId7-LXf(1lc08NF$kDIvFzy&# zG@;76ZmEac6tPb3`D!^K1U(gtlo$(1L#)}0-OZ$<@&7e* zwkA;;NRmXG%A6rI7(=FH$jV$OB}9AeLJ~5QDMP#3G9_dx%1}fKA>v%Gz4!Uf`M&R* zALrNkqXBEJ_kHi@e(vYKufdh`+4Zn#9=FvtrwlUQsTG)~+Mm*)8*Gj<%V_cKQKX;0 zU+DYCVDksQ9=)<-xBEO^pZl&E+M5`2)3(@@@0y{by}OOh{WeEky+fOYg$-YBda<$) zW6bQVO#P-$j`uQ0UzJAc=y^(UxUD4x;;F=^WG}9!=O59pU?V70mj_lZbyn59qB|Dz zd!4J&96WFDs^Hal8lYI>{Y8S2FOc)6Rf}_hKk=@(c3F9{CfS*kj6h1*fYqqj*H8 zs~<^5=b)E7^htmS3$P;pQD$lZHbFR=L@1D&ikRbR0$9vOQZ=8=fT@cy+$~6j9X$yg zA++Kj9_g^jebv?mrEd#Ibwb5|p+sM(n~+yjTm<=gL28TfUksO5YejJ#KqitPqSZ-l z3HS>u;iKKX72ES{()D^7>+~o{|7( zsAPfhdH^|$t*z-ic_uYG7-aJP`Zk<%%J{gJopA(pU-4DcUs`6vurZM;t%95~*?*(v z4>^*qIT9|eHy(Eaa0NorZHT^L#t(qYo(SeuAM`ccMOc=Gcl`sbxA=46Z5Qc+!Tq9P zn*>i+jZwbIoc4;63w>V#bikJD%>I{tEG=G$u~+%!5VO0YI&YCkB=xCHZjVRnAsc-< z1fy~~pau78&@Icf@C;_~L21 z_SUc3Q=P}mPCFIKxa6yHvcD@9$#^3*c7=EF!SRakH_V*BsAdp3u1Qq^Vp4*7M^;te zIv7(o;&O3#4wPSVx@r^94y;7LNJcuX2@!`~FDMRD2hc;+qJRZFJUIB*J0Jg7=8O9H zl0osetb!ubYsH;wSyp`MzM}1N)$eJT+qbc9SsLqU1uowXr;-;{mv3U9;gts{X(B@J(@xcN9`i`bqI^GR)- zOaj)a*lWEP2?xTG-PV+KFOLyjeR2wY@aKv;C0+5XdG4tW8Jna?i@O#hV5P^!@%QI1 z9UNOS%XKQ0=YBaZ65UtE7oq9Gcm8RZePR-^b%*?Jqmn1@?oIpiS``NIXs^vRo7l23{63Y_)h?Jl<+o1*Q4b7o*L^xDMk=_>FzZ&Hq^qR(Zh&x_*|7w zI7P;r%DwC=zS@1}&>MSqK?(wD@eThA5lKLX*o(i-bLVwgB~lLDF*qg5Dy&?WQiGG2 zv^hh5r`nlU_n=9Qep8svdx@;lE9wrzihg}Q{?MOQRaHe*1L0l>%GVTcIDW>Zvz?Qg zmDN7rq#UZQzEC9DowVInY-%}H9jcbvW2{ti4{7g;7q8CMBA4l^j=F*7Wk!q=o$>!C z-R1A}@h_yCRX+QT4Gi9SmL=(UrwprPH(Xr%^oyF zGIill!IRi9g!>5WGHSMVN9B25--Iov4It!f7*4WH%$*eaJhop@I~55^MZ|AlXpl^n z6lLyw(=5)~0w1&jtxbv=Y9V1J{^lpph?K@cHTQ?{YzGGC@6?fLlp`X-=*C5 zvb4Jj%ap>+Zl|A&jEp!}&J6(+n+d=r$% zqB8OI0L5cgFln`XFIo9S`U4r5$QPyaHT$>KH#a!b-9 zhEj7bCm&vLVb^x1E#bu&L}hpuYi&2JIiTg=#z!(EW$|Lz2<0M#)V!4lNIYKGk$(0k z?*Wro=tSn{=l`HoK!O;EWt=g6rPZ1wX^A?CcMLKpj9}F+)e=D9Ueah8vfgrNdDb|# zq*UCU)pn-aPtv2*Jg9sC$qk~4D(A>gugM(#v&6qync`xYR|2!iqX9CC;5O%5m@AM^ zs3lNyzskQuHj;QW928JX?7%*IaRtZ1U-VBFDXz%EuzoqS%lNykWuyUOuLD95*9<=n9A);0U!szUtQ@M#rS zXt30C<$QhoKOp_ro`c~VQyOaD2Mgr#VUH zRCn6!@0Sg^m#d#5N;Tu{iF7OCk@Ilw&=b&KG_&h)g_ctq#bBv zc~ccys1sX4$uPMo$#`YrAK=a|15xjsyu3}EE3L_((F#B8SO3C%w)dG*!QPa`?VMT5 zvE*6pPOGZfi=l)K4hF86bbRr=qOLr+%JvMwcpXiIwB$h2>8wKzZf553!c6368DK|(qq3bmCZ%yYXfEvePJgSTk`7YduPrp87x zS`l6tnxq>qi>Bd1fu;>W1KK~S)so}kY;gqxK~R#9?p$AcNz>4<^`{My_X&bC^nj-N z5zuc+S2tCclPqr_PAbyV(~%-8E8iG(0^KkR{t%xAAnx#K+^MFvR@*&jkN9%(RwEiW z`K@Ip!fuEMHydfY2K%s48>`E6KLVwwicE#6D#~8JHgK>&g#%vKDDGwSuJ3Z{T3}pC zCI08qUu#{>NGOB(%|TI&V8yu8FzT*YC_m6VQ!eHg?#Jh&V1P&JI;H07FAS`qW@k_ZRklG7+@NsJj@>N&KF+S(e~D7 zrYh}oqqolW^dC>@RTGYFj>@c?gkabdt|?JvC=OK0HuYws>YXz^IER*%l-xTkw{YCU z-+z~+WN^2rzCL2~Hsa!Wzbgm<`r*7&**H5p zg&%bBH0ffglS4Z_;g7NK>s5Og9j`d zU%tGmD>FI2W~!Klj=z|3B1t^Ff6b0u@IdQAm<9^uXV)Dc=KabouY2GCj#q$@G_H=- z)|9_qKUkT)j%RVHXid=L>5~=w9Ly52a?;XIK6l6Jq#|$@wyv+oB(q8#dlXX3f;}{L z3>B`=LhGMrpE?UQwE$}1RaI43VZZ+RxU{iB@JvxXn=NicS8;%c^$T)B~feB_Fh?MIcZOhB{O4~ z`V5HngLuhQy1%5PMDW?p8ic~0ao)y;20MHEcVzP6e=_)dlh&P$U1o^vx8Wt68u;^Y zc$GLxrU$|2EY9w_&qNRtW=|}6^mKI}zLKV_IDmZC(#=H>b;YwB8eTRZGYmyq-Z10e znX}v)?O&pE=I>On5k#2L7cp9#UBXcjepZ4AcRWhoKrm89BRYI&f(8_8YFOUwVCV0iL71aS5kPp5ESqKY@u}80$bFehubL>Q0gbrbq zwVMFG1C{!z1?WN07vSpj*lCTWwtq8GB}fK>^IhGllyS}=*$$H#w2(s3yrKr zrdd7T80AlxD93Da7NzguS+X=_BI@XU!4Kif>wfmk8HH`z;(2N=Fb|k`pqY-wAW6q_ ze**fo&V?1h&uzb&ZxQrs{1w2Z9&_2n;#!MW?mBV9mjpuq(tzB)CH1YJ*XG@_z7ha-&F52-p#4K*Y=uj8yOSu7D&j9Ez8`UeicCoj7RPYRo^ zuV-;W-Le z%sXTX1%L z{TeZU^~jg_-}jy@?4>`#J(s$oa^5xoOVPgTN2y^AWnm$K5i!AiR)B^b@_5Gv0gfh1 z$Vd^hd?(dEB=>%cev@PQdS>|4X9UW-rq2=-lO4QD0ZVTuVVu$Oh8=N{@OB>$pN3q* z?b1P???U+1Diw|zJm!vh4ldyXe3oF-=XFDkujHqb>a)eIOJhG)=uoJ@7{>NsB`KK0jfPIZFDX7;QGKbwmw}gucK?TixT@d6HF-j8Mf*r=R)<@Vm%%Oo#N{n*QDnY<{~iqi(=1OAN3kuy$`gbnq(11{ z)!8|Rxci3ar9&7%gWl}aJLg~v!R?!%$6|gCxAb@KY&vAjL6R@^ogRTna&bXHTtWgq zCR*M@yo`kAKBlZwPsCT?^a~2M>*>?-0iEcr0b>ST-MRY2q@=fiv);P3b}&$a*0@gS&vPwHz>9Xc9@+9 zn=4kqDRizLAQ)7k-Yg4laGj|H+vSo*wI!4trXDkKqkZr-ekB(dybphoh$Weo2!n+u zcw??uN)6N|QEx&|{P*9r%6b<;rzxx44!dp@6~XbHbtMf++vYEyArpxRHf-Q=yyy7sp@cOIFZxin8t0 zoc@%q1k+~UqI0%gkRu<;df8(an|*xKyT4CO6&PepJn7zTY_S>T7oy=s-?qety2}(IZ^D-?>RsfjSv%_%xd@)@y>R z4;vb8R#S5+IOR9Xnm~1&x=UR#v~3m6fHZ?+1w)=(6LV z3zItX#Feqt()rt(?!3ue-`(DRBR)PJj1&NM2M6t%5<$WQ05i*?tmx4r)AZIAy-&xs z^<`!`B%JI{OOwf|I$BKL*-j~tg1H-oRv>T$A}uc?;cIeUk>u)6pkAR=+W literal 0 HcmV?d00001 diff --git a/docs/src/man/mle_reml.md b/docs/src/man/mle_reml.md index a9c4a10..d31d94b 100644 --- a/docs/src/man/mle_reml.md +++ b/docs/src/man/mle_reml.md @@ -8,16 +8,14 @@ Machine information versioninfo() ``` - Julia Version 0.6.0 - Commit 903644385b (2017-06-19 13:05 UTC) + Julia Version 0.7.0 + Commit a4cb80f3ed (2018-08-08 06:46 UTC) Platform Info: - OS: macOS (x86_64-apple-darwin13.4.0) - CPU: Intel(R) Core(TM) i7-4790K CPU @ 4.00GHz + OS: macOS (x86_64-apple-darwin14.5.0) + CPU: Intel(R) Core(TM) i5-6267U CPU @ 2.90GHz WORD_SIZE: 64 - BLAS: libopenblas (USE64BITINT DYNAMIC_ARCH NO_AFFINITY Haswell) - LAPACK: libopenblas64_ LIBM: libopenlibm - LLVM: libLLVM-3.9.1 (ORCJIT, haswell) + LLVM: libLLVM-6.0.0 (ORCJIT, skylake) ## Demo data @@ -27,7 +25,8 @@ For demonstration, we generate a random data set. ```julia # generate data from a d-variate response variane component model -srand(123) +using Random, LinearAlgebra +Random.seed!(123) n = 1000 # no. observations d = 2 # dimension of responses m = 2 # no. variance components @@ -40,23 +39,23 @@ B = ones(p, d) V = ntuple(x -> zeros(n, n), m) for i = 1:m-1 Vi = randn(n, 50) - copy!(V[i], Vi * Vi') + copyto!(V[i], Vi * Vi') end -copy!(V[m], eye(n)) # last covarianec matrix is idendity +copyto!(V[m], Matrix(I, n, n)) # last covarianec matrix is idendity # a tuple of m d-by-d variance component parameters Σ = ntuple(x -> zeros(d, d), m) for i in 1:m Σi = randn(d, d) - copy!(Σ[i], Σi' * Σi) + copyto!(Σ[i], Σi' * Σi) end # form overall nd-by-nd covariance matrix Ω Ω = zeros(n * d, n * d) for i = 1:m Ω += kron(Σ[i], V[i]) end -Ωchol = cholfact(Ω) +Ωchol = cholesky(Ω) # n-by-d responses -Y = X * B + reshape(Ωchol[:L] * randn(n*d), n, d); +Y = X * B + reshape(Ωchol.L * randn(n*d), n, d); ``` ## Maximum likelihood estimation (MLE) @@ -73,16 +72,13 @@ To find the MLE of parameters $(B,\Sigma_1,\ldots,\Sigma_m)$, we take 3 steps: ```julia using VarianceComponentModels vcdata = VarianceComponentVariate(Y, X, V) -fieldnames(vcdata) +fieldnames(typeof(vcdata)) ``` - 3-element Array{Symbol,1}: - :Y - :X - :V + (:Y, :X, :V) @@ -98,20 +94,13 @@ When constructed from a `VarianceComponentVariate` instance, the mean parameters ```julia vcmodel = VarianceComponentModel(vcdata) -fieldnames(vcmodel) +fieldnames(typeof(vcmodel)) ``` - 7-element Array{Symbol,1}: - :B - :Σ - :A - :sense - :b - :lb - :ub + (:B, :Σ, :A, :sense, :b, :lb, :ub) @@ -123,7 +112,7 @@ vcmodel - VarianceComponentModels.VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}([0.0 0.0; 0.0 0.0], ([1.0 0.0; 0.0 1.0], [1.0 0.0; 0.0 1.0]), Array{Float64}(0,4), Char[], Float64[], -Inf, Inf) + VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}([0.0 0.0; 0.0 0.0], ([1.0 0.0; 0.0 1.0], [1.0 0.0; 0.0 1.0]), Array{Float64}(0,4), Char[], Float64[], -Inf, Inf) @@ -150,6 +139,13 @@ vcmodel_mle = deepcopy(vcmodel) Iter Objective -------- ------------- 0 -6.253551e+03 + + ****************************************************************************** + This program contains Ipopt, a library for large-scale nonlinear optimization. + Ipopt is released as open source code under the Eclipse Public License (EPL). + For more information visit http://projects.coin-or.org/Ipopt + ****************************************************************************** + 1 -3.881454e+03 2 -3.853179e+03 3 -3.846525e+03 @@ -161,7 +157,7 @@ vcmodel_mle = deepcopy(vcmodel) 9 -3.844374e+03 10 -3.844373e+03 - 0.290970 seconds (10.45 k allocations: 24.036 MiB, 4.73% gc time) + 4.546981 seconds (11.23 M allocations: 566.109 MiB, 5.23% gc time) The output of `fit_mle!` contains @@ -176,7 +172,7 @@ logl - -3844.3731814180805 + -3844.3731814180883 @@ -184,20 +180,13 @@ logl ```julia -fieldnames(vcmodel_mle) +fieldnames(typeof(vcmodel_mle)) ``` - 7-element Array{Symbol,1}: - :B - :Σ - :A - :sense - :b - :lb - :ub + (:B, :Σ, :A, :sense, :b, :lb, :ub) @@ -209,7 +198,7 @@ vcmodel_mle - VarianceComponentModels.VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}([1.092 1.04727; 0.955346 1.01632], ([0.380637 -0.305465; -0.305465 4.51938], [1.84009 0.265569; 0.265569 2.17275]), Array{Float64}(0,4), Char[], Float64[], -Inf, Inf) + VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}([1.092 1.04727; 0.955346 1.01632], ([0.380637 -0.305465; -0.305465 4.51938], [1.84009 0.265569; 0.265569 2.17275]), Array{Float64}(0,4), Char[], Float64[], -Inf, Inf) @@ -260,8 +249,8 @@ Bse 2×2 Array{Float64,2}: - 0.042559 0.0487086 - 0.0430588 0.049178 + 0.0425562 0.0483834 + 0.0430596 0.0492809 @@ -276,10 +265,10 @@ Bcov 4×4 Array{Float64,2}: - 0.00181127 -1.98035e-5 0.000240705 -2.59506e-6 - -1.98035e-5 0.00185406 -2.59506e-6 0.000247285 - 0.000240705 -2.59506e-6 0.00237252 -2.63542e-5 - -2.59506e-6 0.000247285 -2.63542e-5 0.00241848 + 0.00181103 -1.96485e-5 0.000243441 -4.38252e-6 + -1.96485e-5 0.00185413 -4.38252e-6 0.000246407 + 0.000243441 -4.38252e-6 0.00234096 -5.73331e-6 + -4.38252e-6 0.000246407 -5.73331e-6 0.00242861 @@ -311,7 +300,8 @@ vcmodel_reml = deepcopy(vcmodel) 9 -3.846631e+03 10 -3.846630e+03 - 0 + 0.443964 seconds (8.09 k allocations: 62.532 MiB, 2.39% gc time) + The output of `fit_reml!` contains @@ -325,7 +315,7 @@ logl - -3844.3777179025096 + -3844.3777179025046 @@ -333,20 +323,13 @@ logl ```julia -fieldnames(vcmodel_reml) +fieldnames(typeof(vcmodel_reml)) ``` - 7-element Array{Symbol,1}: - :B - :Σ - :A - :sense - :b - :lb - :ub + (:B, :Σ, :A, :sense, :b, :lb, :ub) @@ -358,7 +341,7 @@ vcmodel_reml - VarianceComponentModels.VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}([1.092 1.04727; 0.955345 1.01632], ([0.380594 -0.305485; -0.305485 4.51994], [1.84285 0.261963; 0.261963 2.17842]), Array{Float64}(0,4), Char[], Float64[], -Inf, Inf) + VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}([1.092 1.04727; 0.955345 1.01632], ([0.380594 -0.305485; -0.305485 4.51994], [1.84285 0.261963; 0.261963 2.17842]), Array{Float64}(0,4), Char[], Float64[], -Inf, Inf) @@ -409,8 +392,8 @@ Bse 2×2 Array{Float64,2}: - 0.0425909 0.0487744 - 0.043091 0.0492444 + 0.0425881 0.0484485 + 0.0430919 0.0493475 @@ -425,10 +408,10 @@ Bcov 4×4 Array{Float64,2}: - 0.00181398 -1.98331e-5 0.000237127 -2.55589e-6 - -1.98331e-5 0.00185683 -2.55589e-6 0.000243624 - 0.000237127 -2.55589e-6 0.00237894 -2.6426e-5 - -2.55589e-6 0.000243624 -2.6426e-5 0.00242501 + 0.00181375 -1.96783e-5 0.000239868 -4.34611e-6 + -1.96783e-5 0.00185691 -4.34611e-6 0.000242745 + 0.000239868 -4.34611e-6 0.00234726 -5.73082e-6 + -4.34611e-6 0.000242745 -5.73082e-6 0.00243518 @@ -441,18 +424,13 @@ In general the optimization algorithm needs to invert the $nd$ by $nd$ overall c ```julia vcdatarot = TwoVarCompVariateRotate(vcdata) -fieldnames(vcdatarot) +fieldnames(typeof(vcdatarot)) ``` - 5-element Array{Symbol,1}: - :Yrot - :Xrot - :eigval - :eigvec - :logdetV2 + (:Yrot, :Xrot, :eigval, :eigvec, :logdetV2) @@ -480,7 +458,7 @@ vcmodel_mm = deepcopy(vcmodel) 9 -3.844374e+03 10 -3.844373e+03 - 0.018754 seconds (9.15 k allocations: 680.172 KiB) + 0.042187 seconds (21.56 k allocations: 1.366 MiB) @@ -520,7 +498,7 @@ vcmodel_ipopt = deepcopy(vcmodel) @time mle_fs!(vcmodel_ipopt, vcdatarot; solver=:Ipopt, maxiter=1000, verbose=true); ``` - This is Ipopt version 3.12.4, running with linear solver mumps. + This is Ipopt version 3.12.10, running with linear solver mumps. NOTE: Other linear solvers might be more efficient (see Ipopt documentation). Number of nonzeros in equality constraint Jacobian...: 0 @@ -556,11 +534,11 @@ vcmodel_ipopt = deepcopy(vcmodel) Number of Iterations....: 63 (scaled) (unscaled) - Objective...............: 3.4496886481728075e+02 3.8443731733053696e+03 - Dual infeasibility......: 2.2693631692678575e-07 2.5290047242499938e-06 + Objective...............: 3.4496886481728779e+02 3.8443731733053728e+03 + Dual infeasibility......: 2.2693631701157965e-07 2.5290047251948971e-06 Constraint violation....: 0.0000000000000000e+00 0.0000000000000000e+00 Complementarity.........: 0.0000000000000000e+00 0.0000000000000000e+00 - Overall NLP error.......: 2.2693631692678575e-07 2.5290047242499938e-06 + Overall NLP error.......: 2.2693631701157965e-07 2.5290047251948971e-06 Number of objective function evaluations = 64 @@ -570,11 +548,12 @@ vcmodel_ipopt = deepcopy(vcmodel) Number of equality constraint Jacobian evaluations = 0 Number of inequality constraint Jacobian evaluations = 0 Number of Lagrangian Hessian evaluations = 63 - Total CPU secs in IPOPT (w/o function evaluations) = 0.020 - Total CPU secs in NLP function evaluations = 0.256 + Total CPU secs in IPOPT (w/o function evaluations) = 1.488 + Total CPU secs in NLP function evaluations = 0.375 EXIT: Solved To Acceptable Level. - + 2.486697 seconds (4.13 M allocations: 201.947 MiB, 3.03% gc time) + ```julia @@ -629,6 +608,7 @@ Here are a few strategies for successful optimization. ## Constrained estimation of `B` + Many applications invoke constraints on the mean parameters `B`. For demonstration, we enforce `B[1,1]=B[1,2]` and all entries of `B` are within [0, 2]. @@ -646,7 +626,7 @@ vcmodel_constr - VarianceComponentModels.VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}([0.0 0.0; 0.0 0.0], ([1.0 0.0; 0.0 1.0], [1.0 0.0; 0.0 1.0]), [1.0 0.0 -1.0 0.0], '=', 0.0, 0.0, 2.0) + VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}([0.0 0.0; 0.0 0.0], ([1.0 0.0; 0.0 1.0], [1.0 0.0; 0.0 1.0]), [1.0 0.0 -1.0 0.0], '=', 0.0, 0.0, 2.0) @@ -674,25 +654,18 @@ We first try the MM algorithm. 9 -3.844650e+03 10 -3.844650e+03 - 0.031954 seconds (10.70 k allocations: 781.828 KiB) + 0.170236 seconds (170.93 k allocations: 8.918 MiB) ```julia -fieldnames(vcmodel_constr) +fieldnames(typeof(vcmodel_constr)) ``` - 7-element Array{Symbol,1}: - :B - :Σ - :A - :sense - :b - :lb - :ub + (:B, :Σ, :A, :sense, :b, :lb, :ub) @@ -737,7 +710,7 @@ vcmodel_constr @time mle_fs!(vcmodel_constr, vcdatarot; solver=:Ipopt, maxiter=1000, verbose=true); ``` - This is Ipopt version 3.12.4, running with linear solver mumps. + This is Ipopt version 3.12.10, running with linear solver mumps. NOTE: Other linear solvers might be more efficient (see Ipopt documentation). Number of nonzeros in equality constraint Jacobian...: 0 @@ -767,17 +740,17 @@ vcmodel_constr 45 3.8446498e+03 0.00e+00 2.49e-05 -11.0 1.51e-06 - 1.00e+00 1.00e+00f 1 MaxS iter objective inf_pr inf_du lg(mu) ||d|| lg(rg) alpha_du alpha_pr ls 50 3.8446498e+03 0.00e+00 6.76e-06 -11.0 4.08e-07 - 1.00e+00 1.00e+00f 1 MaxSA - 55 3.8446498e+03 0.00e+00 1.83e-06 -11.0 1.11e-07 - 1.00e+00 1.00e+00f 1 MaxSA + 55 3.8446498e+03 0.00e+00 1.83e-06 -11.0 1.11e-07 - 1.00e+00 1.00e+00h 1 MaxSA 60 3.8446498e+03 0.00e+00 4.97e-07 -11.0 3.00e-08 - 1.00e+00 1.00e+00f 1 MaxSA Number of Iterations....: 63 (scaled) (unscaled) - Objective...............: 3.4484507551949008e+02 3.8446498170293403e+03 - Dual infeasibility......: 2.2694405349430929e-07 2.5301808715939735e-06 + Objective...............: 3.4484507551949679e+02 3.8446498170293380e+03 + Dual infeasibility......: 2.2694405212011240e-07 2.5301808562731130e-06 Constraint violation....: 0.0000000000000000e+00 0.0000000000000000e+00 Complementarity.........: 0.0000000000000000e+00 0.0000000000000000e+00 - Overall NLP error.......: 2.2694405349430929e-07 2.5301808715939735e-06 + Overall NLP error.......: 2.2694405212011240e-07 2.5301808562731130e-06 Number of objective function evaluations = 64 @@ -787,11 +760,12 @@ vcmodel_constr Number of equality constraint Jacobian evaluations = 0 Number of inequality constraint Jacobian evaluations = 0 Number of Lagrangian Hessian evaluations = 63 - Total CPU secs in IPOPT (w/o function evaluations) = 0.016 - Total CPU secs in NLP function evaluations = 0.417 + Total CPU secs in IPOPT (w/o function evaluations) = 0.024 + Total CPU secs in NLP function evaluations = 0.625 EXIT: Solved To Acceptable Level. - + 0.729483 seconds (102.83 k allocations: 8.150 MiB, 0.88% gc time) + ```julia From e5685a1835ad1981a2bf866f880408994cbadbf7 Mon Sep 17 00:00:00 2001 From: Juhyun Kim Date: Wed, 30 Jan 2019 14:54:51 -0800 Subject: [PATCH 51/56] initial commit for ipynb with v1.1 --- docs/heritability1.1.ipynb | 1870 ++++++++++++++++++++++++++++++++++++ 1 file changed, 1870 insertions(+) create mode 100644 docs/heritability1.1.ipynb diff --git a/docs/heritability1.1.ipynb b/docs/heritability1.1.ipynb new file mode 100644 index 0000000..d769838 --- /dev/null +++ b/docs/heritability1.1.ipynb @@ -0,0 +1,1870 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Heritability Analysis\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As an application of the variance component model, this note demonstrates the workflow for heritability analysis in genetics, using a sample data set `cg10k` with **6,670** individuals and **630,860** SNPs. Person IDs and phenotype names are masked for privacy. `cg10k.bed`, `cg10k.bim`, and `cg10k.fam` is a set of Plink files in binary format. `cg10k_traits.txt` contains 13 phenotypes of the 6,670 individuals." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "cg10k.bed\n", + "cg10k.bim\n", + "cg10k.fam\n", + "cg10k_traits.txt\n" + ] + } + ], + "source": [ + ";ls cg10k.bed cg10k.bim cg10k.fam cg10k_traits.txt" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Machine information:" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Julia Version 1.1.0\n", + "Commit 80516ca202 (2019-01-21 21:24 UTC)\n", + "Platform Info:\n", + " OS: macOS (x86_64-apple-darwin14.5.0)\n", + " CPU: Intel(R) Core(TM) i5-6267U CPU @ 2.90GHz\n", + " WORD_SIZE: 64\n", + " LIBM: libopenlibm\n", + " LLVM: libLLVM-6.0.1 (ORCJIT, skylake)\n" + ] + } + ], + "source": [ + "versioninfo()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Read in binary SNP data" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We will use the [`SnpArrays.jl`](https://github.com/OpenMendel/SnpArrays.jl) package to read in binary SNP data and compute the empirical kinship matrix. The package has not yet been registered and must be installed using the repository location. Start julia and use the `]` key to switch to the package manager REPL\n", + "```julia\n", + "(v0.7) pkg> add https://github.com/OpenMendel/SnpArrays.jl.git#juliav0.7\n", + "```\n", + "Use the backspace key to return to the Julia REPL." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "using SnpArrays" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " 0.030779 seconds (54 allocations: 19.467 MiB, 28.08% gc time)\n" + ] + }, + { + "data": { + "text/plain": [ + "6670×630860 SnpArray:\n", + " 0x02 0x02 0x03 0x03 0x02 0x03 … 0x02 0x03 0x03 0x03 0x03 0x03\n", + " 0x03 0x03 0x02 0x03 0x02 0x03 0x03 0x02 0x02 0x03 0x02 0x01\n", + " 0x03 0x03 0x02 0x03 0x02 0x03 0x03 0x03 0x03 0x03 0x03 0x03\n", + " 0x03 0x03 0x03 0x03 0x03 0x03 0x03 0x02 0x02 0x02 0x02 0x03\n", + " 0x03 0x03 0x03 0x03 0x03 0x03 0x03 0x03 0x03 0x03 0x03 0x02\n", + " 0x02 0x02 0x03 0x03 0x02 0x03 … 0x03 0x03 0x03 0x03 0x03 0x03\n", + " 0x00 0x00 0x03 0x03 0x00 0x03 0x03 0x02 0x03 0x03 0x03 0x03\n", + " 0x03 0x03 0x03 0x03 0x03 0x02 0x03 0x03 0x03 0x03 0x03 0x02\n", + " 0x03 0x03 0x02 0x03 0x02 0x03 0x02 0x02 0x03 0x03 0x03 0x03\n", + " 0x03 0x03 0x03 0x00 0x03 0x03 0x02 0x02 0x03 0x03 0x02 0x03\n", + " 0x03 0x03 0x02 0x03 0x02 0x02 … 0x02 0x03 0x03 0x03 0x03 0x03\n", + " 0x02 0x02 0x03 0x03 0x02 0x03 0x02 0x03 0x03 0x03 0x03 0x02\n", + " 0x03 0x03 0x03 0x00 0x03 0x03 0x03 0x03 0x03 0x03 0x03 0x02\n", + " ⋮ ⋮ ⋱ ⋮ \n", + " 0x02 0x02 0x03 0x03 0x02 0x03 0x03 0x02 0x03 0x03 0x02 0x02\n", + " 0x02 0x02 0x02 0x03 0x00 0x02 0x02 0x03 0x03 0x03 0x02 0x03\n", + " 0x03 0x03 0x02 0x02 0x02 0x03 … 0x03 0x02 0x03 0x03 0x02 0x03\n", + " 0x02 0x02 0x03 0x03 0x02 0x03 0x03 0x02 0x03 0x03 0x03 0x02\n", + " 0x03 0x03 0x03 0x02 0x03 0x02 0x03 0x03 0x03 0x03 0x02 0x03\n", + " 0x03 0x03 0x01 0x02 0x02 0x01 0x03 0x00 0x00 0x02 0x00 0x02\n", + " 0x03 0x03 0x03 0x02 0x03 0x03 0x03 0x03 0x03 0x03 0x03 0x02\n", + " 0x03 0x03 0x02 0x02 0x02 0x03 … 0x00 0x03 0x03 0x03 0x03 0x03\n", + " 0x03 0x03 0x03 0x02 0x02 0x03 0x02 0x02 0x02 0x02 0x02 0x03\n", + " 0x03 0x03 0x03 0x02 0x03 0x02 0x02 0x03 0x02 0x03 0x03 0x02\n", + " 0x02 0x02 0x03 0x03 0x02 0x03 0x03 0x03 0x01 0x03 0x03 0x03\n", + " 0x03 0x03 0x03 0x03 0x03 0x02 0x03 0x00 0x03 0x03 0x03 0x03" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# read in genotype data from Plink binary file (~50 secs on my laptop)\n", + "@time cg10k = SnpArray(\"cg10k.bed\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary statistics of SNP data" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(6670, 630860)" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "people, snps = size(cg10k)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The positions of the missing data are evaluated by" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "6670×630860 SparseArrays.SparseMatrixCSC{Bool,Int32} with 5524131 stored entries:\n", + " [5688 , 1] = true\n", + " [6445 , 1] = true\n", + " [109 , 3] = true\n", + " [189 , 3] = true\n", + " [216 , 3] = true\n", + " [254 , 3] = true\n", + " [331 , 3] = true\n", + " [522 , 3] = true\n", + " [525 , 3] = true\n", + " [597 , 3] = true\n", + " [619 , 3] = true\n", + " [672 , 3] = true\n", + " ⋮\n", + " [4929 , 630860] = true\n", + " [5594 , 630860] = true\n", + " [5650 , 630860] = true\n", + " [5780 , 630860] = true\n", + " [5854 , 630860] = true\n", + " [5867 , 630860] = true\n", + " [6084 , 630860] = true\n", + " [6175 , 630860] = true\n", + " [6178 , 630860] = true\n", + " [6239 , 630860] = true\n", + " [6478 , 630860] = true\n", + " [6511 , 630860] = true" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "mp = missingpos(cg10k)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The number of missing data values in each column can be evaluated as" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "1×630860 Array{Int64,2}:\n", + " 2 0 132 77 0 27 2 2 6 27 2 … 6 4 5 11 0 0 4 29 0 5 43" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "missings_by_snp = sum(mp, dims=1)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Minor allele frequencies (MAF) for each SNP." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "630860-element Array{Float64,1}:\n", + " 0.1699160167966407 \n", + " 0.17098950524737633 \n", + " 0.11402569593147749 \n", + " 0.2686940694676172 \n", + " 0.21926536731634183 \n", + " 0.23934969140448592 \n", + " 0.19061187762447507 \n", + " 0.20200959808038388 \n", + " 0.027160864345738278\n", + " 0.2997139846454915 \n", + " 0.24625074985003004 \n", + " 0.05555555555555558 \n", + " 0.3659067046647667 \n", + " ⋮ \n", + " 0.22547254725472543 \n", + " 0.4035864345738295 \n", + " 0.20799579957995795 \n", + " 0.44801200300075017 \n", + " 0.2954647845021775 \n", + " 0.14265367316341826 \n", + " 0.1709145427286357 \n", + " 0.2814281428142814 \n", + " 0.06113537117903933 \n", + " 0.052473763118440764\n", + " 0.13930982745686427 \n", + " 0.1324128564961521 " + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "maf_cg10k = maf(cg10k)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "([0.00841726 0.124063 … 0.364253 0.5], 0.24536516625042462)" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# 5 number summary and average MAF (minor allele frequencies)\n", + "using Statistics\n", + "Statistics.quantile(maf_cg10k, [0.0 .25 .5 .75 1.0]), mean(maf_cg10k)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlgAAAH0CAIAAABuMsSDAAAABmJLR0QA/wD/AP+gvaeTAAAgAElEQVR4nO3deXxTdb7/8W/SJd2g0MpaoQwglEVEATdQRNSrYCmI1xkEkRFnUFyBhyOOiAsMUJhRAcfLD6UjOoNAK6UoDIvQCoIbiqAIVBSklJaldG+znZzfH7nmdpKT0CU9Sfp9PR/8Eb6fnHM++eacvJuTzaCqqgAAQFbGQDcAAEAgEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpteQgPHXqVGlpqX/Xabfb+VI6IYTD4VAUJdBdBAWbzRboFoKCoigOhyPQXQSeqqp2uz3QXQSFEHq0bMlB+Pzzz3/00Uf+XWdZWRl7uRDCbDZXVVUFuougcOHChUC3EBQqKystFkuguwg8m81WXl4e6C6CwsWLF0Plb6OWHIQAAFwSQQgAkBpBCACQGkEIAJAaQQgAkBpBCACQGkEIAJAaQQgAkBpBCACQGkEIAJBaeKAbQOCZzWZvX44VHh4eGxurcz8AoCeCUBZnzpzJz8/3HDebzQ89NqPk7BnPkupQBlx19f7P9jR/dwAQMAShLIbflXbOFmEMi3AbV6y1lYUF4g2tn+n48VNL3lw9mgOAwCEIZVFccLLqhYOiVTv3wtl8Me+6QHQEAEGBN8sAAKRGEKIxvv322w5JXYxeLF26NNANAkB9cWoUjfH1118br/2tOuENz5Ip65lQ+VlqABAEIZrEwBkFACGPBzIAgNR4RggAQWThwoXr168PdBd+oChKWFhYIxbs3bv32rVr/d6PDyEfhAcOHBg2bFh1dXWgG2mJqkryj/8U1/Yyz4rdYlYGjde/I6DFKygouOuuu/77v/870I0ERn5+/oIFC3TeaGgHYXFx8csvv1xTUxPoRlqosjP2mETrrO0apX8+3pi/9ADUw+WXX3711VcHuovAMBoD8IJdCL9GaDabp02btmzZskA30pIZjGEi7jKNf+GmQLcGAP4RwkH4xBNPzJgxo2vXroFuBAAQwkL41GhmZubbb7/tvGwwGGpra6OiogLbEoQQQrFt+mjL6aJiz4pBiD8+PPWKK67QvykA8CaEg7CsrMx5wWAw8Anu4GH/+avcVh1zj7fyLEV9/s7tI28lCAEEFf+fGlUUJSUlRbM0dOhQw68eeeSRhq6ntLQ0NTU1ISFhzJgxpaVav5aAINFnhLjrT57/TO27BbozAHDn52eES5cuXbNmzbFjxzxLqqoePXr09OnT8fHxQojwcF+b1lxPenp6cnJyVlbWrFmzFi9evHDhQteaNVdis9l27NjheuLocu2111555ZX1v1F11dbWRkRERES4/5hRkMjds+/NlW9plsxms5831ogn4apqsVha2Lt8a2trW9gtapza2lpFUTg3Y7Vam7hL2O12P/YTilRVbcQEmkymxn1sUfg9CAcMGNCjR4/U1FTP0tmzZ61Wa1pa2rFjx0aOHLly5UrnS3p2u90Viq7LmuvJzs7OyckxmUyPP/54WlqaKwi9URTl5MmTnlPTpUsXb89ZL8lms9lstsYtq4PZL7z0ffvholNvjZryoe7taLDb7cE8gY0Q5LuEbmw2m9FoZCpsv2r0GhwOh9vIa0uX/TMrp2l91ZdRiBeeeWrMmDH6bE6TqqqNmMCIiIhgCcIRI0Z4KxUXFw8ZMuTVV1/t2rXrjBkznnrqqffff18IMW3atFGjRo0fPz49Pd1iscydO9fbegoLC5OTk4UQycnJRUVFl2wmKirqD3/4wwMPPND42+PBZrPFx8cH7TNCY1iY6Hur6HWzRu3dS5yLbjBDIxYxxMbGOk8JtBg1NTUt7BY1jqqqJpMpOjo60I0EmNVqNRqNTdklIiMj3UYOHT76jamvuOruprVWv63nvXH69GkdNuRDEyewEfR7s8zAgQN37drlvLxo0aJ+/fo5Ly9ZsuTOO+/MyckpKSnZsGGDjzWoqmowGJwXFEVp7oYBIFh07CX63KrDdsIOZmuOGwyGCRMmrFmzxjUyceLENWvWuE6G2+32pKSkhISEH374wflA7VzK7VlacJ741e9zhN98882+ffuclyMjI02m//1EdkJCwrhx49atWzdlyhTXoKbOnTsXFBQIIQoLC5OSkpq7YQCAS15ensVicV62Wq15eXl1q5988slll1124cKFw4cP1x23/yfdum0QPZ4R5uXl3XLLLdXV1ffee29eXl6PHj3mzZs3duxYZ3XFihW5ubn5+fljx46NjY0dNWqUt/WkpqZmZGQsWLAgIyMjLS1Nh87hXw6bedU/3tmZm+dZMhrE7Gef5RwjELSGDRu2Y8eOu+++Wwixa9euoUOHZmZmuqrr169/8MEHT5w4kZmZ2b9//8C12Rh6BOGIESNUVR02bNiLL76YmppaXl4+atSo5cuXO6tdunTZuHFjTEzMli1bNN9u6jJ37tyJEyd26dLlmmuuee+993ToHP5VU3xqfZsewqLxMlLk9r9Nf/RRghAIWvfcc88HH3zgDMLs7Oxx48a5gtBms23cuPGbb7758ccfH3300Zdeesl1djQkNEsQur2F2vlfg8Ewffr06dOnu1159OjRzgudOnXq1KmTj/W0adNm8+bN/m8Xehpyn+h3u+dwxKcr9e8FQP2NHj161qxZNpstLCxs69atixcvdpV27do1YMCApKSkjh07lpeXHz582PWksG4ivvXWWw8//LDefddDCH+zDFoSi0MMGHKD5hfPt2kV99OxH/RvCUBdrVq1GjRoUF5eXkxMTP/+/euev1m/fv3HH3/syry6Z0dD4qOlBCGCglJdXjbz3yK+k3vBUl254DrXiXQ3HTp0uO+++5q9OQBCiF/PjsbFxY0bN841aLVas7Ozf/nlF+dPIGzbtu3pp58OrbOjBCGCRnxnkXC5+2B5sU1x/OnDo55Xd1Sc+435F4IQ0M2YMWPmzJkTExPzpz/9yTW4Y8eOrl27un4IaPjw4adOnap7djT4EYQIcqpQhfm+1zUqJ75SNz+tez+AvBISEvr27WuxWNq3b+8aXL9+fd1vAYuKiho5cmRovXeUIAQAXILrpb7t27e7Da5evdrtyps2bXJbKsgRhAhZ5srTxee79blKszhhfNrC+a/o3BHQHBISE0y73o345XMdtmX/+SshBuiwoaBCECJkXSwwOwy/3JehUTr44Y8nTurdD9A8Frz0wtX9Gvk7AQ1317XXXqvXtoIFQYhQFmESXbT+ei34VlhO6t0M0DxMJtOkSZMC3UVLpt93jQIAEIQIQgCA1AhCAIDUeI0QLdMvPx9ftmyZZmno0KGDBg3SuR8AQYsgREv085cHTp777qN8z4rj+GfzamsJQgAuBCFaIsVmSL7act9rnpXwDX/Wvx0AwYwgBIAgEh0dnZmZefSoxvfryuDChQv6b5QgBIAgMn/+/LfeeivQXfhBZWVlbGys5m+r+dCzZ8+631yqD4IQAIJIdHT0k08+Gegu/ODcuXOJiYlhYWGBbuTSCEJIx2az1dTUaJZMJlNIHLcA/IggDD2Kohw/flyzZLFYdG4m5Kjnfp770qaX5y/wLDls1uzsDWPGjNG/KwABRBCGntf+vuKFl+ZFxMZ7lqpKz4vb9O8olKhlZ9SxL9vvfMazFL/yHv37ARBwBGHoOXToO/PoF8zD/+hZCnu2m6J/QwAQyviKNQCA1AhCAIDUODUK/C/FUpO+9O/vrc/2LEUYxet/TW/fvr3+XQFobgQh8L/MxSf2tRstIq7yLEVtfP6Vuc8ThECLRBACdfS5VVx1t+dw+PZF+vcCQB+8RggAkBpBCACQGkEIAJAarxECl6Yq9ry8vPx8jV/6LSsru//++/VvCYC/EITApdWazU+/tjo8Ks6jolYe3EkQAiGNIAQuTbWZqye+Jdr3cC8oNuNjGl/6CiCEEIRAU50+fVpzPDIyko8eAsGPIASawKE4TLG9r7nes6LabXGxsWPHaHwqUQhx45BBU6Y82MzNAagXghBoAtUhaitrlp7XKO3PMv/ribcu/kajdOrA6eJ/E4RAkCAIgeZiiI5XRz6uUfhqvbjwoe7tANDG5wgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFLjc4RAACh2e3l5uWYpKirKZDLp3A8gM4IQ0N2Fkzs/3tE+qatnRbFa5s7589y5cz1LFRUV27dv97bKO+64o3Xr1v5sEpAGQQjorrzI0Hu4dfoHGqXsuVu2f1xSVuFZ2Z736clKR+RlXTxL5u8+/uaLff369fN7p4AMCEIgmPy494vI6C8K2nlWjCcLHPf9zTx4vGep9fyBzd8Z0GIRhECQ6XG9+K+ZnsPGT1c59G8GkADvGgUASI0gBABIjVOjQMiz2mwTpj4aHR3tWYoKE3k7thoMBv27AkIFQQiEPEtFyXe9ZohEjc9jiKWpS5Ys0QzCxMTEhx56qNmbA4IeQQi0CD1vFJ37aowbjHNyz3oOq9UX2xd+RhACgiAEWjiHYrtngTB4vBug+Jj4x72BaAgIOgRhkCorK3v4iZnm2lrP0sGDh8T1V+rfEgC0SARhkHr33Xc3HSy0Xf+AZ8lY8aX+/QBAS0UQBi9jxyvEtb/1HDdsW6J/MwDQUhGEgKSs5tr33ntPs9S1a9fhw4fr3A8QKAQhIKXS0yVlFdPf2uZZsZcUDP1NAkEIeRCEgJTsNmGKrZr8D43SgRzHyX/p3hAQMHzFGgBAagQhAEBqBCEAP1jxj/diYmIMXnz99deBbhDwitcIAfjB7j2fiqnviOvv9yzFp1+nfz9A/fGMEAAgNZ4RAmgAVVVVVdWseF1Ese/evbuwsFCzevfddxuN/EWOQCIIAdTXnj17brllhEN1aNRMsWLiaM2lqivKnl/5QWSrtp6lih8+7ZPSx6AVhL+5vNOHGzKb1i9QLwQhgP9UW3Hg+x/6DRnqWblQdFq9c5YYN9+zFP7iALuX9ak2c+29f6tNvkajNi3mh1teEuGR7uPlZ4s3PduwtoHGIggB/Keio5Xh8T+MXKBRWv+MQfu8aBP0uF5ERLsPlvzi560A3hGEANwZYtqInjdqFKLjde8FaHYEIYDgY60pr7UmdOqiWbxj5K1r/7la547QghGEAIJPdalis5bO/kqj9P2/d+554+b/StVc7sH7xk2d+lDz9oYWhyAEEJQMRpFwucZ4+dmLNuOeXlM1St/mXPX94ebuCy0PQQggxBjiEsRVWh/VOJsvxFnd20HI43OsAACpEYQAAKkRhAAAqfEaIQD4044dO6qqqjRL/fr1u+GGG3TuB5dEEAKANrvdbrFYNEtGozE62uMLcYT47LPPpj49W/S5VWNthUcmjxxEEAYhghCAFGw2m+b4x3l7Vry9SgiDZ2nv51+Wnyv0LKiqGh8ff+FskeciFoslvF23ion/T2NLu948eWLb2rVrNdsYPHhwz549fd2AhqioqPD2rDQyMvKyyy7z14ZaBoIQQMu3YtXq6X+cajBopJ3DFCdGPi7aa4XQmSzxtwIR08Z9vLzItqjhvzZ88uvcn499sTzbs2I78fVfn5/hryCsqam58tphFy5c8Cw57LY+vXt988Vev2yoxSAIAbR8ebt3qw+8qQ77vWfJ+Eyyo9/toofWGcvV0/zZhLVGpNxaOelNz0rUuif9uB2z2Xz2zGnLq1ofqTy+z7Lzz37cVstAEAJoKSrO/SN37foPNmpUqqrEmIY/h5OM1Wr18qvLIjw8PCwsTOd+dEMQAmgpio/W9r6terTGM56wN9L0bye0fPXVVzffervNUutZUhVlyZLFM2fO1L8rfRCEAFqQ6HjRrrvnsCHMz491DsWel5fnOX7o0CGHQ/HvtvTx7bffGoeMV+5f4VmKzHpG/370RBACQAPVlFVbbOOeeNGzYr5w2hadoE8XP/744xdffOE5XlVVFaJhHCgEIQA0kN0qhCh76mON0vbXwvZn6tPFiLvSyi7rY4iIchtXzVU2m12fHs6fP5+Tk6NZqqysfOihh+LjQ+DHnAlCAAgw1Wb+1/oPvv7uB89SmEH8aebTV1xxhWepsuxi9aOvi/iO7oXzP4tD13jblmK3FxVpfAKyvLzcyxtlfJn4h8f2nCg3JiR5lmxfrE9LSyMIAQCXZj/57Wften5W2duzFJ23/P777tUMwsYoO3P8+PGe/TVi0lpdLgaNb+j6KioqzCOeEP3/y7MU84PWM+agRBACQBDodZMY8ajnsOm7Df7cStVFtW1SzdyvNUoZv/eWB46Lp2fPefv5F1/xLFlFuBjgzwYDgiAEgOCl2m27du0qLi72LNns2l8a5/8eSk7Z73zWdssjnqWweYP16aFZEYQAELyqS4qXfLA7MvdHz1JtrVm/PiKiNL5qTgjN72gNOQQhAAQv1W6xjJpjSblFo7ZfM5nQYPwwLwBAajwjDKSKioppTz9bUHDKs1RUcFJJuln/lgBANgRhIH3xxRcffvJZ9eiXNWon08Ma8aEeAEADEYQBFt7qMjFglEZh72rdewEAGfEaIQBAagQhAEBqBCEAQGoEIQBAagQhAEBqBCEAQGoEIQBAanyO0G9KSko0vyFeCBEREdGrVy+d+wEA1AdB6DdDb7/7zIUyg9H9Sbaq2CNtVReKCwPSFQDAN4LQb07mH7YsPCGiW7sXys60WXJjIDoCAFwarxECAKRGEAIApEYQAgCkRhACAKTGm2X0oNhtH3zwgef4wYMHFcWufz8AABeCsPlVlVTVWh5a8i/PiuVMvt0Up39HAAAXgrD5KTZhCKt4eJ1G6d9Lwg5t0r0hAMD/4TVCAIDUCEIAgNQIQgCA1AhCAIDUCEIAgNQIQgCA1EI+CA8cOBAbGxvoLgAAoSq0g7C4uPjll1+uqakJdCMAgFAVwkFoNpunTZu2bNmyQDcCAAhhIRyETzzxxIwZM7p27RroRgAAISyEgzAzM3PEiBEGg0EIYTAYzGZzoDsCAISeEP6u0bKyMucFg8GgqmpgmwEAhCj/PyNUFCUlJUWzVFpampqampCQMGbMmNLS0oaup0GLAwBQH35+Rrh06dI1a9YcO3ZMs5qenp6cnJyVlTVr1qzFixcvXLiwQevxtri3p4O1tbVvvPHGRx995DY+fvz422+/vQG3qo6ysjKHwxEREaFRa9yz0tB9KtuYzvW7td7uDR8dNENz/l2ljrOnyyJ6rtLn0Rncu3IQ8HnKzXtJVcvLy3V70hIXF6f9yFwPfg7CAQMG9OjRIzU1VbOanZ2dk5NjMpkef/zxtLQ0Z5LZ7fbw8P9tw3VZcz2ai/sQERExYMCAG264wW38iiuuiI6ObtwNjI6Ojo6O9jLdhsassVEL4ZIMXibWoOtDGPduSGjM3aTvjhRgBm+H06VERUU1+sG2oYzGxp/g9HMQjhgxwke1sLAwOTlZCJGcnFxUVOQcnDZt2qhRo8aPH5+enm6xWObOnettPZqL+xAeHn7zzTc/8MADjbgh3phMpqioKO0glO1Br5GPHsErqJsTQs8GG7GlZmjOn6ts7CM5Lsn7zBoMzgdMHZtpJF3fLKOqqvMvC1VVFUVxDi5ZsuTOO+/MyckpKSnZsGFDQxcHAKApdP34ROfOnQsKCoQQhYWFSUlJzsGEhIRx48atW7duypQpJpOpoYsDANAUegRhXl6e80JqampGRoaqqhkZGWlpac7BFStW5Obm5ufnL1iwYMuWLT7Wo7k4AABNoUcQul7wmzt37qFDh7p06XL48OE5c+Y4B7t06bJx48bk5OQtW7bExMT4WI/m4gAANEWzvEbo9l5b13/btGmzefNmtyuPHj3aeaFTp06dOnXysR7NxQEAaIoQ/oo1AACajiAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASC2Eg9DhcEyZMmXw4MEpKSmZmZmBbgcAEJJCOAh37Nhhs9n279+fmZk5ffr0QLcDAAhJIRyEV1111aJFi8xmc0FBQadOnQLdDgAgJIUHuoHG69ixoxDi7rvv3rx58/bt2wPdDgAgJIXwM0JFURwOR05OTlZW1iOPPBLodgAAISmEg/DNN9989tlnw8LCevToUVNTE+h2AAAhyc9BOHToUMOvPJ+llZaWpqamJiQkjBkzprS01PeqFEVJSUnxsfjkyZN/+umnAQMGTJgwYeXKlf69IQAASfjzNUJVVY8ePXr69On4+HghRHi4+8rT09OTk5OzsrJmzZq1ePHihQsXelvV0qVL16xZc+zYMd+Lb9iwwUc/VVVVM2bMeP75593Gn3766UmTJjXstv3q4sWLFoslIiJCo6aq3pfzXvKxUMP5aqFxK2xkzZ/LNI63qfB1J/m/O69rVBuzMR1nr+Hbaobm/LlKn/Md+F1Zv7vWJ29t+Nxdfe3kJSUlcXFxTeyqnuLj400mU+OW9WcQnj171mq1pqWlHTt2bOTIkStXroyKiqp7hezs7JycHJPJ9Pjjj6elpTmD0G63uyLTdXnAgAE9evRITU295OI+xMbGzp49e+zYsW7jCQkJzqhuBFVV4+PjtYPQYPC+nPeSj4UazlcLjVthI2v+XKZxvE2FwceB6//uvK7R0Ji7SsfZ03Fb3vmzB5/z3chd2Y/pFQzTLby34XN39bWTt23bNjExsYld1ZPR2PgTnP4MwuLi4iFDhrz66qtdu3adMWPGU0899f7779e9QmFhYXJyshAiOTm5qKjIOTht2rRRo0aNHz8+PT3dYrHMnTtXCDFixAjP9Wsu7oPBYOjQoUPPnj2bftNcwn7lx3UiSATJg1GIYvbgyWg0hsSjpT+DcODAgbt27XJeXrRoUb9+/dyuoKqq8y8LVVUVRXEOLlmy5M4778zJySkpKfF9qlNzcQAAmsKfb5b55ptv9u3b57wcGRnpebq2c+fOBQUFQojCwsKkpCTnYEJCwrhx49atWzdlyhTfZ3g1FwcAoCn8GYTV1dXjxo07cuSI1WqdN2+e68W5vLw854XU1NSMjAxVVTMyMtLS0pyDK1asyM3Nzc/PX7BgwZYtW3ysX3NxAACawp9BOGzYsBdffDE1NTUpKam0tDQ9Pd057nrBb+7cuYcOHerSpcvhw4fnzJnjHOzSpcvGjRuTk5O3bNkSExPjY/2aiwMA0BT+fI3QYDBMnz7d8/uvXW+9bdOmzebNm92qo0ePdl7o1KmT21eGur1nV3NxAACaIoS/WQYAgKYjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSCw90AyFm45btGe/+02gweJZsNpv+/QAAmoggbJj0v712POU+0aGXRi3vY93bAQA0FUHYcD1uEN2vC3QTAAD/4DVCAIDUQjsIS0tLU1NTExISxowZU1paGuh2AAChJ7SDMD09PTk5uaioqGvXrosXLw50OwCA0BParxFmZ2fn5OSYTKbHH388LS1t4cKFdauqqpaUlJw6dcptqfj4+Li4uEZuUlVF5QVRdka7Wl4sLBWAbDcAABFESURBVFXug5XnhXBoL2KuFHabdslWKyzV2iWHImrKNEuqKkRVidf2Ks4KxeOtrRXnhapqL1Jbripe2rPWCDVSu6TYRW25dkl1iOqLXtpTRcVZYfT4y6zinBBe2qspE4rdW3uqrVZ7ihSrqK30Mns+2hOi4qyIjPbYkFkIob1I9UWhKtolS7VqM2uX7BZhqfJ6v/tq75yIaaNdKisSnu9zri7x3l6lard4b097t1Qdiqgp9dpe5XmvpbIiERHlPlh1Xqg+jhqrl6PG6+yp3o8aoTp8HTXlxcJa4z5Y6eOoqRDejhpbrbCavLTn46hRvbanClFxVqgO9/GKs16Pmtpyr+1Za1WjwctRY/N21Pg4qK12x4BB1xoMGk+3WsXGFJz8SWNtTWA0Gg1a7+evD4Oqqv7tRk9xcXHnz5+Pjo6ura3t0KFDRUVF3eq4ceNyc3Ojo90fvGbOnPnggw82bovj7pvw/eEfhNZsV1VUxLVq7VlSHY6aqqrY1q09F1FsVqvVFh0b61mymWsdwmiKMnmWLLW1xrDwiMgIz1JtTW1kZGRYeJhnqbqyMiYuznNHUVVHTWVlbOt4jfbsNqvF6q09RTVERXs8fglhqa0xhkVotmeuqY3w1l5VVUxMrMHo2Z5aXVEeF6/xEK/YbRazJUbrbxqbxexwCJNWe9baGhEWHhkZqdFebW14RGS4dnvVMTExnu0JIarKSuPatPUcdyh2c61Zsz271WJXHFEee6YQwlpTLcIiIk1e2guPCI/Q+OO1promKirKGKbxiFNVVhoX39Zzt3QoirmmJqZVK+327EpUTIxGe7U1whjurb2w8PCICI373Vd75eVxreM12nMotdXVsa00jhq7zWq32bXbM9eoIkzzqPHRXm1NTaQpKkyrveqKilivB7WXo8Zqtdp0PKhj47SOGp8HtdkaHafVnsXsUIUpSvOoqTV4ac9cY46IjPDSXlVMrMZBLYSIi4468NUXnuNNER8fbzJpzG19hPYzQlVVnY/sqqoqiuJWjYuLW758+QMPPODHLeZkrYuPj9c8nKRSU1Njs9ni4zWONNkUFRV16tQp0F0EXllZmclk8vy7UzZWq7WysjIxMTHQjQTeuXPnEhMTw8I0MjLYhPZrhJ07dy4oKBBCFBYWJiUlBbodAEDoCe0gTE1NzcjIUFU1IyMjLS0t0O0AAEJPaAfh3LlzDx061KVLl8OHD8+ZMyfQ7QAAQk9ov0bYpk2bzZs3B7oLAEAIC+1nhPr7y1/+cvz48UB3EXi5ubnvvPNOoLsIPEVRHn300UB3ERRWrly5d+/eQHcReIcPH16yZEmguwgKzz777IULFwLdRb0QhA2zZ8+ekpKSQHcReD///PO3334b6C4Cz+FwcE7Caf/+/Z6f2ZXQ+fPn+YPAaceOHdXV1YHuol4IQgCA1AhCAIDUCEIAgNQIQgCA1EL74xO+Wa3WTZs2Ob96xl+Ki4vffffd3bt3+3Gdoeizzz47c+bMggULAt1IgCmK4nA4mAchxNGjRxVFKS4uDnQjAfbjjz8WFBSwSwghKisr33zzzYSEBH02d++99/bq1atxy4b2l277dv78+VdffbXR30euyeFwGD1/IUE+zt3Gv3MbotglnNglXNglnHSeh0mTJvXt27dxy7bkIAQA4JL4swUAIDWCEAAgNYIQACA1ghAAIDWC0JfS0tLU1NSEhIQxY8aUlpY2qNrCXPLGKoqSkpKif2N68j0JOTk5/fv3b9Omzc0335yfnx+QDnXjeyq2bt3at2/fNm3a9O3bd/v27QHpUB/1eRD4/vvvY2NjdW5Mf76nYujQoYZfPfLIIwHp0AeC0Jf09PTk5OSioqKuXbsuXry4QdUWxveNXbp06Y033njs2LGA9KYbH5Nw6tSpSZMmvfXWW0VFRWPGjPn9738fqCb14WMqHA7HxIkTly9ffvHixVdeeaVlT8UlHwTKy8unTJlSU1Ojf2868zEVqqoePXr09OnTlZWVlZWVr7/+eqCa9EqFd7169Tpy5IiqqkeOHOnVq1eDqi2M7xu7a9euDz/8sMXvTj4mITc39+GHH3ZePnfuXGJiYgD605GPqbBYLJs3b3Y4HBUVFZs2berbt2+AetSD7+PC4XCMHTs2MzOzxR8aqs+pKCoqiouLGzRoUFxcXFpa2tmzZwPUo1ct/+5pitjY2JqaGlVVa2pqWrVq1aBqC1OfG9vij/b6TILdbn/kkUemT5+ub2t6u+RUVFZWCiEMBsPevXt1704/vudh4cKFs2bNUiU4NFSfU3HgwIERI0YcOHCgpKRk8uTJv/vd7wLUo1ecGvVFVVXnN2WoqqooSoOqLYxUN9abS07Cxx9/PGTIkPj4+KVLl+rena4uORVxcXFVVVXz589/6qmndO9OPz7mITc3d+vWrQsXLgxQa3rzMRUDBw7ctWvXwIEDExISFi1atG3btgD16BVB6Evnzp2dX1VaWFiYlJTUoGoLI9WN9cbHJKiq+txzz73yyitr165dtGhReHhL/hZf4XMqTp48+cwzzwghYmNjp06deuTIkcC0qAsf87Bz585PPvkkMjLSGQ8Gg+HTTz8NTJe68DEV33zzzb59+5yXIyMjTSZTAPrziSD0JTU1NSMjQ1XVjIyMtLQ052BeXp6Pakvleyok4WMS9u3bl52dvWnTps6dO1dVVVVVVQWy0ebnYyo6d+68atWqTz75RFXVdevWXX311YFstJn5mIf58+e7zrwJIVRVHTZsWABbbW4+pqK6unrcuHFHjhyxWq3z5s0bO3ZsIBvVpMPp19BVWlo6atSopKSk1NTUsrIy56Br0jSrLZXvqdD8b8vjYxLmz58v1ZHle3/Iy8u75ppr2rZte8MNNzjfQNFS1ee40BxpeXxMhcPh+Pvf/96jR4/LLrts8uTJ5eXlAe1UA1+6DQCQGqdGAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgRpAwGg8lkKisrcxsvKyszmUzOHzt1Xu3o0aPN3Uz//v0NBsOqVas8m3RtvZ6dNKJhgxcNWklosVqtQ4YMOXfunKj3nuDi487yNo1nz54dPHiw1WptthuEoNbCf0cbIc1qtW7YsOGhhx6qO5idna3zA9bhw4cPHz4cGxubmZk5depUPTftsnz58ssvvzwgmw6I5cuX33bbbe3bt3f+t/57gu87y9s0dujQ4dZbb33jjTdmzpzpvxuB0BHoH0QEtAkhrr766pEjR7qN33777c4fPXf+t7KyUlGUZu3khRdeiI+P/8tf/hIWFnbhwgW3Jl0/PFv3sg/1vFoTFwlpNputS5cuJ06ccP63nnuCUz3vLE8//fRT165d7Xa7f24DQgqnRhG87r///l27dhUVFblGzp07t3PnzgkTJrhG4uLijMZm3I1VVV23bt299957//33K4qSnZ3dfNuC07Zt2y6//PJu3bq5RuqzJ4im3Vndu3fv1KnTtm3bmtw+Qg9BiOB1xx13tG3bdt26da6RrKys1q1b33HHHa4Rt1fpvvrqq9/+9rcJCQndu3evu6DD4Xj99df79esXFxc3aNCgrKysumv46quvUlNTr7vuOs8eDh48mJ+fP2HChG7dug0ePDgzM7M+nauqumLFij59+sTExFx99dWrV69WVbXRV/PNrX8f6/zyyy9HjhzZtm3b7t27/+EPf/j88881X2o9evRo3RfefKzQx4Srqrps2bK+ffu2bt36pptu+vTTT4UQc+bM+c1vfuNa/NSpU0ajMScnx+0Wbdu2bfjw4XVH6rMniMbeWS7Dhw/fvn17gxZBCxGYJ6LApQghjhw58sc//nHIkCGuwWHDhk2dOvXIkSOuXVf858nJ6667bsWKFfv27Zs8eXJkZGRVVZWztGTJkri4uMWLF2/ZsmX27NlhYWEffviha6nBgwdPnTr1X//6l2cbs2fP7tixo/OM2aJFi8LCws6fP+/WpOfld999t2/fvqtXr/73v//t3Nz//M//NOhqblOxefPmI//JbDZr9u9tnfv37w8LCxs7duyaNWsyMzMnTJiQmJioOY11p/eSt8XbhC9dujQ+Pn7ZsmVZWVnjxo0LCws7ePDg999/L4T4/PPPnddZsGBBhw4drFar2+295pprsrKy3Cb5kntCfe4sH9Ooquq6desGDRrkOf9o8QhCBCnnw19eXp4QIj8/X1XVU6dOCSE+/vhjH0H42muvOS9XVFS4Sg6HIzEx0flUxmn27Nk33XSTa6lnnnlGsweHw9G9e/cnn3zS+d/jx48LIVauXOnWpOfla6655vjx466rzZw5c9iwYQ26mttUePruu+80+/e2zrvuumvy5Ml1V+t8L4lnV27p4vu2eJvwjh07rlmzxllSFOWuu+567733VFUdMGDAjBkznNdJSUnRnPl27drt3bu37s2vz55QnzvLxzSqqrp79+727dt79oMWjyBEkHI+/CmKkpSU9PLLL6uqumTJkg4dOthsNh9BuH//frc1qKp69uxZIURJSYmrtHv37sTERNfVdu/erdnDl19+6fnQefvtt3tuwu1ybGys21IdOnRo0NU8p8LHRNXt39s627dvn5ubW3fBXbt21ScIfd8WzQk/f/68EMLtvSpOCxcuTEpKUhTFObc//PCD53XCw8MPHTrkttpL7gkNurM0HTx4MDw83McV0FLx8QkENaPROGHChDVr1rzwwgtr16697777wsN97bSej9reVqsoiuu/7dq107zaunXr2rVr98knn7heM1u9evWSJUvOnz/vbRGnmJiYPXv2REdH+26jnle7pLrNeFun1Wp1+8hdWFiY5tpqamrq36TmhNvtdm/r/93vfvfcc8/t3bs3MzPzhhtu6NOnj+d12rRpU1VV5TZ4yT2h0XeWS2VlZdu2betzTbQwvFkGwe7+++8/duzY+vXrv/76a7d3CdZTu3btEhMTt2zZ4hrZvHlz//79fS/lcDjWr1+flpbWp0+flF89+OCD9Xk7Yr9+/QoLC52L9O7de/ny5atXr2701RrE2zqvvPLKd955p+4116xZU/e/rvz77LPPmthkhw4d2rZtu3PnTud/HQ7H4MGDFyxYIITo1q3bjTfe+M9//vP999/39qHMyy+/vO4bRF187AlNubNcioqKpPqwJv5PoJ+SAtpEnRecevfu3bZt2+TkZIfDof7nuTvh5eSk238XL17cqlWrv/71r5pvltE8Y7Z3714hxObNm+sOOhyOnj173nbbbb63npOT06pVq9deey07O3vy5MlCCM3N+bia5lT4nijf69yxY4cQ4p577lm7dm1WVtbEiRPj4+Nd05icnDxy5MitW7e+8847bh/Oq+dtcfvvggULEhIS3njjja1btz744IORkZGuV+OWL19uNBpjYmIqKio0b9H06dP//Oc/e67Wx57Q0DtL03PPPffYY4/5uAJaKoIQQaruw9Yrr7wihHj22Wed/21EECqK8uqrrzo/AzBw4EDPNyV6NvDkk0+2atWq7rsKnWbNmmU0Gs+dO+d766tWrUpJSYmJiRk0aNAHH3zgbXPeruZtKupT9bbOrVu3Xn/99XFxcd26dZs0adKePXtc07hz586UlJS4uLiRI0f+/PPPbn8i1/O2uE34okWLevbsGRMTM2TIkG3btrmuVlxcbDQap0yZ4u0WZWdn133TUH32hIbeWZqGDRu2ceNGH1dAS2VQG/65JQAtwNGjR/v06aP/I8CJEye6d+++e/fum266SfMKNpstOTn5yy+/1PNE5alTp66//vpffvklIiJCt40iSPAaIQCd2O32ysrKefPmpaSkDBs2zNvVIiIipk+f/vbbb+vZ26pVqx577DFSUE7/Hz0kdvMCJ2OgAAAAAElFTkSuQmCC" + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "#using Pkg\n", + "#pkg \"add Plots\"\n", + "#pkg\"add PyPlot\"\n", + "\n", + "using Plots\n", + "gr(size=(600,500), html_output_format=:png)\n", + "histogram(maf_cg10k, xlab = \"Minor Allele Frequency (MAF)\", label = \"MAF\")" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0.0013128198764010824" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# proportion of missing genotypes\n", + "sum(missings_by_snp) / length(cg10k)" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0.07228069619249913" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# proportion of rare SNPs with maf < 0.05\n", + "count(!iszero, maf_cg10k .< 0.05) / length(maf_cg10k)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Empirical kinship matrix\n", + "\n", + "We estimate empirical kinship based on all SNPs by the genetic relation matrix (GRM). Missing genotypes are imputed on the fly by drawing according to the minor allele frequencies." + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "568.176660 seconds (2.91 M allocations: 494.981 MiB, 0.40% gc time)\n" + ] + }, + { + "data": { + "text/plain": [ + "6670×6670 Array{Float64,2}:\n", + " 0.502735 0.00328112 -6.79435e-5 … -6.09591e-5 -0.00277687 \n", + " 0.00328112 0.49807 -0.00195548 0.000884994 0.00341591 \n", + " -6.79435e-5 -0.00195548 0.492348 0.000198191 -0.000337529\n", + " 0.00087876 -0.00322605 -0.00192088 -0.00235314 -0.00124267 \n", + " -5.03227e-5 -0.00352498 0.00184588 0.00219109 -0.00163189 \n", + " 0.00203199 0.000597697 0.00251237 … 0.00088428 2.21226e-5 \n", + " 0.000560862 0.00244517 -0.00183233 0.00120214 -0.00120686 \n", + " -0.000656949 0.00322249 -0.00101472 0.00355832 -0.000240444\n", + " -0.00103874 -0.00125164 -0.000599731 0.00176492 0.00176928 \n", + " -0.00137058 0.00209596 0.000146711 -0.0014453 -0.00103066 \n", + " -0.00209312 0.000140721 -0.000442031 … -0.000214963 -0.00107391 \n", + " 0.000933587 0.00168842 0.00185731 -0.000787908 -0.00311063 \n", + " 0.000334572 -0.000886623 0.00304182 0.000752285 -0.00123334 \n", + " ⋮ ⋱ \n", + " 0.00298109 0.00121739 0.00102822 9.66935e-6 0.00306953 \n", + " -0.00209728 0.00271452 -0.00182325 -0.00108139 0.00366901 \n", + " 0.000549425 -0.00244419 -0.00301368 … -0.000631961 0.00215641 \n", + " -0.00423362 -0.00208073 -0.00107904 -0.000619315 -0.000593852\n", + " -0.00326697 -0.000769552 0.00310511 0.000520658 -0.000113441\n", + " 0.000430563 -0.0020236 0.00265425 -0.00635493 -0.00520252 \n", + " 0.00218746 0.000798767 -0.00105684 -0.000918245 -0.00061484 \n", + " -0.00230525 -0.000101149 0.000117936 … 0.000879829 -0.00233479 \n", + " -0.00201305 0.00233864 -0.00134496 0.00197044 -0.000486275\n", + " -0.000990534 -0.000924159 -9.12302e-5 0.00122311 -0.00298296 \n", + " -6.09591e-5 0.000884994 0.000198191 0.499289 0.000481492\n", + " -0.00277687 0.00341591 -0.000337529 0.000481492 0.499799 " + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "## GRM using SNPs with maf > 0.01 (default) (~10 mins on my laptop)\n", + "using Random \n", + "Random.seed!(123)\n", + "@time Φgrm = grm(cg10k; method = :GRM)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Phenotypes" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Read in the phenotype data and compute descriptive statistics." + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "

6,670 rows × 13 columns

Trait1Trait2Trait3Trait4Trait5Trait6Trait7Trait8Trait9Trait10Trait11Trait12Trait13
Float64⍰Float64⍰Float64⍰Float64⍰Float64⍰Float64⍰Float64⍰Float64⍰Float64⍰Float64⍰Float64⍰Float64⍰Float64⍰
1-1.81573-0.946151.11363-2.098670.7444170.001391720.934732-1.226771.11608-0.4436280.824466-1.02853-0.394049
2-1.24440.109660.467119-1.621311.056680.9789471.000150.3248741.162322.692273.082641.090650.0256616
31.455671.538671.094030.586655-0.327965-0.303377-0.0334355-0.464463-0.33194-0.486839-1.10649-1.42016-0.687463
4-0.7688090.5134910.244263-1.31741.193941.173441.087370.5360230.8027590.2341590.394175-0.7673660.0635386
5-0.264415-0.34824-0.02390650.004739161.256191.203891.298010.3101140.626160.8992890.5499680.5406880.179675
6-1.37617-1.471920.29118-0.803111-0.26424-0.260573-0.165372-0.2192571.04702-0.09858160.9473930.5940150.245407
70.100942-0.191616-0.5674210.378571-0.246656-0.6088110.189081-1.27078-0.4524760.7025630.3326360.002691650.317117
8-0.3198181.357740.81869-1.155660.6344840.2914620.933324-0.7410830.647478-0.9708780.2208610.852512-0.225905
9-0.2883340.5660830.254958-0.6525780.6689220.9783090.1228631.479090.06721320.07959040.1675320.2469160.539933
10-1.1576-0.781199-0.595808-1.005550.7898290.5710580.951304-0.2959630.990420.5613090.7331-1.73468-1.35278
110.7405691.408740.734690.0208323-0.337441-0.458304-0.142583-0.580392-0.684685-0.00785381-0.712244-0.313346-0.345419
12-0.6758920.2798930.267916-1.041040.9107420.8660281.074140.03817510.766355-0.340118-0.8090140.548522-0.0201829
13-0.79541-0.699990.39913-0.5104761.515521.287431.537720.1339891.020260.499019-0.369483-1.10153-0.598132
14-0.193483-0.286021-0.6914940.01315821.523371.401061.531150.3330661.043720.163207-0.422884-0.383528-0.489222
150.1512462.091852.038-1.124751.665571.625361.587510.6358520.8425780.450762-1.39479-0.5609840.28935
16-0.4646090.3612771.23277-0.8260341.434751.744520.2110972.648161.025110.1197570.0596832-0.631232-0.207879
17-0.732977-0.5262230.616579-0.554480.9474850.9368330.9725170.2902511.012850.516207-0.03006890.8787320.450255
18-0.1673260.1753270.287468-0.4026530.5511820.5222050.4368380.2995650.58311-0.704416-0.73081-1.95141-0.933505
191.411591.787220.8439760.481278-0.0887674-0.4995780.304196-1.23884-0.153476-0.8704860.0955473-0.983708-0.356345
20-1.42997-0.4901470.27273-1.61030.9907880.7116881.18858-0.3712291.24703-0.03891620.8834962.589883.3354
21-0.1472470.1232840.617549-0.1871310.2564380.177950.412612-0.2448090.09476250.723017-0.6839480.0873751-0.26221
22-0.187113-0.270777-1.015570.06028510.272420.869133-0.6575192.32389-0.9999361.446720.971158-0.358748-0.439658
23-1.82434-0.933481.29474-1.945450.3358470.3592020.513653-0.07319771.571391.533291.820772.22741.50063
24-2.29344-2.491620.40384-2.364881.410531.422441.170240.8447671.790270.648182-0.0857231-1.02790.491288
25-0.4341360.7408820.699576-1.024060.7595290.9566560.63330.7707340.8249891.842871.91046-0.5023170.13267
26-2.1921-2.494660.354855-1.931560.9419790.9789170.894860.4632391.125371.705280.7177930.6458880.783968
27-1.46602-1.249220.307978-1.550970.6189080.6625080.4759570.4847190.4015650.55988-0.376938-0.9339830.390013
28-1.83318-1.532692.55674-1.518280.789410.9087480.6499720.6683741.200580.2779631.25053.31372.22036
29-0.7845470.2765833.01105-1.119790.9208240.7502181.26154-0.4033640.400667-0.217598-0.72467-0.391945-0.650024
300.4644561.33264-1.2306-0.3579761.18251.54316-0.603393.383090.823741-0.129951-0.65798-0.499535-0.414477
⋮⋮⋮⋮⋮⋮⋮⋮⋮⋮⋮⋮⋮⋮
" + ], + "text/latex": [ + "\\begin{tabular}{r|ccccccccccccc}\n", + "\t& Trait1 & Trait2 & Trait3 & Trait4 & Trait5 & Trait6 & Trait7 & Trait8 & Trait9 & Trait10 & Trait11 & Trait12 & Trait13\\\\\n", + "\t\\hline\n", + "\t& Float64⍰ & Float64⍰ & Float64⍰ & Float64⍰ & Float64⍰ & Float64⍰ & Float64⍰ & Float64⍰ & Float64⍰ & Float64⍰ & Float64⍰ & Float64⍰ & Float64⍰\\\\\n", + "\t\\hline\n", + "\t1 & -1.81573 & -0.94615 & 1.11363 & -2.09867 & 0.744417 & 0.00139172 & 0.934732 & -1.22677 & 1.11608 & -0.443628 & 0.824466 & -1.02853 & -0.394049 \\\\\n", + "\t2 & -1.2444 & 0.10966 & 0.467119 & -1.62131 & 1.05668 & 0.978947 & 1.00015 & 0.324874 & 1.16232 & 2.69227 & 3.08264 & 1.09065 & 0.0256616 \\\\\n", + "\t3 & 1.45567 & 1.53867 & 1.09403 & 0.586655 & -0.327965 & -0.303377 & -0.0334355 & -0.464463 & -0.33194 & -0.486839 & -1.10649 & -1.42016 & -0.687463 \\\\\n", + "\t4 & -0.768809 & 0.513491 & 0.244263 & -1.3174 & 1.19394 & 1.17344 & 1.08737 & 0.536023 & 0.802759 & 0.234159 & 0.394175 & -0.767366 & 0.0635386 \\\\\n", + "\t5 & -0.264415 & -0.34824 & -0.0239065 & 0.00473916 & 1.25619 & 1.20389 & 1.29801 & 0.310114 & 0.62616 & 0.899289 & 0.549968 & 0.540688 & 0.179675 \\\\\n", + "\t6 & -1.37617 & -1.47192 & 0.29118 & -0.803111 & -0.26424 & -0.260573 & -0.165372 & -0.219257 & 1.04702 & -0.0985816 & 0.947393 & 0.594015 & 0.245407 \\\\\n", + "\t7 & 0.100942 & -0.191616 & -0.567421 & 0.378571 & -0.246656 & -0.608811 & 0.189081 & -1.27078 & -0.452476 & 0.702563 & 0.332636 & 0.00269165 & 0.317117 \\\\\n", + "\t8 & -0.319818 & 1.35774 & 0.81869 & -1.15566 & 0.634484 & 0.291462 & 0.933324 & -0.741083 & 0.647478 & -0.970878 & 0.220861 & 0.852512 & -0.225905 \\\\\n", + "\t9 & -0.288334 & 0.566083 & 0.254958 & -0.652578 & 0.668922 & 0.978309 & 0.122863 & 1.47909 & 0.0672132 & 0.0795904 & 0.167532 & 0.246916 & 0.539933 \\\\\n", + "\t10 & -1.1576 & -0.781199 & -0.595808 & -1.00555 & 0.789829 & 0.571058 & 0.951304 & -0.295963 & 0.99042 & 0.561309 & 0.7331 & -1.73468 & -1.35278 \\\\\n", + "\t11 & 0.740569 & 1.40874 & 0.73469 & 0.0208323 & -0.337441 & -0.458304 & -0.142583 & -0.580392 & -0.684685 & -0.00785381 & -0.712244 & -0.313346 & -0.345419 \\\\\n", + "\t12 & -0.675892 & 0.279893 & 0.267916 & -1.04104 & 0.910742 & 0.866028 & 1.07414 & 0.0381751 & 0.766355 & -0.340118 & -0.809014 & 0.548522 & -0.0201829 \\\\\n", + "\t13 & -0.79541 & -0.69999 & 0.39913 & -0.510476 & 1.51552 & 1.28743 & 1.53772 & 0.133989 & 1.02026 & 0.499019 & -0.369483 & -1.10153 & -0.598132 \\\\\n", + "\t14 & -0.193483 & -0.286021 & -0.691494 & 0.0131582 & 1.52337 & 1.40106 & 1.53115 & 0.333066 & 1.04372 & 0.163207 & -0.422884 & -0.383528 & -0.489222 \\\\\n", + "\t15 & 0.151246 & 2.09185 & 2.038 & -1.12475 & 1.66557 & 1.62536 & 1.58751 & 0.635852 & 0.842578 & 0.450762 & -1.39479 & -0.560984 & 0.28935 \\\\\n", + "\t16 & -0.464609 & 0.361277 & 1.23277 & -0.826034 & 1.43475 & 1.74452 & 0.211097 & 2.64816 & 1.02511 & 0.119757 & 0.0596832 & -0.631232 & -0.207879 \\\\\n", + "\t17 & -0.732977 & -0.526223 & 0.616579 & -0.55448 & 0.947485 & 0.936833 & 0.972517 & 0.290251 & 1.01285 & 0.516207 & -0.0300689 & 0.878732 & 0.450255 \\\\\n", + "\t18 & -0.167326 & 0.175327 & 0.287468 & -0.402653 & 0.551182 & 0.522205 & 0.436838 & 0.299565 & 0.58311 & -0.704416 & -0.73081 & -1.95141 & -0.933505 \\\\\n", + "\t19 & 1.41159 & 1.78722 & 0.843976 & 0.481278 & -0.0887674 & -0.499578 & 0.304196 & -1.23884 & -0.153476 & -0.870486 & 0.0955473 & -0.983708 & -0.356345 \\\\\n", + "\t20 & -1.42997 & -0.490147 & 0.27273 & -1.6103 & 0.990788 & 0.711688 & 1.18858 & -0.371229 & 1.24703 & -0.0389162 & 0.883496 & 2.58988 & 3.3354 \\\\\n", + "\t21 & -0.147247 & 0.123284 & 0.617549 & -0.187131 & 0.256438 & 0.17795 & 0.412612 & -0.244809 & 0.0947625 & 0.723017 & -0.683948 & 0.0873751 & -0.26221 \\\\\n", + "\t22 & -0.187113 & -0.270777 & -1.01557 & 0.0602851 & 0.27242 & 0.869133 & -0.657519 & 2.32389 & -0.999936 & 1.44672 & 0.971158 & -0.358748 & -0.439658 \\\\\n", + "\t23 & -1.82434 & -0.93348 & 1.29474 & -1.94545 & 0.335847 & 0.359202 & 0.513653 & -0.0731977 & 1.57139 & 1.53329 & 1.82077 & 2.2274 & 1.50063 \\\\\n", + "\t24 & -2.29344 & -2.49162 & 0.40384 & -2.36488 & 1.41053 & 1.42244 & 1.17024 & 0.844767 & 1.79027 & 0.648182 & -0.0857231 & -1.0279 & 0.491288 \\\\\n", + "\t25 & -0.434136 & 0.740882 & 0.699576 & -1.02406 & 0.759529 & 0.956656 & 0.6333 & 0.770734 & 0.824989 & 1.84287 & 1.91046 & -0.502317 & 0.13267 \\\\\n", + "\t26 & -2.1921 & -2.49466 & 0.354855 & -1.93156 & 0.941979 & 0.978917 & 0.89486 & 0.463239 & 1.12537 & 1.70528 & 0.717793 & 0.645888 & 0.783968 \\\\\n", + "\t27 & -1.46602 & -1.24922 & 0.307978 & -1.55097 & 0.618908 & 0.662508 & 0.475957 & 0.484719 & 0.401565 & 0.55988 & -0.376938 & -0.933983 & 0.390013 \\\\\n", + "\t28 & -1.83318 & -1.53269 & 2.55674 & -1.51828 & 0.78941 & 0.908748 & 0.649972 & 0.668374 & 1.20058 & 0.277963 & 1.2505 & 3.3137 & 2.22036 \\\\\n", + "\t29 & -0.784547 & 0.276583 & 3.01105 & -1.11979 & 0.920824 & 0.750218 & 1.26154 & -0.403364 & 0.400667 & -0.217598 & -0.72467 & -0.391945 & -0.650024 \\\\\n", + "\t30 & 0.464456 & 1.33264 & -1.2306 & -0.357976 & 1.1825 & 1.54316 & -0.60339 & 3.38309 & 0.823741 & -0.129951 & -0.65798 & -0.499535 & -0.414477 \\\\\n", + "\t$\\dots$ & $\\dots$ & $\\dots$ & $\\dots$ & $\\dots$ & $\\dots$ & $\\dots$ & $\\dots$ & $\\dots$ & $\\dots$ & $\\dots$ & $\\dots$ & $\\dots$ & $\\dots$ \\\\\n", + "\\end{tabular}\n" + ], + "text/plain": [ + "6670×13 DataFrame. Omitted printing of 8 columns\n", + "│ Row │ Trait1 │ Trait2 │ Trait3 │ Trait4 │ Trait5 │\n", + "│ │ \u001b[90mFloat64⍰\u001b[39m │ \u001b[90mFloat64⍰\u001b[39m │ \u001b[90mFloat64⍰\u001b[39m │ \u001b[90mFloat64⍰\u001b[39m │ \u001b[90mFloat64⍰\u001b[39m │\n", + "├──────┼───────────┼───────────┼────────────┼────────────┼────────────┤\n", + "│ 1 │ -1.81573 │ -0.94615 │ 1.11363 │ -2.09867 │ 0.744417 │\n", + "│ 2 │ -1.2444 │ 0.10966 │ 0.467119 │ -1.62131 │ 1.05668 │\n", + "│ 3 │ 1.45567 │ 1.53867 │ 1.09403 │ 0.586655 │ -0.327965 │\n", + "│ 4 │ -0.768809 │ 0.513491 │ 0.244263 │ -1.3174 │ 1.19394 │\n", + "│ 5 │ -0.264415 │ -0.34824 │ -0.0239065 │ 0.00473916 │ 1.25619 │\n", + "│ 6 │ -1.37617 │ -1.47192 │ 0.29118 │ -0.803111 │ -0.26424 │\n", + "│ 7 │ 0.100942 │ -0.191616 │ -0.567421 │ 0.378571 │ -0.246656 │\n", + "│ 8 │ -0.319818 │ 1.35774 │ 0.81869 │ -1.15566 │ 0.634484 │\n", + "│ 9 │ -0.288334 │ 0.566083 │ 0.254958 │ -0.652578 │ 0.668922 │\n", + "│ 10 │ -1.1576 │ -0.781199 │ -0.595808 │ -1.00555 │ 0.789829 │\n", + "⋮\n", + "│ 6660 │ -0.52427 │ 1.04173 │ 1.13749 │ -1.27262 │ -0.607382 │\n", + "│ 6661 │ 1.32516 │ 0.905899 │ 0.84261 │ 0.869143 │ -0.691461 │\n", + "│ 6662 │ -1.44368 │ -2.55708 │ -0.868193 │ 0.390457 │ -0.0331364 │\n", + "│ 6663 │ -1.8518 │ -1.25726 │ 1.81724 │ -2.17285 │ 0.948696 │\n", + "│ 6664 │ -0.810034 │ 0.0896703 │ 0.530939 │ -1.14212 │ 1.26049 │\n", + "│ 6665 │ -1.22395 │ -1.48953 │ -2.95847 │ -0.645307 │ -2.67022 │\n", + "│ 6666 │ -0.282847 │ -1.54129 │ -1.38819 │ 1.48327 │ -1.23406 │\n", + "│ 6667 │ 0.475008 │ 1.46697 │ 0.497403 │ -0.46861 │ 0.496553 │\n", + "│ 6668 │ -0.408154 │ -0.325323 │ 0.0850869 │ -0.182984 │ -0.46577 │\n", + "│ 6669 │ 0.886626 │ 0.487408 │ -0.0977307 │ 0.830857 │ -0.168597 │\n", + "│ 6670 │ -1.24394 │ 0.213697 │ 2.74965 │ -1.80285 │ 1.11401 │" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "#using Pkg\n", + "#pkg\"add CSV DataFrames\"\n", + "#using CSV, DataFrames\n", + "\n", + "cg10k_trait = CSV.File(\"cg10k_traits.txt\"; \n", + " delim = ' ') |> DataFrame\n", + "names!(cg10k_trait, [:FID; :IID; :Trait1; :Trait2; :Trait3; :Trait4; :Trait5; :Trait6; \n", + " :Trait7; :Trait8; :Trait9; :Trait10; :Trait11; :Trait12; :Trait13])\n", + "# do not display FID and IID for privacy\n", + "cg10k_trait[:, 3:end]" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "

13 rows × 8 columns

variablemeanminmedianmaxnuniquenmissingeltype
SymbolFloat64Float64Float64Float64NothingInt64DataType
1Trait10.00221138-3.204130.125013.47940Float64
2Trait20.00135253-3.511660.03351734.913420Float64
3Trait3-0.00129591-3.93844-0.0007821627.91630Float64
4Trait40.00230893-3.60840.2281653.127690Float64
5Trait5-0.00179039-4.148750.03103432.717180Float64
6Trait6-0.00119598-3.824790.0362422.589730Float64
7Trait7-0.00198906-4.272460.0698012.653780Float64
8Trait80.000614075-5.62549-0.03863015.80570Float64
9Trait9-0.00180965-5.381970.1065712.571940Float64
10Trait10-0.000437029-3.54851-0.09665076.537820Float64
11Trait11-0.000615918-3.26491-0.06804374.262410Float64
12Trait12-0.000588783-8.85191-0.14109913.21140Float64
13Trait13-0.000151238-5.5921-0.14102224.17440Float64
" + ], + "text/latex": [ + "\\begin{tabular}{r|cccccccc}\n", + "\t& variable & mean & min & median & max & nunique & nmissing & eltype\\\\\n", + "\t\\hline\n", + "\t& Symbol & Float64 & Float64 & Float64 & Float64 & Nothing & Int64 & DataType\\\\\n", + "\t\\hline\n", + "\t1 & Trait1 & 0.00221138 & -3.20413 & 0.12501 & 3.4794 & & 0 & Float64 \\\\\n", + "\t2 & Trait2 & 0.00135253 & -3.51166 & 0.0335173 & 4.91342 & & 0 & Float64 \\\\\n", + "\t3 & Trait3 & -0.00129591 & -3.93844 & -0.000782162 & 7.9163 & & 0 & Float64 \\\\\n", + "\t4 & Trait4 & 0.00230893 & -3.6084 & 0.228165 & 3.12769 & & 0 & Float64 \\\\\n", + "\t5 & Trait5 & -0.00179039 & -4.14875 & 0.0310343 & 2.71718 & & 0 & Float64 \\\\\n", + "\t6 & Trait6 & -0.00119598 & -3.82479 & 0.036242 & 2.58973 & & 0 & Float64 \\\\\n", + "\t7 & Trait7 & -0.00198906 & -4.27246 & 0.069801 & 2.65378 & & 0 & Float64 \\\\\n", + "\t8 & Trait8 & 0.000614075 & -5.62549 & -0.0386301 & 5.8057 & & 0 & Float64 \\\\\n", + "\t9 & Trait9 & -0.00180965 & -5.38197 & 0.106571 & 2.57194 & & 0 & Float64 \\\\\n", + "\t10 & Trait10 & -0.000437029 & -3.54851 & -0.0966507 & 6.53782 & & 0 & Float64 \\\\\n", + "\t11 & Trait11 & -0.000615918 & -3.26491 & -0.0680437 & 4.26241 & & 0 & Float64 \\\\\n", + "\t12 & Trait12 & -0.000588783 & -8.85191 & -0.141099 & 13.2114 & & 0 & Float64 \\\\\n", + "\t13 & Trait13 & -0.000151238 & -5.5921 & -0.141022 & 24.1744 & & 0 & Float64 \\\\\n", + "\\end{tabular}\n" + ], + "text/plain": [ + "13×8 DataFrame. Omitted printing of 2 columns\n", + "│ Row │ variable │ mean │ min │ median │ max │ nunique │\n", + "│ │ \u001b[90mSymbol\u001b[39m │ \u001b[90mFloat64\u001b[39m │ \u001b[90mFloat64\u001b[39m │ \u001b[90mFloat64\u001b[39m │ \u001b[90mFloat64\u001b[39m │ \u001b[90mNothing\u001b[39m │\n", + "├─────┼──────────┼──────────────┼──────────┼──────────────┼─────────┼─────────┤\n", + "│ 1 │ Trait1 │ 0.00221138 │ -3.20413 │ 0.12501 │ 3.4794 │ │\n", + "│ 2 │ Trait2 │ 0.00135253 │ -3.51166 │ 0.0335173 │ 4.91342 │ │\n", + "│ 3 │ Trait3 │ -0.00129591 │ -3.93844 │ -0.000782162 │ 7.9163 │ │\n", + "│ 4 │ Trait4 │ 0.00230893 │ -3.6084 │ 0.228165 │ 3.12769 │ │\n", + "│ 5 │ Trait5 │ -0.00179039 │ -4.14875 │ 0.0310343 │ 2.71718 │ │\n", + "│ 6 │ Trait6 │ -0.00119598 │ -3.82479 │ 0.036242 │ 2.58973 │ │\n", + "│ 7 │ Trait7 │ -0.00198906 │ -4.27246 │ 0.069801 │ 2.65378 │ │\n", + "│ 8 │ Trait8 │ 0.000614075 │ -5.62549 │ -0.0386301 │ 5.8057 │ │\n", + "│ 9 │ Trait9 │ -0.00180965 │ -5.38197 │ 0.106571 │ 2.57194 │ │\n", + "│ 10 │ Trait10 │ -0.000437029 │ -3.54851 │ -0.0966507 │ 6.53782 │ │\n", + "│ 11 │ Trait11 │ -0.000615918 │ -3.26491 │ -0.0680437 │ 4.26241 │ │\n", + "│ 12 │ Trait12 │ -0.000588783 │ -8.85191 │ -0.141099 │ 13.2114 │ │\n", + "│ 13 │ Trait13 │ -0.000151238 │ -5.5921 │ -0.141022 │ 24.1744 │ │" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "describe(cg10k_trait[:, 3:end])" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlgAAAH0CAIAAABuMsSDAAAABmJLR0QA/wD/AP+gvaeTAAAgAElEQVR4nOydeWAURfb4X/fck8mdyQUBBULCIQIiqyEgh6IiEF1RUVjQVVYEdgFRyQIGD+SUa/2xX5dL1FXDIRhcBbkRiHiAHAmQcEguyDn3Pd1dvz8axjgzmcz0TKZnJvX5AyZvuqte95uq19VV9R6BEAIMBoPBYNorJN8KYDAYDAbDJ9gRYjAYDKZdgx0hBoPBYNo12BFiMBgMpl2DHSEGg8Fg2jXYEWIwGAymXYMdIQaDwWDaNdgRYjAYDKZdgx0hBhOuUBQ1bdo0pVI5aNCgmpoaVqhWq8eMGZOQkDB27Fi1Wu1BiMFgWLg7QtwIMRh+WbNmjU6nq6ioyMnJWbhwIStctmxZ586db9682alTp+XLl3sQYjAYFoJziLX333//zJkz69evX7hwoVqt3rhxIwDk5+cbDIaVK1fOmTMnOjp6yZIlLQkdlJeXZ2RkyGQyDjrQNE0QBElyd+eRUQK/lJeXp6eny+VyXy+BoiiBQEAQRMieZbfbRSKRT6cEk/79+3/00Ud33323Xq8vLy+/5557ACArK6uoqCg7O/vSpUt5eXllZWUtCR0cO3bs+vXr48ePb7VGhmEQQgKBwEsNfT0eIUTTtFAoDP7xAoEgrNtgRkaGVCr16W644uv9DM0SOMDdEQaqEfbv33/jxo39+/fnoINGo5FIJNycKItWqxWJRHK5nHMJOp1OIBBERUX5U4JYLJZKpZxL4Jf+/fuvXr164MCBvhpCrVbLZDJfL7ypqSk6OlosFvt0VkNDQ3x8vK+tq66uTqlUhmz/mJiYOGXKlPXr13fp0uWjjz666667AEChUDQ0NMhkMrPZnJKSotPpWhI6yMrKSk9P37lzZ6s1ms1mu90eExPjpYYWi8VqtcbGxnp5vNVqNZlM8fHxXh5vs9kMBkNCQoKXx1MUpdFokpKSXL9SKBSh/NDjGbYX7d27t16vT0xM5FyO3W7XarVu70/QSqAoSq1WK5VKziVwgLvXraio+OKLL4YNG8Y2QlZYU1PTuXNnAGDfw3gQOtDpdCNGjLh06RIHHbRarUQi8ceF6HQ6kUgkk8nOnTtntVqjoqJ69uzpUwl6vV4gEPjjSvV6vVKpDF9HGGR0Ol1JSYlIJLr//vt9HeFFGDqdDiFUWlq6bt26KVOmnDx5EgAQQuxtYZ+s2SPdCh1ERUWtXLnSG/cjkUjsdruXjs1qtZ45cyY6OvqOO+7w8oqsVqtYLPbJEQqFQu+PpygKALw/HuMnGo3mwoULANClS5fU1FS+1WkR7o4wUI0wOjr6ww8/5Ob/RSKRnyNCsVgsEom+//77p154hYyKTZOTl86d9qkEiUTi54hQIpFIJBLOp7c3Xn71nz/8WmKsvGCxWHwdF0YYSqVy1qxZaWlpM2bMWLNmDStMT0+vqqrKzMysqanp0KGDB2Fb8+aiZatXr35wxPA9RV8Gp0ZMqPHMi9OKT5+ndI3vv7Ng+vTpfKvTItzf+TRvhCUlJayQbW8A4NoInYQOCIJISEggeaWmpgZ6PWiYuIlCBC8KcLZCO6Sy4rph/DqCFGzdunXv3r18q8MnDz/88JYtW6xW6/r16wcMGMAKx4wZs3nzZoTQ5s2b8/LyPAjbmqtXr1J3/onGed7aMRXXrhqe+xD6P863Iq3AvQsO8UaIiWwYgXjq+59Of3Uu34rwyZIlSw4dOpSSknLw4EF2tRoAFBQUnDt3LiMjo7S0dMGCBR6EGAyGhfur0SVLlvzlL39ZtmzZ3XffvWnTJlZYUFAwYcKEjIyM/v37f/rppx6EGIxf2C2mMW/TX07lWw8+SU1N3b9/v5MwLi7um2++8UaIwWBYuDtC3AgxQeZ86YXJL8+oqKzkWxEMBhNR4NmpZlj0Nxua0u7s3rlrd4Zh+NYG48y/P1z/qyjLRjmvt8JgMBh/wI6wGZobNoaofemrqutXOW+vDCMGDRpE3Gbq1FvvGEM9NlBKNyC83Z2NwWAw3oAd4R8RiiElk28lggFC6NKlS9XV1Xq9Xq/XOxbf4wBdGAwm4NA0bbPZXLfPhQjYEbZT6urqbDZbXl5eWlraxIkTHaFGdu3aNWPGDIlEMmPGDEeoEbfCYFJZWdl74OBPv9ga/KoxflJ3o2b58uV/f2N+5+w+//73v/lWB8MDdP31WbNnS2XyyS9O4VsX9wQ1nlsIYjab//Of/5w6dYqmuYeGCUdqa2vvvffeVatWderUafbs2TNnzvziiy+AU2wgs9n873//u0ePHmzYDu8xGAwURdnt9laP3L9//290jEXmHLcJIUav13tZl1Ao9DXEmsFgkEql3mz0lMlkQY6OCACDBg0qLi5mP7/88ssffvghAKjV6kmTJp04cSI3N/fjjz9mo6i4FbYper3eaDKCvr6kpi7/rfeAFKK07BsuPx5Me4DR1qLxq0Eks9M/8K2Le7iPCMNyhsmFuW++PX/Lnh3HztE+duLhTt++fQ8dOtS3b9+EhISlS5d+9913rJxDbCCCIBQKBdGWAAApiyFEf4y/Y9LcqK1PSkmb+eprbVq7lxoGmRB/uT1s9JP79h8AhiaUXRBlI0QSkEYHoV4MhgMcn2EdjZCNOuh4Fmbb244dO+bMmbN8+XI20YRbIe/877sD6zd9dPr0r9T9M+BqcXt7R3z69GmLxZKTkwMAYrHYEeONQ4AuqVQ6adIkhULha6w7u93uZdBtqVTqxtnoGxmxnHpsvtl+TqFQeC7BbDYrFApfB21Go1GhUIRm6B/Hy+2ysrIRI0asX7+evZO7du0qKipi32Pn5eWxzc2t0IHNZjt48KA3UXYtFovdbvcmImDZ+V9JRRINAPD7ujOKoiwWi4ezrFar1Wr1fExzbDabT8dTFNXS8SKRyPssGZgIg2MLD+AME8MwNTU1dNCZ9dobX9uzbuqcmsStRnv16tUrV66YTKbgKBP8RapGo/GJJ564ePGizWZ79913H3/8VgykcIsNRIKwnYZpZV9ub9y4saKiIjY2dubMmaycw8ttu92+e/dusxdYLBaLxeLNkQiguQtkP9vtlDfn8gLeMdWe4TgiDOAMk8FgePnll8+cOcNBDX+yT9A0A70fJk7dCgf8hyYrie4/5CGrqu7r3V+1mh+KzT7h/TOp2xKSkpL8CR3Ogdzc3IULF44ZM0ar1Y4aNeqDDz5g5Tg2ULjAvtxmPy9durRXr17sZ27ZJ9auXRvY7BMEQQCwg/jf/5XJpJ5raW/ZJyiK+sc//rF9+/bu3btv27aNfd0SIrO87QqOjjCAjTAmJmbbtm3Jyckc1GDf6XFzIbdeg9x+3/aH/y0645ra2FVD4uPjW1VMKpX6mX1CKpUGP4sCQRDTpk2bNm2akzwcYwPdrKn+8ssv+/fvf+edd/KtS/AI4MvtgHP48GHKbm/3S/FaZ82aNTqdrqKiYuHChQsXLmQDxobRBFPEwPHV6OnTpx3L1VwbIfiSfSJkYWyW/6xfX1BQ8NNPP/GtC6ZlKn89fubixOmv7du3j29VgkrIvtwuKSkZM36yxWr943sWjBs+//zz119/XS6XFxQUvPLKK6wwNLcwRTYcn9mMRuO4ceOOHDnStWtX10a4ePFi10boJAx9jKr6j66QwqN7U1JSBg4cyLc6mBawGqH7YFIa1BfLoUDIvty2Wq3CmCTQNLVpLZFBoNKb6/X6p59++vvvvzcYDP4sOLDb7V7uR/KyBPYVoEMhq83a2NjouQSKorRaLYeV2DExMZxfrXF0hCHbCANM7vPCn/7LtxIYjBsi6eV2uyVQ6c3lcnl+fn5sbCxJkt7M4LYEu6k3ICVcvXq1vLzcbDbD7xNPIBKJWi2coiiGYTjo4M+iX4KXoJqPPPJIXV0d+7mmpiYpKYlbinaGYTjv4qqqqaHkSWBoBGk02K0AAGIZmDQQ3wEar0PSHaCqgphkwmpIkItjYmI86AAAjhX2WVlZhYWFPmmi0+nEYjG3JT984WTBxMREiUTiqyG8NB9N00ajUWW0IMoOEgWYtRCdDNqbEK0EkxqEUgAgSCIxShId3eJONZqmSZL0RsPmFqyrq1MqlaG5fcJPuLVBhBBCyPMNsVqttQ2NDGUHUgAECQQJdiuQJAjFcXKJ51Ue3pTf6vEe2iBFUWq1WqlUell+W5Oenn7q1Km0tLTa2tpu3boZDAYAyMzM/PbbbzMzMy9fvvzYY4+Vl5e7FbpaUCwW+3T3XEEIMQzjj0dxlFBRWQUCEWO3QmwKGJpAGg1ARIElubWbjxDq0qVLkF//8jOdXVJSsmnTJm4LZEKZ8vLyxYsX861FMMAWDHewBUMBNr35q6++6prevNUJpgi24LvvvhvkSnlb19WrV6+OHTvyVXsbEZFDh5bAFgx3sAV5x8/05tiCgQIvcG7vlJSU/OlPfzIajeyfeA8TBhM0cHrzECGcnp4wAUer1T7//PMmk8khCZFIlRgMBhM0sCNsvyCEnn/++fz8/OZCvIcJABjf4VHbkpKS5vEcwi7wPQbDO/46QtwIw5dly5Z17dp13LhxzYUc9jBFGDRNN9ymqampwTusVisv2uIxPQbjP345QtwIw5fDhw/v3bvXNVAThz1MEYZAIEi5jVKpTPEObvt//ASP6d2CELK2DJuwwhUcdLs949eOE9wIXUEIWXwn+N7l4MGDR48eFYvFrIcjCOL48eMQWUHyIh48pncLQsjoO76mlcZEEtxXjQa2ET7+1Piq2gbOyviEmISvv9yalOSc7jwgIITYYAreY7FYgpx6AgAWLVq0aNEi9jNB/B5XgXOQvIixYLjAjuld1xxyHtNHjAVJkkxISHD7FUVRBEG09G24EzEWDD4cHWHAG+HRY8c1ectBkchNH5+QbproT9Ykz5Ak6evuAoFAIBKJ2kgfX+EcJC9iLBguOMb07J8EQRw7diw3N5dz9glswXAHW5AzHB1hwBshAEC3HIgPxms3UtxOU7m2RPMwe37tYcIWDCIBH9MDYAuGP9iCnOA4R7ho0SJ0GwBACOXm5kIIpIAJODRNZ2dn860FhiNFRUW9e/eOi4sbMmQIG7Mx4ikoKDh37lxGRkZpaemCBQs8CMOCvXv39uzZMy4urmfPnu0t01Yk4bS/INQIcGSZCMs+sXbt2s8//7ysrIxvRTBcqKysnDhx4r59+/r27btu3boXXnjhxIkTfCvVJgRsTB9iMAwzYcKEbdu2DRs2bOfOnS+88EJNTQ3fSmF8xnV/QagRgA31ro2wurp69+7djjwaboWhxpQpU1avXs1+fvHFF1etWgUAffr0efPNN3nVC+Mtrha8du3a+PHj77//fplMNnnyZPxAE+K4WpCiqE8//XT48OFGo1EikcTFxfGrIcYzbntRt/sLQg0cWeYWTz755K5duwDAarUWFRWNHz8eAIYNGzZ69Gi+VcN4hasFhw4dumHDBgCgabqgoOCZZ57hW0eMJ1wtKBaLR40aZTQaY2Ji8vLyWGtiQha3vajb/QWhBnaEtxg+fPiFCxfq6ur27dvXr1+/9PR0vjXC+EZLFjxw4MC9994bGxu7du1afjXEeKYlCyoUCoPBsGjRopkzZ/KrIcYzrhZsKXBHqIEd4S3EYvFjjz22e/fuwsLCiRMn8q1OMHC7DCF8g+S5WhAh9M9//vOdd94pLCxcunSpUIhzrYQ0rha8fv3666+/DgBRUVEvvvjixYsX+daxTYiYQJWuFmwpcEeogR3h74wbN+6zzz7bv3//E088wbcubQ67DOGDDz5QqVTvvPPOCy+8wMrDOkiekwWLi4t37dq1e/fu9PR0g8HApv/GhDJOFkxPT9+0adPRo0cRQlu3bu3Xrx/fCgaeCAtU6WTBlvYXhBrcHWGEjScA4KGHHjp16tTw4cNjYmL41qXNaWkZQlgHyXOy4JEjR8rKyuLj46Nvw7eCASbi26BYLN61a9err76amJhYWFgYeXOEkReoMkx7UY6OMCLHE1KptGfPnq7vRZsvi40YWlqGENaRKp0sOH/+fPRHvCmEYRjtbXQ6ndY77HZ7W16Zez3bQxt84IEHTp06pVKpiouLI29Hb+RFi22pFwVfOlKGYep8x58MMBxnTRzjCYPB4DSeKCoqYh9b8vLy2DlSt0InYuIS7DtmE2I55yvxHqvezROx3W4/f/58VVXVyJEjg6BDiMAuQ1i7du3MmTN//vln8CNIXsRYkCAIR8Q7kUgkEolIsvXnRW+OCSy4DbYEm33C7VcURbHZJ1y/8tLQASTggSojxoIkSSqVSg5nca6RoyNkxxMGgyEmJqb5/CfnZ5lD3+z64YcfuCnjMy8/6hp19+uvv37llVc+/PBDR9w4bjAMo9frfTqFvY3+VMqB69evr1u3bsWKFewyhMWLF7NyzkHyIsaCBEHI5be6EplMJpfLg+/kvAG3wZZgs0+4/YqiKKPR6DbGvUKh8LNeXwl4oMqIsSAE/8kS+YfBYHjvvfcGDBjA/imXy81mM/tDlMvlLQk7dOhQVVXlZ9UhyJkzZ+666y69j9y4cUOv1wdZVavVGh8ff+TIEYZh1q5dm5uby8pnz56dn5/PMEx+fv5rr73WkjCCLdinTx/Hn7W1tTRN86iPN+A22BwnCzpht9vr6+uDqY+XNO+KcRvs1atXkCvl6HVbWtaMs9kRBKHwkaioqOCv7G9pGUKERaqMYHAbjFRwGww+HB1hS8uaIy/odgTjdhlC+AbJa2/gNhhJoIgIVBm+cHSEeDyBwfALboMYTKDg/kaOHU84CSMp8j0GE+LgNojBBIRQXA6HwWAwGEzQwI4Qg8FgMO0afsIQy2SygoIChUIBADabTSQSsXtFA4jdTjEMDQBACgExAAAkCQwNAjFQVhBKgLKCQAQMBYSghQNsIBDePgABMAAEAcjDFpnGxsbAXkXI4mRBoVAYqH0/FEXRiACGBoEQaDsIhEDTQJAACAABIXD5iv3lsF9RIBABbZdIJNxqb7cW9L8NWq1WAAIcZSD0+5/sZ4Df/yRIoYAUCAT+1OgWbMHgY6coBhFe9KK3D6BpEIqBsrptp7xYkEB8xA+7cOHCgQMH2M8rVqx49tlnO3bsyKEcs9ksFAodcUCa8/bi5ap+48mTnzOZg6H+CikUMspuxIUD6NG5sHUOjF9FfFWAHvgb+VMh03UQNP5GEsCk9iBK9qDH5sHWOfD0CmL32yj3r+SpHcydfwJVNYloJnt4wunPFs6f27wii8VCkqTDOyqVymeffdanq9DpdGKxWCqVcrgDfOFkwSeeeKJz585uDeEBk8nEhm5pLtz99dcH68Xk1WJm4Hg4uhFyJhKndqEOvUmrDkwaptM9xKVDaNBkOPj/4L4JxPk9KOlOkgDQ3GC6DiJK9qAhU+DIhx07dsxISx3/9O+RqwwGg5db45tbsK6uTqlUhuaGej/h1gZtNhtN064b0i0Wy7w3CxiBBAnFIFYQdiMyakl5NCOQkGIp01RFiGVIKAaJgrCZkFEtSu36zLAB9957r1M5bOQXR0CDVqFp2mKxNE/dAB7bIEVRarWaQ9SSEMTVgmlpaa53wyfc3k8PfL7jqx+PHQaBGPrleduLdriLOPs1Gj2f2Pb6yvdXuD4MMQwjk8lefvllzlfBhSDvW3SlX79+p06d4nauWq02mUxOwsOHD7/zzjtxaZ2h4GdBRm+Y9CEMHC8YPBn+uplM7Q7/sQAAbLCRMUpYcFLQqQ9MXAf3TxDmTIQpn5Ip3WCDDQgC/mMmY1Ng3nFB577w3FrImSy871lYeKpz1l1O1Wm1WoPBwE1/RwnsZucg89VXX/Xq1Ss2Nnbw4MFlZWWsUKVSjR49Oj4+fsyYMSqVyoPQQb9+/Y4cOeJqiFZRqVSuF/763Hx44l1BYga8ugeEEnilkIhLh6FThfc8LrjzHnjmfTIuDeYeAYKElz4hlV3g/omCnAnCTn1g4joyRgnzjgNBwHNrs/r9qXmx9fX1drvdVw3DYkO9/3jfBo1Go0ajcZVrtVqxPJqMS4OUbtDlPjI1E8RyQXIXyOgj6J4LIikZlwbJ3aDrfWRqdxDLZHeP/OSTT1zLsVgsTU1N3mtutVobGxu9Pz5kN9T7CWtBX++GKzabraGhwfvju/buDwtOCp170a6eetGphaTyTthgI4Uiq9XqWiYvNuL+qFtUVNS7d++4uLghQ4aUl5ezQt4j39M0PXzEgwt/smh1ujatKNyprKycOHHihg0bbt68OXbs2MgI2fw7nfvzrUEwCM02iMGEHRwdYch2o4gNVpT3FkEGfvohkrh27dr48ePvv/9+mUw2efLksrIyVs4hBQzDMDdv3qQDBMME6l09CpRK3oCCPsUQsm3QO5Ber8cZIjEhAsfFMo5uFAAmT568dOlSVs458j0myAwdOnTo0KEAQNN0QUHBM888w8o5hGw2GAyzZs3KycnxdZpTo9HIZDKnCXOLxQyggFtuBTX71y2I/cf1CJqmm5qaHH+q1WqGYXwNZadWq0mS9GaOMDY2lvMKHW4EsA3SNP3LL7/06tWr1UqtVqvdbnfN3uAxA44b81Gaxhn/mLn/8PeF//3YqRyWVjVhsdlsLWWTcEtIZZ/AhA4cHWFgu9EXXnjh8OHDHNTQarUSicRsNrN/njhxoqamxn2/2VJfilr4382BCABohlGpVM3ler1eIBD4kwpLr9fHx8fzsljmwIEDb7zxxsiRIxctWsRKkO8pYGJiYjZt2qRUKt0G9feASCSSyWSOCy8vL79/2EOahjoYveD24kOi2b9uubUW0fUIgUCYnJz8+3EEER8f76sjRAiF7GKZALZBs9m8bt06p6x4bjGbzXa73ek23rx585kXptopO+Ep5cAfWhZj1qIeI0x2yilNhNVqNZlM3j9S2Gw2NrO0l8eHVPYJzC+//CISiXr06MGufeURv7ZPBKQblclk06dP57bSyW63SyQS9md97ty5Z1+aTnS9z/07qpb6UqKV750OJQnCSVWapgUCgZ8rtYIfdBshNG/evBMnThQWFnbv3t0h55wCxn9KSkqojn2RrL6Nyo9IAtIGFQrFxo0bXbPquGIymex2e/NYl7/8enbSCy9e0QFCqIVG1OLTjFgkdqrUarVKJBJvNGFhtw14fzxFUQRBeH88pu1gRPJHJ//dcqP88P7vcnJy+FWGY/8bwG5UIBAMGDCA22slyW2AdSfxadpJm+Dkl9wuCgA8OMRbrZkgnFSVSCQCgcCf12ISiST4Y47i4uJdu3adPHlSKBSyUzXsQxkbnXnx4sWuIZudhG0CQQJPe6HCjhB5lFn6/uqLNU2CRC7bnzDhS0lJyfnz5w0G33Kv/gGLXjfn+9h/PRg4pbjDsf9lu9Hdu3enp6cbDAbHpDeOfB8uHDlypKysLD4+Pvo2rByHbA4XQqgNynAyBI6E77rf+wYPfXldUX1DhIQv4OgI22k3ajM1aPW9B+YOuD+Xoii+tfGL+fPnO+2kYeU4BUy40E7bYAQRvut+EUJGTZP++U8JAT+xyQIOx8uYP3/+/PnzXeURHvleXWO2WEuHLyLff4hhGL61iSiqq6srKioQvqte007bYAQRwHW/CCGtVsvchrNKrZZgNBqnvvrPK5fLOVfhDAKnGjlfhT8TTOHtz2vr6lb/6/8JBMIRQwd3SE8PQo2ESIq6DcLzWIGlqamp/6BhOp2WzugXmBJNmuuV1fGpHadN+et7774TmDIxmIASwHW/Op0uLy/vwoULBoPBH0dot9v1er2HTbFnz5796tu9pqfXwNH9DiFy+uBydrMDnDdEIYQ0Gk1DQ4NDQlGUVqvloHxcXBzntRrh7Qjnvvn27goGKKvNbn9p8kS+1cFwxGQymcwW658mCW+cD0yJ6hq7SG7903MNf9zrggkpbtZUffbZZxMmTOBbET4JyLrf2NjY7du3Jycny2SyxMREzsrY7XapVJqUlNRcaDQaN2/eDADV1dUMw4BYDj0ean4A4fTBZZjQ7ADnJcQESSQkJKSkpDgkFEWJxeIgx4MNb0cIANDrIVBVF+39/PDRY2YzDwHEMSGKQASyWADsCEMVXf2vNXV/mzaj3TrCEFn364EPNmz5cP2Gmzdv6mI6gbYWgCQT0mk/9kyHLKG4U9hnqs+r4rOu3fEw5fKghAl9Rj4xvlOnTiaziW9FMD6z7+ChM2fPcjyZoYnU7nwkvwkVQmjdbwusWr3mwl0v6mgBfe+zdMd+0HWgfeAET4GewpaIcIQAREwKKLvyrUX4QdN0dnZ2c0nwl27/eOwI/H1XRLauiGfc089eq6nlW4twJZTX/TY1NV2+fJmi7JDRB0ThlCGOG345wlDoRjGcWbt2bU5OjiPcNkvoL93GNIf3NkiKucdUAoTspPi+B0ZcvHgxcBqFDaG5hen8+fP79u3rec999wwbVV1T03YVhRTcHSG/3ahKpTp58qRarQlIae2TPn36vPnmm05CDtknQpn6utrjx483X5MWSYT9owxDU0ZtyW83uK0SxAScCxcu5D37/NOvL62vvKZ/6wIp8C3VdvjCfbFMnz59unbtOmbMmObCoGWfeHzii2cuXTU21EDGk/6X5isIiH/OXyAUCKZPeyUuLi74CgSEYcOGuQo5LN02Go0FBQWFhYUWi8UnBTQaDZuZ0yFpYd01l+wTAADV578pObb3wOE1S98dMWIEAPga01Wr1QqFQm/2JykUCpEo2L1GoNogTdPnzp276667Wq3RfhvnLzy92fb41pskhfIYiqLYMlss31d9WoCtyO3xAoEgNKOrBxO9Xk/EpmhfKYJp4dqtcYO7IwxUN6rRaO67776Kigqfar9WXqaf/JlgfWvrzRydbIvZJ5y2vjgf53YHDKKpVVeiJCc25Q7KycrKEggEcrncB+3/iF6vT05O5iX7hCsclm6LRKLBgwdLpVJfs09oNJrS0lLa07Ynb7NPuMekQf3yRHa9WCyWyWQymcxXR8helDf9Iy99aKDaoNlsXr58+WsWOJUAACAASURBVOjRo1utkc0+4fgTIdSKn/udlvO6IGQwGNhBIZtNQiDwNpmozWYzGAzem5WiKL1e7zbLRExMDM4+EXwYu23Hjh2nTp164IEH+vTpw5caAd4+waEbjYuLKywsTEtL86kitqkQrW5sdxzQYvYJp60vzse1uAPm4dekZXsTExNTU1P9zD4RFRUVOi2Qw9JtsVj80EMPSaVSX335gsXvf/XtPpPp90Q8nOMUeDqRIEQikUQikUqlvjpC9qzwGihwzj7htHvMLc2zT5SUlDAM08qzyO/ftRzOniDi4uLY2q1Wq1Qq9Sn7hEQi8X7nHEVRAoHAmyvFBAeTqv7/HbtOfHlgBUI8OsIAt3C2xwQA127USdic4CchwrREMJdu/3btmunPK3CYnsDCrQ36ypUrV3JHjjaZTV6PCDEYNyCGtj84i8gawq8aAXaEobMDBsONUFi6jfGH4LRBvV4P8jg/xvC/Y9Zrc3IGvf322/4XheFMY2PjY+Oe+8fr/7RYfZvpjwwCPBQrKCiYMGFCRkZG//79P/30Uw9CTIjgFFcwaCGbV61adePmDegZ2FJbpLa29vjx4xKJZNSoUUGqkifCrg1SJi3qhaPY88ae/YemzXxVp27UJvWilXeR1FG+NeIBfx1hkLtRhNDs+W8dO3a8tp7/POaIoa9cuQIAHTt29GeOsB2ybNmy+Rt2oca6YFTGUGfPnl2wZGUTKOiKs2ajH6lEQxJeHmWuXLnicZUTJmzYtOWT65l5cKVYkNgJMu6G8vboCMNpFQAA2Gy2D1YsOT3wDbud/3SAZq1q6ty3ho8au23bNr51CRsoilq0dMWOnV/RPR4kxL6tMuWG9cblD7btrbz+m/HPK/B0VkD48ssvn5v8V5PJHMAyjx8//tFHHwWwQIwPJHZq59mVw8wRArtStMfw1teLtj20xWR+fgsMfIZvRcKJ/fv3L163+ZfrjRCsKJPIZrY99BpBersiH9MqZWVldIfWNx36QMPVQ2evTn1tfkp6R40GR8kIEqd/PfPa3Pzz5wOU8sU/dDpdbW0tX6EVws8RYsKXL7Zt/8+GTRCXBvEd+dYFE0oggLg0Sqyw2Gk8WRg0nhj/l5WnzVdq+J9momuvvPXe0s7dsv827e+8KBA2jrC+vr5f7oj0O7vTdMi1E6vVqtfrfY2r0t5obGycNPn5onKdzeZtHJAAgxhKIBk5dtyChXiBInfWf/LFijUftEnRpADvpQkCCKFF768dOXZcQ2Mj5D5PymL41giQvoF+/F3bM6vtPPXuYeMIL1++fK1Oo3q5KNT2LTH1vxW89XZCUvLkF//Gty6hjlAig245vFVvt9Am7X7o+e3B9rgcwH8oinr2penzCxZqGHFbtUKEqqqqHAmJMG2BzWZbmP/a/g5PWSIxsyA3guEIAxX5nhBJIDEjgIoFBEZbx4xbRk1YV3Lh4mtz81etWs23RoHHHws2NDQsXLhw6KN5A4aMsNlsbaShtxAC6J4LAGq1+s/PTX7gkTGDHxr152eea2xs5FmxNiYgbbC8vPzbnVtpIEEoCax6DmwgGPnIY2P+/NTHH3/SRlWEKYHqRauqqsrKyoAg4Z4/E0TIDYRomjIYDNevX7906dKlS5dQsFYSBONG+Bn5vr6+/qG8p6bNfiOk3z1e/eFSo2Xlz7p5S1Z36Jp9Z2Z2SGvrI/5Y8C9/m/7Ot6VHjxyu6PtiSEz/6BtLL1xMSe/4VWnj8V/OHEfd9ny332g0tn5iOOOPBWmaLr98ZVTen/vnPEDRdJv2nshiYOLSjpRc//Kb7w4fPrxy5craWpzvECBA+UPKy8vvvm9I7qhxDBOSOcwbr3/zv/8pU9PvH/7IwJGP9+zZ02wO5MpkDwQjtlmrke8vXrzIRr5nGObIkSMajcZoNHbt2rWhSbVz9zdlF0vPNCFGoSTNl6CmFACg6iwCBLXliLJBUyWYtYhAoKoGuwWqzjc/AOw2UFeDSQN2MaiqgLJCzTn2AEAI6i4juxXUNWBUI5KEpkqgrFB1FgCg+hwgBHVXkN0KmhtgUiHGDo0ViLLdPuA8IAbqroDdApobYGwi4tIgJdP2s+nGn5eQm55ftmxZbW1tbGwsQig2NjY1NbWxsTE9PX3voaMMg4Qk/PtfayQSCQDY7fYQDzLXqgXLysr69esnFAoZhtmxY8elS5dMJlNWVlZpaenVq1ch6ymidD8SCAAh0NaC1QiEAPT1wNDQ+BsggMbrwNCgb0BmPVhNoK0Fhob6awAIGiuAtoOhEZm1yGYBzQ1ADNRfBQBQVSDaBiY1ALplR4Sg7jIA3PoxmLVIIADKBqpKQAjqrwIguP4zJY1l4jJIRSJBCiG+A0MKps6cI2CoJ5/IO3fuXEJCQmpqalpamsFgqKmpGTBgQG1tLRuiMzs7GyF0uPinLZ9+JhXA4X17HHHGQzx3ga9t8Ny5c3FxcXK5/NT5i198uau2qgJEUkFiZzCcR4gByopsRoKyAgCiKbCbwUICQoAYsNvAbgLKBoAQQ4HdgkjBra8oG9hMQFkBsWdZwCYAhIBhgLKC1QR2KwAAQ4Esbv/BQ//btYMUisxm8x133KFSqUpKSrp169bU1JSRkTFo0CBWVZ1Od/XqVYTQ3XffbTAYGhoaDAZD//79GYZBCJEkSRAEQohhGIIgSJKkKEqr1Uokkps3byKExo4d62h6EWDBrKwsu91utVqPHz+u1+uvXbuWnp5+s7Z+/9HjNGX79VwpY7fqQUZN2gDLh7ddL4rsNo69aMNV6Hof1ftRYs9y/Qv/JVeMePb5KQatuusdnUQiUUbHDtnZ2Xa7vUOHDrW1tayxHnnkEUeIdn8sSARh7KlQKBoaGmQymdlsTklJ0el0zb/NzMwEgB9++AEArFbrgyMfphFhNhrkUrGNRoxQYtRpaCClErFer09ITKq7UZ3aIaOxvi4uIdGg04qlMqDtFAMyqUSn0ycqk2pr2APqY+MTTAadSMIegGQyqU6rTVQm11ZXpnbs1NRQHxMXb9TrRBIZwVB2BsllUp1Gm5icXFtTldIhQ9VQHxMXbzbohRIpwVA2CkXJZRqNOik5pbamKiU9Q93UoIiJMxt0ArFUAIyNoqOi5Bq1Oik5pa6mKiW9o6qxITo2zmLUS8QikiAsFmtMbGx9Y1NSSqqYhEPffct2o3q9PjExMSaG/ynrlmjVggzDHD16VCqVWq3WoQ89YjSZjXqtTCq1A0nZ7VJ5lF6jjktMVDc2xMbFm0xmkYAAgdhmtcgVCp1aFZ+YpFE1RcfEWKxWAQGEUGK3mOXRMZqmhoQkpUatUiiirDaKBCQQS80mQ0xsnKqhLjEpWavVyOUyO42AoUUSqdloiImLb6q7mahM1un0MomYAgJRdrFMbtBp4xISVA318QmJBqNRIhQwpJC22aRRt3TT1NcmJibpjOb4xES7XiWTK0iRWK9VK+Qys42SRyloqzkuLtZqtZrsNE0Im1sQQj53ga9t8GZtbWxcvM1kICRRao1aLJIQBIEArGajUCwRkgQiBbSdomlKLJEQBDCIsFlMIrFEQBCIJGmKoilKLJEAAIPAbjWLxBKSIAhSQFF2mqLEYjFBEAwQNotZKBILSQBSSFF2irKLRWKSFDAAdotZKCATExPMeq1IHtPUUCeTR4lkUXIB6t+vr81mq6ioMBqNNpphKKprlzsrKiqEEpndZr37rt6VlZU0Td9xxx0SiUStVtfX18fHxycnJ9tstt9++81isdgZkMuk+/buiSQLHj161GAwREVFPTTyYQagpqYmOTlFo9UJpHLGbm2pFzXqtGKZHFE2igGpRKzT65OUSp96UZNBL5JIgabsDJJLJVqNNiklxade1GLUC8RSEtE2ipbL5Rq1SpmSWnejOjmtg6qpMTom1mrUS8WimJhou8UsFArNVps8KooE+C5QFkRtj1wuN5vNCCGj0SiXy52+7dev36lTp7iV/NFHH504ccIf3f773/8ePXrUnxIKCwsPHDjgTwk7duzYv3+/PyW0Na1a8K233iouLva12HXr1p05c8bXs95///3Lly/7etbbb79948YNX8+aO3euTqfz9awQJFBt8MiRI5999pn39f70008bN270/vjz589/8MEH3h9/5cqVFStWeH/8jRs33n77be+PDx28sWBZWdmqVav8qeXatWvLli3zp4SKioolS5b4U0JNTc0777zjTwkcCMYbOc9JfNLT0//xj39wy+d39erVmJgYpVLJWbdr165FRUWlpKRwLuH69etSqTQ1NZVzCRUVFU8//fSDDz7IuYS2plULbty48auvvvLVEGVlZUql0vucOyylpaU7duyIjo726ayzZ8/u37/f14yJp0+fPnnypDePmW+99VZODn8LYlsjUG2wrq7OZDJt2bLFy3obGxvVavXWrVu9PF6tVtfV1e3evdvL4/V6fWVl5b59+7w83mw2l5eXHz9+3PWrCLAgRVHV1dV79uzhXIvBYKioqDhw4ADnEoxG47Vr1w4dOsS5BJPJdOXKlWPHjvl6oj8WDMar0VdffVUikSxevHjevHkURa1YsaL5tw0NDb/++mtb6xDi9O3bNzk5mW8tWgRbsFWwBcMdbMFwxx8LBsMRajSaCRMmnD17lo18z2b1xIQR2ILhDrZguIMt2KYEwxFiMBgMBhOyhO5yYQwGg8FgggB2hBgMBoNp12BHiMFgMJh2Tdg7wqKiot69e8fFxQ0ZMqS8vNync/0P3+dP7Q5KSkoiI8E9hwvx9QZyMxlnM0WMafzB890bNGgQcZsXXnjBg3Vcbed9yVOnTnUr8Vw+4YLn8iMGf363EdCjciHI+xYDS0VFhUKhKC4uNplMK1asyMnJ8en0uXPnTp8+3WKxTJ8+PT8/P8i1s2g0mnvuuSfcDYE4XQiHG8jBZJzNFDGm8QfPd49hmISEhOrqar1er9fr58yZ48E6TrbzqWSz2ewq8Vw+QkjfjDfffHPu3Lkeyg/kXeMVf363EdCjciO8G/nhw4dfeukl9nN9fX1iYqJPp3fv3v3ixYsIoYsXL3bv3j3ItSOEGIZ5/PHHt2/fHu69LbcL4XADOZiMm5kixjR+4vnu3bx5U6FQ3HPPPQqFIi8vr2vXrh6s42Q7n0quq6tzlXguv/lX586dGzFihN1u91A+1zsUWvj5uw33HpUzEdLIKYqaOnXqtGnTfDorKirKZDIhhEwmU3R0dJBrRwgtWbJkzpw5CKFw7239vBDvb6A/JvPJTBFjmkDh9u79+uuvw4YN+/XXX5uamiZNmiQUCj1YpyXbeVPy+PHjXSVelm+1WgcOHFhaWuq5fM53JqQI1O82THtUzoRfI8/KynJ6qbt///5+/frNnTu3+ROfN3gO3+clnGs/dOjQAw88YLPZUBj2ts2t4NOF+Gk+zibzqZawNo3/cLPRjRs3CILwYB23tvOy5Pj4eM+SlspHCC1evHjGjBkertdtaeEC55botgSWMO1R/SG8GznDMPn5+YMHDy4rK+Nwerdu3crLyxFC5eXlmZmZQa59/vz5TvO1x44d41AO73C+EA43kIPJONQSMabxH89379SpU46o942NjQKBwIN1nGznU8mpqamuEs/ls0KKojp16uQapb3V0sIR/3+3Yd2j+kN4O8Ljx49nZWWp1WrHlLhPp8+ePTs/P5+9+6+99lqQa29OxAw7fLoQDjeQg8n8NFPEmIYbLd29w4cPI4S+//775OTkCxcuWK3WmTNn9unTx9U67JHIxXY+lTx16lRXiefyWeG+ffsGDRrU/IpaKr8t7h6PcPvdRkyP6ivh3cgXLVrkzyJYtVo9atSoDh06jBkzRqPRBLn25kRMb+vThXC4gRxM5qeZIsY03Gjp7rEfGIZZt25d165dk5KSJk2aVFlZ6WodxylOtvOpZK1W6yrxXD4rfO6555zyLrVUfpvdQn7g9ruNmB7VV3CsUQwGg8G0a8J+Qz0Gg8FgMP6AHSEGg8Fg2jXYEWIwGAymXYMdIQaDwWDaNdgRYjAYDKZdgx0hBoPBYNo12BFiMBgMpl2DHSEGg8Fg2jXYEWIwGAymXYMdIQaDwWDaNdgRYjAYDKZdgx0hBoPBYNo12BFiMBgMpl2DHSEGg8Fg2jXYEWIwGAymXYMdIQaDwWDaNdgRYjAYDKZdgx0hBoPBYNo12BFiMBgMpl2DHSEGg8Fg2jXYEWIwGAymXYMdIQaDwWDaNdwd4aBBg4jbTJ06lRWq1eoxY8YkJCSMHTtWrVZ7EDooLy83m81OQoqiEEKcdYuYEsICJwva7Xb/y6RpmmEYPwthGIamaf+VCcgVhTJu2yCLn79h/03g580PyA8p9PFgQfDdiL5aDSFEUZT3xwdBJV/h6AgRQpcuXaqurtbr9Xq9fs2aNax82bJlnTt3vnnzZqdOnZYvX+5B6GD8+PEXL150EqrVaj8vW6PR+GqbgJeg1Wojvg+FP1oQIdTU1OR/mTqdzmaz+VmIzWbT6XT+K9PU1BTZDzRu2yCLn79hq9Wq1+s5nw4AKpXKH09mMplMJpM/CoQFHiwIvndlvt40mqZdRzieaWuVfIWjI6yrq7PZbHl5eWlpaRMnTnR0N7t27ZoxY4ZEIpkxY8bOnTs9CB0ghHQ6HRNKVFZWHj9+/McffwyaYpzt5w+BGtOHIGazubi4+OTJk1euXOFbF4xf1NfXFxcXFxcXGwwGvnXBtMi1a9eKi4s9jEpDHCG302pra++9995Vq1Z16tRp9uzZM2fO/OKLLwCgpqamc+fOAMAOAdmD3Qod6HS6MWPGOHVYTU1NFEUJhRzVAwCVSmWz2UQiEYdz7x083CJLMleWfLn1i4EDB3LWQa1WW61Wb3SIi4uTSCScK+KAY0wfGxsLAI5bzQ7fd+zYMWfOnOXLly9ZsqQlYSjzxsJFmz/fDgRxX+/uB/d8zbc6GO7kjZ/URAkt1ZeOHT7oT2PEtB01NTWPPPGMUVV/7szp7OxsvtXhAkdP07dv30OHDrGfly5d2qtXL/YzQoggCPaD492mW6GD2NjY7du3p6SkNBeSJBkfH++PIxQIBLGxsb46wnPnzjU0NDRU/YY+LIl5f3BCQoJSqayrqwMAuVzO+gzvEQqF0dHRYrHYp7OCg2NMX1ZWNmLEiPXr10ulUgDYtWtXUVERO3zPy8tjfZ5bYShz9epV02MLQSCmrn3Cty4Yv6itum5+91Lsv0fzrQimRUwmE0WKpMmd+FaEOxw9zenTpy0WS05ODgCIxWLHaCY9Pb2qqiozM7OmpqZDhw4ehCHI5cuXhzwyllB2QfD7hND6zR/PnDULgBj96COrVy4Xi8Wpqak8KhkoAjim12g09957b3V1NQAghBoaGvyfUdNqtVKplPMo2WqxAABoak4cO0qQJAFQWVkpEAi4lVZfX88wDPsw1xLx8fHskwQG037QarWlpaURMAvL0REajcZx48YdOXKka9eu77777uOPP87Kx4wZs3nz5sWLF2/evDkvL8+DMAQxGAwgj9P8/Tv42+/970+//GLLexfUNUV71u85dLRDivLyhXM8KhkoAjimj4uL2759e1paGnsASZL+PytIpVKZTMbZtUjYE1XV0PV++PtumBqVmprK+QUDQRApKSmeHSEG0w75y9SZe4p2IsqGFIlRimi+1eEOx8Uyubm5CxcuHDNmTIcOHdRq9bJly1h5QUHBuXPnMjIySktLFyxY4EEYZmjriLseNf/9G3ukrMQ+ffp0cXEx+9l1TA8ArmN6J2FzQsdJzJq3sM99Q04Un/xdRODNsmGP2aAbNGSoRBa1bds2vnUJJBRFTZs2TalUDho0qKamhhWG0YK1365dpeI6Ml3uC/dl1Rz7CIIgpk2bduXKlYaGho8//jgmJoaVx8XFffPNN9XV1bt373bMqLkVhj4Wg3bw4CEffbSFb0XaBKPR+MQTT1y8eNFms7mO6RFCrmN6J2Fo8q+l754fstBIu3HMBoNh586dO3fuPHnypOu34UgEr/t1gjJqqX/8T9Tn4ePHj2/btu3ChQt8axQY1qxZo9PpKioqcnJyFi5cyAo5bELjmZB5DuYMflhuEcqkQ3OPCNK6861ImxDJY/rMXELoZoHSzPw3J85d8pd/Ll/w7tLgKxVwAriXNzwQSW11V//zTfGkOW9t376db20Cw+eff/7666/L5fKCgoJXXnmFFXLYhEbT9JkzZ+xB548DQURRVPB1cODPPjTuyzLbBWTEPiiwY/pp06Y5ydnhuzfCcEIaff+IR69evWZ+9C2wmYq/fTchLUNEQtVvV0NzTa83RPa6XwBQqVQ6na55V4tsVvtjr0FdOY9aBZaKioovvvhi2LBhXbp0+eijj1ghhwVrZrN51apVY8eOdVsLG9PA+yX0RqMRIeSNX2GarRhACOn1eq1W600VbaFSTEwM5+aMHaFvIIZhLR1G73gxyKz9pd9MwcW/AwA0XLMl3Gl+8RPBguywntgI4LpfvV4/bty4n376ybUWlUpltVo59y8Wi8VqtXKI0GQyme4b/rBOb7JYLQ4havZtY2OjN+UYDAaCICwWS6tH+tONcob19KWlpevWrZsyZQr70p7DgjWFQrFx48akpKSWKvJpL5lUKkUIRUe3vvhF0GwBGkGQ8fHxHnQIjkrcwI7QFwyNNTdvJnfoZDPq/OkdMDzQcwQhvLUgiBBJIaEjv+r4TwDX/crl8nnz5rl9tqMoyp+9sGKx2Gq1cnhqpGlaVV9nXVUH02IcQsdMlEQi8bJMkiQJglAoFK0eyXl3jT8olcpZs2alpaXNmDHD8XI7rDehhSkR++rPV06cOPHzzz+3EuDUqEayWNuaRlLIJWANpk0pKirasmVLGA/xfCSA634FAkFmZqYoxPBw7QKBIODVkXzMgzz88MNbtmyxWq3r168fMGAAKwz3BWvhCHaEAAD/+9//Rj45cfZ7a03m1l+hYEKQPXv2PDdtzt8/Ogzh/LbTJyJ13W+7YsmSJYcOHUpJSTl48ODGjRtZYZguWLNYLL3v6rNy5Uq+FeECfjUKAHDz5k3oOdzUbRi5+22+dcFwwWg0CjvdrfvLRvi+vcRUc6z71Wq1o0aN+uCDD1h5QUHBhAkTMjIy+vfv/+mnn3oQYngnNTV1//79TsIwXbBGmXREr4f8zNjDF/6OCEtKSqKiohx/RuQeJgwmBIngvbwXLlw4efIkQi0sEbQaP9n21UNjnly99l/B1QvTGmEbvMIvvbVa7fPPP9880FzE7mHCYDBBoaysLOfBx8bPfNNmayEPYsWpa1GZB8zpR3/4ObiqYSIW7o4QIfT888/n5+c3F3LYChqOIGlMl55339G955kzZ/jWxV/wmB4TUmi1WohO0v/1c08HpWVDXIfv9u2PSUwZ8chjwVINE7Fwd4TLli3r2rXruHHjmgs57GEyGo2zZs1S/xGNRqP2Dy9LuH79+szZrxZu20ZT7HrRFpdaNE9JgYzqmgn/baJEdXV1/uugVqvtPCWyx2N6TLjScI3qPEA/ccONBhXfqnCn/QTJC3E4OsLDhw/v3bvXNT4Fhz1MIpFo9OjRMp6Y99aifx29eqi8sZWNEwDNdjEBAEBatkAil0gkAVGDl6XbARzT22y2YGgcYNCZM2d+/fVX3K2EKyIpyOP5VoI74R4k78CBA40NDXxrERgIbsE1FixY8N577zWXHDt2LDc3NzMz89tvv83MzLx8+fJjjz1WXl4OAK7CRx55hM12CwA1NTVJSUlOmedommZ3wnK9Lm9LaGhsNCAx2K0AAGIZmDQQ3wEar0PSHaCqgphkMDSBNBrsVgAE4igwqSC+IzReh8TOpK42JSnBKVVQVlZWYWEh+7mpqSlkE/MCwNKlSxsbG99//32C+P1noFAoGhoaZDKZ2WxOSUnR6XRuhc0tWFVVRdN0RkYG+ydN0/7vTWbz/3n/AzCZTCq9iYlOvmU7dQ3I44CyAW2DmBT3wqZKUiBENJWQEN/85XCXLl3+7//+r3nh9fX1SqUywvIRttoGWXw1hBMIITYzl/enWK3W2oYmRqEETQ0kdobGCkjMAHUNxCSDvhHkcWDRg0gKDA2IAWmMyKzq2CG9eQnN26Ber/dyQ33wqa2tzczMzMrKcgTJS05OBoCsrKyioqLs7OxLly6xIfTcCr20IPjenbKRzDxbjWGYispKIEggSCAFQNsBIRBJExRSb1ZjeVapuQVZDAZDKEaWWbRo0aJFi9jPzbtRL/MRlpSUbNq0ibV6JFFeXr548WK+tfAKdkzvunTbyzF9ZFuQza3oICLzEUa2BfnWwiv8DJIXwRZctGiRSvWHN95srFHPU0gKhSJUYo16v4epV69eHTuGfZgrJ3h5w8mNgwcPHj161PG7IQiCHdN7H94JWzBEKCkp+dOf/mQ0Gtk/1Wr1pEmTTpw4kZub+/HHH8fHx7ckxBbkF/+D5EWqBQmCaP6SBm6/WnASOsE58zYEJLJM85erEbCHqZ2waNEidBsAQAjl5uYCjksSbuDlTuFLYJNjRxIEQUh8x59noLB5esIEhzAN79Q+ac9bmDxD+07wU5HgIHmhAw6xhnEzpnc6IPTDO7VPArWFKcKgabqpqYn9zKZhslqtrZ4VGxvb0mKTNgIHyQsdsCPEYMISP5c7RTACgcCxhEQmk4XsqtH2lRw7tMGvRjGYsMSx3In1cARBHD9+HPAMEwbjO9gRYjBhCV7uhMEEilB5Nfr4U+OraoMUpEBMwtdfbk1KSgpOde0EbMEQgfMME7ZguIMtyJlQcYRHjx3X5C0HRWIQ6pJummix4AS8AQZbkEcCstwJWzDcwRbkTKg4QgCAbjkQH4zZC1Ic1LVh7QhswXAHW5APOIdEcAO2ICe4zxHu3bu3Z8+ecXFxPXv23LdvHyuMsLjpbmPDY8IIiqKmTZumVCoHDRpUU1PDtzoYnyFc4FujABPxIRGOHj3at2/f6OjoDyNpVgAAIABJREFUvn37fv/993yr4x6OjpBhmAkTJnzwwQcqleqdd9554YUXWHkkmbCl2PARQ3t4lFmzZo1Op6uoqMjJyVm4cCHf6mB8Rt+MN998c+7cuXxrFEjaQ0iEiRMnzp8/X6VSzZs3b+LEid6cwoYV9RU2Vjg3ODpCiqI+/fTT4cOHG41GiUQSFxfHysPXhFOmTFm9ejX7+cUXX1y1alVdXZ3NZsvLy0tLS5s4cSKbh6FVEEKW21itVot3BH9rV+Q9yrhaEAA+//zz119/XS6XFxQUvPLKK7wqiGkFtxZU3Oa3334rLi52xPqPDCIsJIJbC8bExGi1WoPBoNfrvdzQyTCM9o/odDqdTqf1CEVRnDXn6AjFYvGoUaOMRmNMTExeXt6GDRtYefia8Mknn9y1axcAWK3WoqKi8ePHs7HhN27cWFFRERsbO3PmTG/KQQiZfcefZxluRN6jjKsFAaCiouKLL75ISEgYNmyY95HpGd9pwwtrN7i1IIvNZnvppZf+9a9/+RNYOdQIYFbXEMGtBT/++OMpU6YkJCS89NJLn3zyiTflCASCpD+SmJiYmJiY5BF/Et759atSKBQGg2Ht2rUzZ878+eefIZxNOHz48IkTJ9bV1f3000/9+vVLT09PT093GxveMyRJOuaxGYYJ2XyE7KOMwWCIiYlx7MWGcH6UcbUgAOh0OoRQaWnpunXrpkyZcvLkyVbLoSiq4Y/pRpuamlpN5xYXFxfkAF2Rh1sLsqxcuXLgwIE9e/bkUb2A438GmFDDrQXnzp37xhtvzJo1a/Xq1fn5+QcOHOBbTTdwHBFev3799ddfB4CoqKgXX3zx4sWLrDx8o1qIxeLHHnts9+7dhYWF7IvslmLDRxLso8yiRYsc493wfZRxtSAAKJXKWbNmpaWlzZgxo6SkxJtyBAJByh9RKpUprcHLzyPCZnndWhAAaJr+8MMPvXwlE0ZEXkgEtxb88ccfZ8+enZaWNnfu3B9//JFfDVuCoyNMT0/ftGnT0aNHEUJbt27t168fKw9fEwLAuHHjPvvss/379z/xxBPQcmz4yCDyHmXAxYIA8PDDD2/ZssVqta5fv37AgAH8qhdYIm+WF9xZEAAOHTqUkZHRrVs3HhULJmGdAcbVgn369Nm0aZPBYPjkk0/uvvtuftVrCe5zhLt27Xr11VcTExMLCwsdc4RhbcKHHnro1KlTw4cPj4mJgWax4Tt06KBWq5ctW8a3goEkIh9lnCwIAEuWLDl06FBKSsrBgwc3btzIr3qBJfJmecGdBQFgy5YtI0eO9L4QhJDpNmaz2eQd/L7qiJisrq4W3Lx587fffpuWlrZjx46QbYPc5wgfeOCBU6dOOQnDOm66VCrt2bOnY0TfUmz4yMDxKPPbb79lZ2dv3ryZlYd1ChgnCwJAamqqa36GyCDyZnnBnQUB4LPPPvOpEHbxPfvZbrcTBOH40wMikUggEPhUEcYVVwtmZ2efOHGCR5W8IVSWYMXEJdh3zCbE8iDUZdW7mSOx2+3nz5+vqqry6dkzrAnsowy2IC8EcMFaxFiQJEnHmIld5RSaaZgCTsRYMPiEiiM89M2uH374IUiVvfxoQkKCk+zrr79+5ZVXPvzww9Bc5Bn6YAsGmevXr69bt27FihXsLO/ixYtZOec1h9iC4Q62IHcQH3To0KGqqoqXqtuUM2fO9OnTx/FnY2Oj1WrlUZ+2I4IteNdddzkJb968yTAML/p4wGq1xsfHHzlyhGGYtWvX5ubmsvLZs2fn5+czDJOfn//aa6+1JIxgCzZvgzqdTq/X86iPZ/bs2dOjR4/Y2NgePXp89913rFClUo0ePTo+Pn7MmDEqlaolYTuxIIter2e3QrUROB8hBhOWROSCtXZFRK77DVNC5dUoBoPxlchbsNaucKz7NRgMTut+i4qK2CW+eXl5bOgZt0JMoMCOMMDU1NzgWwUMBhMGROS634DAMIxKpWouMRqNqNliYLcoFArOc5PYEQYY1PohGAwGc4tIClQZKAiCiIqKai5hZ/KchE74E4eW+xxhUVFR79694+LihgwZUl5ezgrDN7wTBoPBBJOIjO4UEAiCkPgOSXJ3ZxzPrKysnDhx4oYNG27evDl27Fg8zRuO4EcZDIZHIjK6U5hCIMTlZd6RI0c+++wzdqFaQ0NDjx49GhsbASArK6uoqCg7O/vSpUt5eXllZWVuhZmZmYMHD2Z3udpsNpFIFITE0wghm90OAjFQVhBKgLKCQAQMBYQAEAMAQJLA0M0OsIFA6HwATYPQ7QEIADWpNYcO7Fc1NbI1NjU1hWz2icrKyl69eu3bt69v377r1q3btWsXG/0hPz/fYDCsXLlyzpw50dHR7Jy8q9CDBR3vcAKO3W5nCBIQAoRAILplBdoGpCAQQgYE4qa6G+Xl5efOnWteb21tbUpKSoTlRuelDTbHZrMhUggMBaQAGBoIEhAAICBIQDQIREDZbrVToRgo+6221trBer2+tLT07NmzbC16vT6UN9QfPXq0eXSn7OxsANBoNBMmTDh79iwbyIkNDuAq5MWCVqsVgAC2HoSafQYABA4FEOIchr6xsbG5BVkMBgNCKDo6mqvircDRETqgaXrGjBkkSa5btw4AFApFQ0ODTCYzm80pKSlsMltX4YULFxzJON555x2DwfDmm282L9ZoNMpkMn+GuiaTSSKRNI+ZVFNT89H2ry0jZsPWOTB+FfFVAXrgb+RPhUzXQdD4G0kAk9qDKNmDHpsHW+fA0yuI3W+j3L8Sp3agO/8EqmoSUUyHPsTZ3Wj0Atg6B55aTnz9Lhr0PHl6J3PHvaCtJe1mptfDcb98cun8r2yNarVaoVCIRKJWtQ1+Eh8/H2WaW3D58uWjR49m2zAA6PV6/3+vZrNZJBI5vfT/cNOWsth+0PgbaWhkBr8Iha/C+FXkt0uYux6FpkpSV8cMmQKFr8L4leS3y5jeD4OqmtTeZB74G2tQcs9yptdI0N4kVdXM0Jdh62vw1FLyu5VMjxGgbyAbf2OGTSO3z9244T+TJk1qXq83jrDVPE1tQVFR0fz586urq/v06bNx48bu3bsDgFqtnjRp0okTJ3Jzcz/++GM2KZirsLkFV6xY8eyzz3bs2NG1CqPRKJVKOcces9vtFEXJZDLXrxavWF2XPZb8eSvTYwRc+wmSu4LFQFp1THov4tJhNPgl2LMCnloK2+dC3kJizzKU+yL54+dMr5Fw9QdI7gZmHWk3MSlZxOXvUc5k2L8Gnlwi2DnvvXfe6tSp07PPPsvWEuKO0B+8tCD4bkSr1QoAbnukWbNmEYoERiAlJXKmsYKUyBihlJBFg64eaAqkCiSOIiirwKx++sknMjMzXTfde6OSUql0WJClrR2hXxvq9+/f369fv7lz59rtdlYil8vNZjNCyGg0yuVyD0IHffv2PXz4MP1HamtrrVYr7Qd1dXVs5ncHP/74Y2xmf/iPBQBgg42MUcKCk8JOfWDiOrh/gjBnIkz5lEzpChtsQBDwHzMZmwLzjgs694Xn1kLOZOF9z8LUQlJ5J2ywAUHC/xnIuDTIPyq4oz+MXwVDXhLeOw7ePtuhaw9HjfX19Waz2Rtt/bGCn1AUNXXq1GnTprF/RkVFmUwmhJDJZIqOjvYgdNC1a1elUllXV1dXV1dbW3v+/Pk6vykrK6usrHQSPvDIGJhaCA/+Q3D3KNhgY+0oSO0GL38OI2cL+zxyS7jeIkjrDn/7Lzw8R9B75G2DmgRpWfDSJ/DoG4JeI2CDDUgB/J9ekJ4Nf/0IHpsn6DEUNtgEYmlFRYVTvefPn6+trfWssMViCaLREEKooqJCoVAUFxebTKYVK1bk5OSw8rlz506fPt1isUyfPj0/P9+D0EG/fv1OnTrlthY/g0KYTCa1Wu32q+z+90H+98IOPeCvH0HWAzBmAQydKrzncXh+A5meDYsvgVgOG2wgksKaWjI2BeadELIWzBoCY9+EIVOE9z4Jf/k/smMveOccSBWwwSaOitFoNM1rCfEN9YHCgwURQg0NDTabzfvSPOxeJ0hSkNwFMu4W9BgGYrkgPh3SehDZQ4m4NCIqnkjtDt1yyIy7CHmcOD5106ZNQVApIHBcZoMQmjdv3okTJwoLC9nnUBYO4Z0IgoiJiXEa/JG34aae2xK8K83vh3ri94r8v4q25sCBA2+88cbIkSMXLVrESpDvK9ZiYmK2bduWnJwMt5+r2M/+IBKJZDKZVCptLhSLfk9h6hC6HYcRbj79/pEAN6cTt/9VKpVOIxiGYZKTk0Pt1ei1a9fGjx9///33A8DkyZOXLl3KykN/F9rFixd/+OEHrUbDow6YNgVRVjJ7EN9a+ABHR1hcXLxr166TJ08KhUKDwQAA7MsHdkZ38eLFrtO8TkIMvwTwUQbDC0OHDh06dCgA0DRdUFDwzDPPsHIOu9BMJlN+fv7WrVtda9FoNBRFcZ7ntlgs7ICyuXDIyFGmTveZ6+sB2Hkmb3E+lD33D1Kk0WgYhnH8bTAYvMw+4eUsBsZLaF3Ty9Nm7N6z76vthXzr0jocHeGRI0fKysrY6QcW9rcemkl8VCrVC9Nm3aypslgswa89NMGPMm5hxFEDH3hIICCXvTX/4Ycf5lud1gnImF4oFObm5rqdyZPJZDKZzJ8FXyRJOpVs1utMT60iL/T3dXmCy5DcdYxOSKXS5tVRFEUQhNtLc9XTR3UwnmAMjXTWMI3BzLciXsHREc6fP3/+/Pmu8tAM77Rt27a9F2ptyb1IOw77covwepQJGoxJWzJ4gezg6traWr51aYUAjunFYvHo0aOdXkSzSCQSqVTK2RGyDti55OYvmf154Uw0+xcAAGih5NEnn5WKBB+v/3dmZibczkfo9tJCAX+WO/GtuxcQYfNsETaK+olAeQd0vodvLUKI+fPnO00Xs/Kwzo4dGLrlCGJT+Faiddgx/e7du9PT0w0GAzush/a9C4026n7tO+Pc5etarZZvXVoH78YOHdqLI8RgIgzHmD76Nqy8vWefyBoilMfwrYRXOJY7yWSyyZMns1uVAGDXrl0zZsxgVzbt3LnTg9ABTdOlpaX2tqS8vDztzu7I+zlddgIXMW2qVXOazw37Co41igkDho564ucTR62IhAlPBaM+hj537ty3336bnJw8YMCAYNToO+E1PYFxJbDLnd577/+zd+bxURTZA3/dcx+5EwIJoBASWFQOxQsQAY9V5FRUdmURXV1ZZEUOFQUBEcP1U0FlZZVl3XW9UEBAVuRGLkUDBIKBADnIncncV/f0Ub8/GsY4SSbTPZM5MvX9I5/Jm66q1/2m+nXX8d4b999/f4sN2e12AAh8KZAQ4drHr/z888/O5B5QcbGpsE2vyLJci2/nIVHJh8TERBx0G9Nh4Xn+4M6t8HatfFmYFmTTNSXvbTz1j20Hb+qReXAX9h+YdiQky50SEhLWr1+fnp7eWitJSUmBex21Wo2a7V5PTEwkZb7+wt8ELwEAoFAoWtMqeJVCCHaEIYbn2NLSUgDo0aNHpHXpWOhSg1pYIQZEu5h753j0aVzxB+FpEROH4C1M0QOeIwwpdkO9wdR/6N29evVyu2Nj3TAGg4kI8bDcyWxsbD6jGYVgRxhSKDtSah1LzsuUUbpcG4OJIBUVFT/88APf0dPpBUjHX+7kMJ69UPbQQw9FWo+2CcoRchznDbUsEG1JfIrOFt8z5qE1f/+A49hwtosJFe+9995rr70WaS0wIcDpdA66Y+R9U2e62zmuBeOhX1y45OHHHv/xxx/btaEg6fhbmDiGyPpdpJUICOmOcM2aNYMHD/Yu+RWIth0wb6x6c48j4xyl5YJYWduBifJHmc8//3zuyveXnOBEBeLCRCcMw9jNRuvcw+0910uZDfsVA7756VxZWVm7NoTpMEh3hP369fPJnQSSdsC0O90HQkq3cDcaC8TEo4yy2/UwpoVNAhiBKH+U8bJjx44PP/yw+XLH9uKGUYqMa8PUFib2ke4IR4wYMXr0aB+hhB0wbrd7xYoV9t/icDjswSHUwLQVbLfZi0arbx6oha/8vaYIs98BasuyERi5DdWjDM/zFRUVweTMao1gdsheIbg3SWGpeiCI2GgcOmLiUQYACgsLH5n6zKvbzrB4dhATlYR4+4SEHTAkSXbu3Nknxw1xFcmaBFs+FAR/Fu3KiBEjmgslPMo4nc6ZM2cOGTIEABBCZrM5+Cj+FotFo9EI2TibiK98bips0QO16JTQrx9aKI5aOpZlWaPRaDab5XK5fzsmJSWFObUyAPTr1y8nJ2fMmDFNhVGYholhGEVKpmv0Qvjxq3C2GxNwHHfdddedO3fOK4nCWKMURR09ejR8L/RhJ8SOUMIOGJVK9fjjj/ukkHa73Xq93idBuSgoitLr9XKFwv87QQDx7L1ftBDq3k/Ner1eJpPp9fpgIveHH2mbeb/44ovQ5iOsrql96533ykovcWxT73I1d2C75SP0OVahUHTq1Ck68xFC6B5lbDbbXXfd1fRe7MVsNrvdbslPNkIaJpPJhPiW+6GoN+lA0jA1+ZK3Wq0NDQ0EQTidzjYrT05ODv+jzJo1az799NMW3+m/+uqrOXPmrFy5UnhqaVEYNua/9saad9cR3ftJK37ixImcnJxoXuMTYkcYPUl8Xnpt2Sf/+pBFJEy8OTwtNgXJFH96ahrw/OsLX+nbt2/4FZBMlGzmfX7egu/tqWCwyfWxEGU/mpD2KLNu3bqMjIzmtZEkmZCQIPlhzu120zRtsVgIsuXHCFGPFy0/thItH0IQZGJiYnp6OkEQPs/ZLRKRNEyx8k5fX1/Pdeol0VuoEwbfOXLjJ/8ZO3ZsaLUKISF2hNGTxGfP7l30jO2yT54NT3M+cLRrs+xWzcHVz9bVxZYjjJ5HGeg/GkoOgulymJprCavZuHnzZovFMmXKlGDGJ8KJhEcZgiBSU1Nb9ATkVaQpc+7cuTvv/r3TYpR1yWv7aLG05UMJgiBJUvgb+tZDQQinJ5577rnt27e32IqQrFhsYM+mCxc8Hg+IS6J8Neg2AFB21aAHHA5H01VawavUnGBSKwfbt31GNqIr4K9KF7agXC1w+2RFwacRa10q0fMoE3lqfim+VPHEyo8dP21/5JFHAnmriAai6FEG4OzZs+TvRsLv7kf/Wx62RmMdCe/0SqVy/PjxreUfFrIrB+4khMVf3trKy8sF1yv5ZopYz65du5KSku66666QqNQiwTzuxMZDLqb9iLZHmS++2nzg4PcXLl2CnuFpsHU8LuicZ3tqo/xU9M5tNAc/ysQ6Et7pFQrFyJEjW8s/LGRXDtzrsCyLEPLW9vvxj1TV1EB6T3GesMm4tbuu7D/VFz0gf+CBB0KiUsiJ0hEDTHzCsuwfHn14naFbvTEGEqtGCS0+ykRDXJKzZ89WV1fzOBiCSKIt1qjZUE/oUoOqgudQVt9o/h10QEfo8XgcDgffyio1TNRDwN3PEUp/YyCY6GfTpk03DR25eNn/0TQdEQV42v2PDf9+9vk5u3fvjogCkomeWKNms3nEmIlOlzMEN1PKvnnz5m49coKvqT3oIEOjVquV53m5XK7T6frffuel4jNMFJwaQ7uefWF+UmLiS89NmzBhQqTVwWDCgd1u/+STT7799lvPkCcRZZdHaLkTXXP+gPp24vjla7NPRnnvi7bpCS8lJSU/F53jeRSCFyaXhet+k/HyT2+//fakSZOC32ocWjrCG+GhQ4cyu16Tmd29U48+KZnZ504eZ94xk/LIX2ja3HAuZ0KBXVNeXh5pXTDS4RWaO+8be+vwe3bt2hVpXWKAGXPmzXz3ix2HCiI/JnrdPXDNjS6Xq7Gx0ZvkCCMKmUobusqQm/K8snRVFN4PO4IjLCwsJG77I/Pw/3mUetvcQ5FW57f0uJlMuybSSkQ1BoNh1apV+fn5ox969LEnnw42Klo7wLvtJwY+d9pMVFdXR1qXGMDhcHhu/RPXuXekFQEAgPqLb7+7Nqt7j6f/OiPSqsQYCKEDBw6EMvojQiCTqTK62e12qp0zkIgl8uOHoYSUQ1JmpJXAiGPy08/uraRR42UgCH7YU4D+E2mNWqLPcNmZLZFWInopLy8/deoUAAwbNizSuvwGZK1Bo+bxKV0ZOsoekaObS5cuvbbk9W27DxCJIb6jum3m+0c9kJXT5/cjhq1ftza0lUumYzlCTExRWVlZVVXV2GjgbpsBxXvlthp+8J/gv3+LtF4YEZw/f76srOzxac+5U3tSpSdze+XU1dfDPXdGWi+MRHieX//Rx4veWG53OFCXvqTTGNr6Wbed79Sr2qPauWdfaGsOhnAMjbZHCpiampotW7YsXPJG9943LH5jRUSyNwQKQg0NDefPn28eDyJWCJUF7Xb73r179+7d++mnn+7atavfLUPuf3LWqcIzIVQV0yLt0QfNZvOlS5duGTZy0itvN9RW2Sev5wjZLzkPWj1RN7jdAQhbIq2ysrJ58xfYbTakT2u/VkCf1mhzXpvbZ9GiRSdOnNi0adPq1asjtcAYwuMIQ5ICxu12z3jhlfsffHTMhIlFRUX3jH9kyqJ3899cU9nnQWNq74gkwQkQvq7k/977YOCQEVOfeqaoqOjChQuR1kg0wVjwH//4x4oVK9566y0A+Oucl8c9NeuBPzz5+Iw5j8x9w9JQY51ziNTFxnZ11lTzl+kz1PrE0RMmRloX0YSkDyKEFi5a9Mr8BY9PnTrjb891733DDTcPsZmM1unf/Bp1pMfNhCL6tr44TXsOHup53cBbh9zxyvwFy5bHXpib9k6kRVFUY2Pj/v37+w+6lSGUBCkLeRO/wUN5PMzlOsPStf8acd/oR5/+29wXXyorK9u0adPx48fbt+mWCMfQqP9wsRzHFRcX33DDDQDA8/zevXuNRmN9ff3AgQNTUlJ4nr98+fKcBUvsFpPZSfO3PQaH/vnN15tAkwiPfyBrEPbTEMhpAlMVMBRUngEAqCxEgKCuBBgPmKvAZQFGCaZKYGmoPi0cAAhB/QXE0GCuBqcZkSQYLwNLQ2UhAEDVaUAI6i8ihgZLDbhMiGegsQKxnqsHnAHEQ/1FYCiw1IDTiGgHNJYBx/zmgIZLvLmKHzKVddv27Pnkhhu2J3TuPujGgUoSvvz0Y2+gBJlMFrXhEEGMBTmO27t3L0Jo87d7vv7sYwAEiZlEdl9Uefrtdf80GM30vS/AxSMyl9n6wGIoGg6VhYhjobEM7I3IbfOaDzgWjOVXhVdMhjgWGsvBYUBu65WLXFmIWBaMFWBrQJT9qvA04hgwVoDdAO5mQlv9r0dWnUGsB0yXwVaPaOevQk4QNgDluCKsLmINFdyIGaxMceCHT7vlXacioeDHo1rtlTV1HcaCPM/v37/fbDaXlJRce+21J04XlZSWO13uisuX3TaLgdcCzxEeB0rpRvIk/8RH8M64K5eo5hfhB3/l6lEOsNWDx4Xc1itdr7YYEA+VhYAAqs8C4qHh4pXrTzvAVg9OM3JbwVwFDA21564cDE0OFixIO8BaD04jooSDKag7f/VgBNVFV7r2lYOdYK0HS41dlWZN6l1+avtxe6K67LCHpgFg7ty53njiHcmC33//vcFgyMjIEELIms1mnU4nnGBFRYWQpZUkSZvNZnV7Vr/zXmZaaklZhdVmA5eF6N4fTFWAELAMeJzAUAAAHAMeNwIeAIBngaEQ5QDEI8QDSwPlAI4FhID1AO0Cxg0IEMsC40YUCQCI58BzpQgIRTxOAACeQ4TMRqpItQyxzO8G3Ezw7LU5uT26ZWsTk+oNjXcPufXhhydWVlbabDaE0PXXXy+coNPpNBgMADBq1ChvEOBgLEiE4V1Kr9cbDAaNRuN2uzMzM202W9Nvc3NzAeDYsWMAQNP03ffexyJw2a06jbpLly5Go9FqtRKaJIah7XZ7alp6fU1V5+xujQ31yalpTptVqdEi1sPyoFGrbDZ7WkZ6XXVV5+xuhvr65NRUl8OuUGmAY1geaTRqm9WaltGprupy567djYaGxOQUl8OuUKmBYxkeaTVqm8Wa1qlTXXVlZnY3Y0N9YnIK5XTIVWqCZz0s0mk1Fos5vVNmXXVlZlY3s9GgT0ymnHaZUk0izsNyOp3WYjand8qsr6nq1CXbZDDok5I9LgepVJPA0R5Wr9dbTMb0zM5KEvZ99z9v6LzExMRoztYUuAUpihpx1z0gkxsMjWp9As94PCyXmJTUUFOVmd3NbGzU6BMQ6/F4uKTkpLqaqs7Z3cwmo0arQxxDe9jklGTBfGZjo1qrJ3iW8jDJKSmCySxGo1qnA+63QpNRrdUBz1K0rxBxLEXTKalpgr1sZqNSoyMQ56bolNTUuurKzKyuNrNJqdGSwLvcdEpqan11Vaes7KtC5HK5U9LSfnvkFWFHtaDQBzkE1VWVnbtkGRqN2sRkymlXqLU8QyFSriAJysPodFqbxZqant5QV5OZ1bWhtiY9s7PFZNQnJrsdNqVGy3koRMrlBFAeRq/X2azWlLR0Q11tZla2cLDZaExIunow7Qa5UkYg2sPqtFqbzZqSlt5YV9spK7u+pjqjcxezsTExOcVltyk1Oo52CQd7GE6rUdts9pTU1MaGuk5dhIOzzEZDYnKK025Ta3WM2wVypZxAHpbTaNR2uz0l5crBHdqCv6comgQ+PT09Ozvb7XZXVVUhhHr27Hm66KxcqbSbTdqEJNrtVGn1ToqmPSztdiqUahkJiJAxNEWQpEIuA0LGsizHMkqViiAIHiHa7VKptSQJCAiP2y1TKuUkAYTMQ1NAEEq5HGQylmU5hlEqVQRJ8Ahol0Ol0ZIEgQjSQ7lkcqWcJBBJsh4PAhCKcCzHeGiVWk0QBAKgXE6VWqPVJ6iBuenGG0+dPkPKFS6bZdCgQSRJVlVVmUwmHki1WrVr57ehsSBqf7RardvtRgg5nU6tVuvz7cCBAwsKCnyEr732Wk1NTTCN5ufitb6BAAAgAElEQVTnV1RUBFPDihUrSktLg6nhzTffLCkpCaaGKCFwC9I0PWvWrOBbXLduXfNfhVgKCgrWrVsXvDKzZs2iaTr4eiKIhD4oEORv+Pjx4+vXr5dcHCH0wgsvCPmZpfH111/v2LEjGAWiBMkWRAi98cYbly9fDrytHTt2fP3114EfX1tbu3jx4sCPD4NKYgnHG2Fubu7//ve/3NzcCxcuPPDAAyUlJU2/HT16tMVi8Q4xCRQWFubl5fmPNe6fM2fO9OzZU6fTSa6hqKjo2muvDSbnwC+//NK1a9fExMQ2j1y8ePHgwYMlN9TeBG5BnudPnDgxaNCgIFssKSlJS0tLSwtqut5oNBqNxry8YBMA/fzzzzfeeKP/UZcOY0EfAv8Nt0hjY6PZbBbeV6RRUFDQv39/yTmwKisrCYLo2rVrm0d2VAuC+Juh8PrYrVu3AI93u90lJSX9+/cP8Ph2UikYC4bDEc6ePVulUuXn57/yyissy65atarptwaD4eTJk+2tQ5QzYMCA4LO6tx/Ygm2CLRjrYAvGOsFYMByO0GKxPPbYY4WFhUIKmLAFv8eECmzBWAdbMNbBFmxXwuEIMRgMBoOJWqJ3uTAGg8FgMGEAO0IMBoPBxDXYEWIwGAwmrolqR1hUVBTM/oetW7def/31ycnJw4YN81lt3CbBR/YLpvUOQJC2g6AvYKhiM8a5HQWCsWYEu2E8227IkCHEVaZNm+bzbfML6/9aNa3tiSee8GOUFk0WeOWCqmKVDwHtt0UxSCwWy0033SRZw4qKCr1ef/ToUZfLtWrVqsGDB4sq/tJLLz377LMURT377LPz5s0Lc+uxTpC2Q6G4gEFaMFRqdACCsWYEu2E8247n+dTU1KqqKrvdbrfbhZ34TfG5sP6vlU9tc+bM8WOU5iYTVbnb7RarfEiuWJQ6Qp7nx48f/+WXX0q+me7fv/+pp54SPjc0NKSlpYkqnpeXV1xcjBAqLi7Oy8sLc+sxTfC2Q6G4gEFaMFRqxDpBWjOC3TCebVdbW6vX62+66Sa9Xj9u3Lj6+nqfA3wurP9r5VNbTk6OH6M0N5moyuvr68UqL/0yNSFKHeGyZcvmzJmDEAr+nZVl2WnTpk2fPl1UKZ1O53K5EEIulyshISHMrcc0IbQdCuIChsqCQaoR64TKmhHshnFou5MnT44YMeLkyZNGo3HKlCmTJk3yOaC1C9vitfKpTS6X+zGKH5MFUvmkSZMkKx8M0eIIe/fu7R2q3bdv35133unxeJDI7te0EoHdu3cPHDjwpZdeYhhGlD7+I/sFiOTWY4uQ2A6F1HwoRBYMXo2YI3hrRlU3jB/bNb/sAjU1NSkpKT5CrVbrDT3ovbCBXKuamhqCIPwYpTWTBVi5j6qtKR+qru0lWhxhU+bPn+8zkXno0CGxlfA8P2/evDvuuOP8+fMSdOjVq5cQa7ikpCQ3NzfMrccuIbEdCsUFDNKCoVIjpgnemhHshvFsu4KCgiNHjgifGxsbO3fu7HOAz4X1f618apPJZH6M0txkoirv3LmzWOUDuyRtEI2OsCmSB2QOHz7cu3dvs9lsv4qo4rNmzZo3b55gwrlz54a59Y5BMINpwV/AIC0YKjU6DNKsGcFuGM+2+/777zt16vTLL7/QND1z5sxp06YJ8v379wsffC5sa9dKON6ntn79+jU3Sms1o9YN0WLl06ZNE6t8SK5Yh3WES5cuDWZ9rNlsHjVqVHZ29pgxYywWS5hb7xgEc9bBX8AgLRgqNToM0s49gt0wnm3H8/zatWtzcnLS09OnTJlitVoFufci+FzY1q6V8MGntsuXLzc3Sms1o9YN0WLlVqtVrPIhuWI41igGg8Fg4pqo3lCPwWAwGEx7gx0hBoPBYOIa7AgxGAwGE9dgR4jBYDCYuAY7QgwGg8HENdgRYjAYDCauwY4Qg8FgMHENdoQYDAaDiWuwI8RgMBhMXIMdIQaDwWDiGuwIMRgMBhPXYEeIwWAwmLgGO0IMBoPBxDXYEWIwGAwmrsGOEIPBYDBxDXaEGAwGg4lrsCPEYDAYTFyDHSEGg8Fg4hrsCDEYDAYT12BHiMFgMJi4BjtCDAaDwcQ12BFiMBgMJq6JvCMsKSlxu91iS3Ecx/O8hOYQQizLSigIAAzDSCvIsixCSFrZ6CdAC0q+CJILSraX5IIxirQ+GExX4nme4zhpZSX3fei4li0pKbHb7WK7CcdxYq0gzegSunAwvxAJSHeELMtOnz49IyNjyJAh1dXVgtBsNo8ZMyY1NXXs2LFms9mP0MukSZOKi4vFtu50OiV0XQDweDw2m01CQQBobGyUVtBms3k8Hmllo58ALSj5IpjNZmldwmQySbhjIoSMRqOE5sJPZPsgy7IWi0Wa5m632+l0SisbTG+S9pOIfiZNmvTjjz+KPTWXy+VyuUQV4XneZDKJKgIAVqtV7CMIRVEOh0NsQ5KR7ghXr15ts9kqKioGDx68aNEiQbhixYprrrmmtra2e/fuK1eu9COMRWpqao4ePfrTTz9FWpG4oLGx8ejRoyUlJZFWJHqJoT544sSJo0ePXrp0KfxNY0JIYWEhRVGR1qIdQFIZOHDgqVOnEEI2m+3nn38WhHl5ecXFxQih4uLivLw8P0Iv/fv337x5MysSk8lktVrFlmJZ1ul0NjQ0SCjIsmxa9jWJeYMSUtLEFmxoaHA6nX4O4HlesiEizsCBAwsKCto8zGg0UhQVeLU3D/+9OqPbmIcebWhoYBhGgmJ1dXUcx4ktxfN8bW2thObCT6j6YIAW9MHj8RgMhkCO3L17ty6zu6ZLz8em/lmQOBwOq9UqtkUBk8nkdrullZX2k4h+Bg4cuHv3bpZlRZWy2Ww2my3w48vKyjQ6/cGDB0VqhxobG2maFlXE6XRaLBaxDUlGLtmDVlRUfPbZZyNGjOjZs+e//vUvQVhdXX3NNdcAgPD46Ufoxel0zpw5c8iQIaJat9vtMplMwrOJx+NxuVxI/JwTQshYXQFvH9Muvl7s0JnFYqFpWqlUtnZAUlKSSqUSq1LH5nLZJarf2J3ffZrVtetzM2YMGzZs3LhxkVYqughVH7TZbKNGjTp9+rSo1hmGsdvtgQzHnTt3DnKHunsOc7oPNTQ0AIDL5eI4Ttq7hcVi0Wg00vpLY2MjQogkWxgJw32wTex2u9sZvuHKcCLdEQqPEmfPnl27du3TTz/9ww8/AABCiCAI4YN3XqdFoZeEhIQvvviiU6dOolpXq9UymUyn04lVm6Zpp9OZmpoqtqDXdxIEIVZbuVyu0+lwNxONtQ7l3M6XHl+97VhhSRl2hD6Eqg/q9fo1a9akpaWJap1hGLlcHkgpvV5PECQAqFRK4Xi1Ws1xXGJioqgWBUiS1Gg0arVaQlmWZdPS0lp0hC0KMU0J5+qVMCPd9hkZGc8//3yXLl1mzJhRVFQkCLOysiorKwGguro6OzvbjzCmkbxUJ6pgQ7TUon0hZQjx/HX3dtxVt9IJVR8kSbJLly6ydsPrYwiCbL9WgkR4UMD44cnpz0VahfZCuiP8/e9//9FHH9E0/cEHHwwaNEgQjhkzZsOGDQihDRs2eJ/fWxTGKh63m0XJaRl9ru8faVWCJYaWWmBaJE77YIejqKio6eBW1D2MXqXs4sWItBsGpDvCZcuW7du3LzMzc+/evevXrxeECxcuPH36dLdu3c6ePbtgwQI/wliFcfMMzb76syf212B/+umnL7zwglarXbhw4V//+ldBuGXLlhkzZqhUqhkzZmzevNmPsCkURfGhplkjKORNhJB2tFPrxGkf7FhYrdapU6c23cYQtQ+jNOUGTaJ3NrojIX2OsHPnzrt37/YRJicn79ixIxAhJuKEaqmF1WodOXJkRUWF/+YsFovb7fazYqgpW7ZscTmdkPHr1KzHwxgMhoBPDgBAWNMkdu4HIWQ0GmUymahSycnJ4Z8Dxn0w1kEITZ06dd68eQ8//LBXuGXLlq1btwrPnePGjVu2bFlrQi8sy/7www+DBw8W9bulaRoAAuySAMAjHgBoDyMUDByPxyOsGhWlG8OIa0ihUEie6JXuCDGxTqiWWiQlJX355ZeZmZn+m1MoFAGuGPrqq69mv7GGcjoBwDtzo1Qq22yiORkZGRIcIQBIaAuDEcuKFStycnImTpzYVCjhYZSiqC1btjzzzDOiHKHwGiq2gwib0EQVcTqdBEGI2lPvdruF1ViBF9Hr9YE7dR+wI5QE7TTZXZndc24ZdOP2zV9GWhuJNF1qsXr1akEorKrIzc1tvtTCR9h+uFwuWY9BYPqmvRvChBkPTRsMBo1Gg1emAMD+/ft37tzZ/J1e2rrfFStWpKaminKECoUCABISEgI8ngACEL9j53elpaXeOelAQAglJCSI8lIul4thmKSkpMCLBANeMSwJu4FhmIYH8i9WVEVaFengpRaYsGK4tHXr1uxres6a+2KkVYkK9u7de/DgQaVSKXg4giAOHz4M0b72HrlslkhNircf2BFKhZRDclaklQgKvNQCE1YstTBwLPNgPtvR7qISWbp0qTeyCQAghIYOHQpR/jDa4VyggHRHOGTIEOIq06ZNE4RRu/AX0xxhqYXFYjl48GCvXr0EobCqoqqqatu2bd5xiRaFGIwUEDp16uSCVxd99913kVYlSonqh1FeYr6RKEeiI0QInTt3rqqqym632+127wxT1C78xWA6HjH5MHrh8GkDs3pv8c69+yOpRpTRdEUlfhgNPxIdYX19vcfjGTduXJcuXSZPnuxNbCRtFxoGgxFL7D6MEt37w+9GRlYHDKYpEleN1tXV3XzzzW+99Vb37t1nzZo1c+bMzz77DKQG3X7xxRe//FLc2ksh6LaEtGQ0TbtcLrGL1hwOR7PnaAQAPMcF8nxtsVg8Ho+fnQN6vV5YwYXBBIj3YfT8+fN33XXXBx98IITflLALjabp//znP3l5eaIUYBjG6XQGEvOToigECOA3ne6XM2dWrlx544033nbbbaLadTqdHMexknICO51OjUbT4oYBtVotarE+piMh0fADBgzYt2+f8Hn58uXXXXed8FnCwl+FQjF8+HCNRiNKAYZhZDKZ2FIAQJIkQkhsweEPTLh44cJvZQQAECQRSFUURfmPl48D/mLEEsKHUf5q5ilRCjRd6CGaS8eO11l/Wr912n01t956a9jaDUpnTMdFoiM8ceIERVGDBw8GAKVS6b3FS9iFplQqR40aJTaWvMfjkclkEiLQEwTBcZzYgheKCt1LimFWl2a1kYFUpVKp1Gp1tGWfGDJkyNGjR4XPzzzzzLp16wDAbDZPmTLlyJEjQ4cO/fe//52SktKaMMzwPO92u9VqNd6CJhDCh1GNRvPSSy8Fvp9MgGEYnucDKaVWq4nfvg4CxxJ5Q7mUbgqlWWy7LMtKzj7hcrkSEhLwc6cEGhoaaE9HTMkLAJLnCJ1O54QJE4qLiz0ez+uvvz5+/HhBHtULf0MO4s1G499mzVm//p+RVkU0MTbDZDccOnhAp9OdOnUqAq1HJULOd+Fz84dRiNJdaJhYpbS0FHXMrRMAAITkEYb333//rbfeslqto0aNevfdd4XUYhaL5bHHHissLLzxxhs//vhjYXVTc+F9991XX18vVFVdXZ2eni72bUnY0SnhyU4YGBFbsLyiAqV0BeNlSMkGSw0kdAJHI2iTwWUBpUZN8F06X4nI1bt3788//9ynuMlkirZ8hHV1dbm5ub179/bOMAlJFnv37r1169Y+ffqcO3dOmH9avnz5m2++mZmZqVAoGIapr6/v2rVr06qqqqq0Wm2br4kcx5EkGcj7nMPhMDooRLtAqQGEgKFAqQHEkzzXuVN64PEpAm+xKbm5uStXrhQbYk2lUokNTxokhw4dmjhx4oEDB3Jycl588UWapt9//30AmD17tkqlys/Pf+WVV1iWXbVq1fLly4WvUlNTTSaT8KFpVdL6IEKI5/lAztrhcDTa3VfvNAgIEngOZDIg5YlyPk1kclCe54WFsq0d0GIfFKivr5cQdS/iLF++/IsvvvBzQHV1dWpqqtgZH1F3UZqma2qEQXWUlZUl6tfSpsl86N2794YNG8IZWUaiIwySrl27/vOf/xSb3jb6KSkpyc/PLyws9JFHoSM8derU7NmzvTNMHo9HmGHS6/VCECy3252ZmWmz2Z599tnExMRHHnkk0iqHg5KSkqVLl27atEmsI9RqtWFe7hT4w2i8WbDFPigQo44wDi147NixcDrCiK2Suu6663xeLDoAMdTBRM0wde3adeDAgRHRM8yQJEmSZGJiYvRv1SIIYvr06dOnT/eRt5hoIq4sGGkV2gVswfZtNPxNYqIBUTNMGAwG04HBjjBOEbXcCYPBtAc7d+7s27dvcnJy3759d+3aJQijKAxQ3IAdYZwydOjQRYsWjRkzJjs722w2r1ixQpBHS0hDDKajw/P8Y4899u6775pMpiVLljzxxBOCPPKLtOMPHEkhThE1w4TBYEIOy7Iff/zxyJEjHQ6HSqVKTk4W5P7DAGHag2DfCIuKinQ6nfffaA/4i8F0OALpg5goRIgl4nQ6ExMTx40b9+GHHwpy/2GA4gGe5x0Oh8PhsItBWtQ9gaAcodVqnTp1qsvl8kqidDs2BtNBCbAPYqIWvV7vcDiWLl06c+ZMQeI/DFA84M1ULIpgWpQ+NIoQmjp16rx58x5++GGvUELAX4HxD0+qrDNIVkYUShK2b/oiPT29nepv/tvlrtJaEQn7viPI22ve+e9XW8PTFgnw6gszx44dG57mYovA+6AP2ILRQHl5+dq1a1etWqXT6f785z/n5+cLcv8xKQU6tgUJgtDr9QzD6PX68LQo3RGuWLEiJydn4sSJTYUSAv4KHDx02DJuJejTJOsTOOp/Tqao9gqax3Gc0Wj0EVosFpqm/YRESUpKitR2+6KioltvvdXpdAr/BhJW9Ezx+ROqvtB/dBjUUx54T9jOgWlO4H3Qh45tQY7jWjtxg8HAsmyLO9VSUlKkxS+VTFZW1j//+c/Ro0cPGzbsiy++8G4TFFZu5+fn+1m53bEtGH4kOsL9+/fv3Llz9+7dPnIJAX9/pddgSAnHxjVS2Y4uRyaTNY+YI5fLoy2yjEBrA2tfffXVnDlzVq5c2epEfWZueFLKkae/DkMrsYioPtgCHdeCMpmsSxff+PhXlCHJ6Ikso1Qqt2zZMnv27LKysj59+mzYsEGQL1y48LHHHuvWrZsQG6jV8h3XguFH4g9i7969Bw8eVCqV3sHcw4cPQ8cK+NvQ0DB58uQuXbp07dr1L3/5i91uj7RGIcY7sNZUGNNZlDmO69OnT1NJB145IqoPxgrNLdiasGNw5513FhQUmEymo0ePes8xppPRNzfW1q1br7/++uTk5GHDhpWUlERKMf9IdIRLly5tmtkLITR06FDoWNknnnzyyR49elRUVFy8eDElJWXx4sWBlLJYLO2sV8iQPLAWnaxZs2bw4MHnz59vKpSwcoTneYvFYhKJhBzRQSKqD8YELVqwRSEmOmlurMuXL0+ePPnDDz+sra0dO3asd69ktBHiIYIWt2PHxB7tp59++u233xY+//nPf37rrbcOHjw4d+5cpVKpVqvnzZu3adOmQOoRArpHP8LAWvORz5hYsdbcWADQr1+/V1991edICS+4BEFotVqdSKInuXmMdjdoxYItCjERJ0ALlpaWTpo06fbbb9doNI8//njUPtCEwBE2zV/R4kt9TLzpP/TQQ1u2bAEAmqa3bt06adKkQYMGLV++3GKxNDQ0LF68OLZej9okpgfWmhsLAEaMGDF6tO/aAQkvuARBCMFXRRHZaac2+2C0EbgFWxRiIk6AFhw+fLiwP5LjuIULFz766KMR0bZNomLSOBoYOXLkL7/8Ul9fv2vXroEDB2ZlZX300UfCk/Xtt9/es2fPVJFZ06KcmB5Ya26s1o6MiRfcOCRwC2KiE1EW3LNnz80335yUlLRmzZqwaSiKaBnPiThKpfKBBx7Ytm3bgQMHJk+eDAA6nW7Lli3CnoeDBw/27t070jqGg0BXrEWU5sZqjUC2ZGHCT+AWxEQnAVoQIfTKK68cOXLk888/z8vLC6eGosBvhL8yceLETz75ZPfu3RMmTACAF1988ZlnnrHZbLW1tfPmzXvuuecirWB7EXMDa9DMWK0REy+48UmAFsRELYFY8OjRo1u2bNm2bVtWVpYQNS2cGgYOdoS/cs899xQUFIwcOVLI9L1q1SqTydStW7e77rrrySeffPDBByOtIOZXfIzVGjGxciQ+CdCCmKglEAseOHDg/PnzKSkpCVcJp4aBI90RBp5JK1b2cqnV6r59+3pf89PS0rZu3Wq1Wn/55Zenn346srphfPAxlpemb7cQOy+40ojpbHYBWtCPEBNZArHg/Pnz0W8Jr46BInGOUMiktXHjxhEjRmzevPmJJ56orq6GVuKSBBKsJDE5lflqFqHUBnMyAULbW7g7MAxz5syZysrKe++9Nww6xDSpqamqPR8rLv8YhrbY0p9g/PU+QmwsENkHfcAWjHWwBUOLREcoKpNWIFGA9+3YcuzYMcmnIY5n7m++BHT79u1//etf161b5yciKEbgjUULBvQN29Kh+2+55RYfETYWBJfNDlsw1sEWDC0SHaGQScvhcCQmJnq3oEEQQbdzcnJycnKkKRMSHnzwwZDMAiIEzcejLBaLx+PxE2tUr9crFIrgWxfFzp07Z8+eXVNTk5WVtXr1auHJLpCg2yqVKrIr/UJlrJhGVB/0oWNbkOf51gIiOhwOtVrd4qZPjUYT/qgIW7dunT9/flVVVb9+/davXy+sq8R9MPwEtVgmwExa8bWXiwCNeMK/HVsYWHv33XdNJtOSJUu8oY9wNrvYAmeza47YPHbBZ7OTRmvhx3AfDD8S77/l5eUvvPACAAiZtIqLiwV5Rwq6LQ0CQN0MlUrVXNgUmUwWZj29A2tOp9NnYC12g26HBMF/iCX8qwBE9cG4Qshm1yI6na61r8L/Otha+DHcB8OPRNuLyqQVSHotTJgJZmCtY8PzvNlsFjtSHf6MksFks8NEA8OHDx8+fDg0Cz+G+6DQBxmGETWkodfrJU9YSp8jDDyTVkwEK4lPhIG1NWvWzJw586effgI8sAYgk8nS09ObJ5WMNoLNZoeJDvbs2fPiiy/ee++9S5cuFSS4DwqB7xmG0el0gZcK5p1eekkhk5aPUNi2FYiwo8Jy3O7du3v16tWjR49I6+KP8vLytWvXrlq1ShhYy8/PF+Q4JlkMEXgfxEQhrYUfw32QIAghkH3YRllwZJkQ46bosX/6y3//+99IK9IGwsDawYMHEULNB9ZwTDIMpr1pLfwY7oPhJzJBtzUazcKFC/V6PQB4PB6FQhGRVVvNYRiGJ0jgOJDJgWNBJgeOAbkSWM9v/soUwLFAygBxQJCAeAASABmNRqTQ0Df/MdLn0TaBD6xpNJqvvvqqqKhIGLFpPnkWEgvStAeEGhAPBOn3AwIAhVze5lJb7xBT4DQ2NorXPdoRLOgnFVxo+yDP8wzLAUEAQgAEgPevlyZCgrhi1l//Ct1KBoDkJCF2HVkMWdAbfswrEZZcSeuD8gB6hGQQQs1TT7fT61pELEhEJObNL7/8smfPHuHzqlWr/vCHP3Tt2lVUDRRFkSQpYWqUZVmPx6PV+oawcTqd2//37fkLl0w9RpBnvuUHPUQc/xwNfZI4+AG6dxZ8uxLGLoItr8IDL8N3b8LQJ4gfP0fX/5648D3q2o80lPL6DJJx8xxLpnRB19782i3qpgkqTSaTTqcL82KKUOF2u4WMYq0RoAVdLpdSqWxtHP+lV16lkruTSjWqOoMy80hShurOo855JM8hQynq3Jtk3MhUhTJ7k5RVhTx/nTxRWFDgB7vdrtfrxd7f09PTR44c2blzZ1GloplQWdAHjuMoimo+i3P27NmPvz3s1mWStlpen0G6zLwqgWTcPCknCeA5llSoedpJapN4h5FIzETmGkjrDoYyyMwl6s6h7OvIqjN89wEKl2ncDZ3vuOMOn/pdLpdCofCzmikjI+MPf/hDi1/V19dnZGRENnmkBAKx4NixY3Nzc0WdGk3TEJg/KykpWbvuA0DoyrMLkARwq996K8CGnE6nqLXxGRkZ48aNYxgmbGERI/NG2Ldv3759+wqfP/roo0mTJt14442iarDZbDKZTNRUqgBN006ns3lkmWkzZh5zpZNMGVx3D3HxMAwYQ5zahm5+GI58BLf/Cb57E4Y9BVtehdsfgz3vwICxcGo79BlB1J5FPW4mOZrPyCXdFp6hibRuURpNTyoajcZ/5o0ALej/aeDV15dRGT0IbSKqOw+dckiZnGssh069SM7DWWogM5d0W3iHEXXOI201St71yCOP3Hrrrf5blHbXQwjV19eLKhLlhMqCPjAMY7Va09PTfeTffPPNFwVV7vTfkTVn+E65pOkyn5BJui28XE0QCBgPoUkAu5FIyQJDGZHVBxEkXDsIGBp63gKUBXJuJ+wNkDtUZiq7886+M2bM8KnfbDZrNBq1Wi1K25gmEAtOmDBhxIgRol6ghbADgQTC3rt379oP/vmrIyRkBILAE/IYjcaEhARR7y0ulyvwg4MH5yNsQt5Qsq6Yj7QWGEyMYrVa//y3uUWFJyg+OdK6YDAikD5EsHXr1uuvvz45OXnYsGElJSWCMKazT2AiAkLojfxlNE1FWpHYI9r64I8//rjzaMF5TV5oFv0jZLFYqqurw/xygIlDJDpCUcGBcMQgjB/ef//9xRu20LTvVLwfWA81b/HS5Sv/r/20in6isw/KE9KgU2iCBnMNlxbnr+jRKy/612BjYh2JjlBUcCD/EYM4jjt//jwTaXhe0phoK/OBHMeFo/XgiJ73CcW1N4pa0uI2NRxokG/+366QaxJDhLAPIoSMRiMfNKFdecfbGrnxSxW3/wEhFFQwvx4AACAASURBVLxubRJCzTExh8Q5QlHBgfxHDHK5XAsWLLjnnntEKWC322UyGcuyYjX3eDxOp7PplHJhYeGERx+jaBpGvyqmH/96bNPuL3ykadpqtXqFNpuN4zg/c8WJiYlhzmYivE/s2rVrwIABa9eufeKJJ44cOQJBZJQMN936g+VopJWIJCHsg3a7/dFHH/VGKw0QhmHsdntT52exWIR/BdnVvwi8vaVZT/GlSZGrAmS32w0Gg1ditVqF+L2itBUwGo0A0OL6qeTk5Bhd1x0hUGFhYf/+/SOtRmgIarFMgMGB/EcMSkhIWL9+ffO1Z/5RKpWSV42q1eqmq0YPHz5sv+mPsnP7AEDMQnvi1z/Eb6QIQKvVNj0jkiSjbfuE930CAB5//PHly5cLcgkZJSmK+vDDD1etWuW/RafTiRBiGMZHTtM0avXV+le8d9MrN0mGvlBWcfvwu9eszPeuQG7eooTMHgghp9Pp3d0cIGq1OvxRmyFEfTAxMXHjxo2ZmZmimmYYRq1WN/2dp6SkCK0QTfrFFYlwxG97Sgs0KXJVQCQmJjbVTalUBrNqNBa3T0Sc5uNASKFbnL9yyxefRESfkCOx64oKDhQDEYNUOiBDeRf7/vvv8/Pz77vvPrFL0sNGCN8nBM/R5rAYukpzuZQTMFfZWNmZ8jqz2dxaDa21KE3PaKOj9UFMFPPvTz73FRHARXsXEYHEu78QHOiHH36Qy+XCs7MQJgZnnwAAVHJoL6CD679KTk6OWkcoEJL3CY1G8/zzz7e5G4lhmBZfi9VqNdEs9EhzCAB09T1DOJLQJitUpFarba1pl8uVkJAg4Y1QKCiqVPjBfRATNpwdfeGuREcYeHCg1oQdGYRQ37vkjrpI6+EP/D4R68RLH0TIbDZXVVWlpqY2DwiFwYQEiWPl8+fPR79FkAuR76uqqrZt2+aNjtOiEBNZRAX8xVGAo5Co6oMGg+HWkff/cepT7lC/OjD1pQuWvNEzt/fGjRtDW3OUwHFcnz59mkqicDe2xWLxkSAETqe4efRoBk8axyne94mEqwjyhQsXnj59ulu3bmfPnl2wYIEfIQbj5ezZs8VVjcbrH+T4EE8cIYeRfWiF6paJoa02SlizZs3gwYN94qFH4W7sM2eKfEU8t+/b7WFWo/3AjjBOiar3iWBgGEbCLhpMyJGptKAXt/Yb069fv6bR+QUk7ARtbzjeZ2UAAl9JbBPXsUbLyspGPzK5tqIUbnsq0rpgpOC2Gu++595bBg/94dCBSOuCwYhmxIgRzYUSVm5brdbRo0eXlpaKCrotTIgEE8EuwPD0ZrPZ7Xb7SRjSHLfbzTAMRYmIvBjMTtC4doQ7duy4KMv26OhIKxK/FBcXHzt2jOc10oqzThsas4BqOBBSpTCYSCJtJ+iaNWsyMjJEOUJhL2YwC6QzMjICOYwkSQnZJxiRaZiC2R4a1NBoTEzz+odMSAeVvj1q5h2m11at6Xvjrd7Mi5jm3Pn70Z9/e4ANSYzmuCQa+uChQ4cOHjzIcniAOjQIi7QBoPnKbR+hF4Ig1Go12T7s2Lmrtb1Nf//739upUQkEc82lF46Vad7W8Hg8zaOchBC29kJD7zGXuQQhqlN0EvHbqNNmYbP7hbDCuCIa+uCpU6dGPfKnZf/a7Ha3Y/4Qzm6at2T57wbe8t1337VfK1FCtK3c/uBf/2l5l68mcd2/PwunJu2HdEcYK9O8LVJYWJiamT1n7tx29YWQfg2p980AHD1Ew200JJgbG5YtW1ZRURFpRcJNNPRBl8slS+lC3zYlhHU2h2korb/+4csoJZofK0NFzKzcjvroS4EjfY4wVNO8Tqfz+eef37p1q6jWhaDbHo+I3D0CNE27XK7jx48T192FqKaDOWKNin7903ooYSH8mMVi8Xg8fiZy9Xq9qJnkkNCvX7+cnJwxY8Y0FUqINRphjBVVBvOS1f+45ZZbhJ9Z/BCqPuhyuV5++eXPP28WRssvDMPYbDafuNsC6NfO4fNFix+bHfWboNtXSb+WNF0QehNN09KWRVitVrlc3uIwWkT6oIDPBRQWafsc06IwbLRmLA9NeScvY5oQL5aRMM2rUChGjx6t0YhbLsEwjEwmE1sKAEiSRAgplcqgTUe08KlZKGHiaoxgjUbjp+sGOcAtjVDdRu12+8SJE48fP+6/ObPZTFGU74T51VtA8+wEVwXo1wNbfOxAiEjOUiYkWq3WxsbGpmVNJhNBEGKvLULIZDKJjaAd/vwhrSGhD8rlciGdk6iG5HI5wzAqlYpoFkC71c7VZtBt4Sui5c7F2RrnLX5Du+LNd1Ytu//++0VpK6BWq1uLwx6RPhgTWC3mlj0hz10oOklRlIT7cLQRYkcoIUCXUqm8++67xcaS93g8MplMQgR6giA4jlMoFC35LPEQbRUlCIVCoVKpJCeOCTMSbqNarfbll19uc30Xx3FNY40aDIap02fSPu/0za7krzdZ4tdcH6jZ4Sztem3l2yOO/LBq+RveshRFJSUlSXCEQkFRpUSt1mtXpPXBsWPHiu1NMpmMpmmlUtnC7//X1CyhhGmscg+ZRpfsoShKWvYJoSdinycKyt3K/gqEQK7++OOP//KXv4RXo9AT4h9EtE3zNsdqtT4xY/ai/JUShlXjAQkr1mQyWV5enkIkGzduPHDZ7XWrbd40/R9AG2tOQ9ddBw+LVSNURM+9Nfr7YFAkZ5OaxEgrgbkK20H2noW490bzNK/H45m3aOm4hyd9e7jgorxrs1gJGIDw3kbJjJ6hqgoAICOnztA4asLDp06dCmW1sUbY+qDRaLxn/KOjxk90OsObmoDnysrKCgoKmua+xrQfjYZGP9/OnvuCyWQKmzLtRLBDo9E/zevl9OnT7/xjvbvrIJmagtTuYKsJT7scx7EsG/357QRiOHeBtdbqpA4WXqitrR0wYECktQkfkeqD5eXlF6vqHUOnkz99Edqa/UMZLi9ZfWrJspUbP/nP2LFjw9l0HLJt27YyP+uxZQqnw7Fs5f+tWp4fRqVCT7xEltm+ffvOnTtBnQDd+0NRmFwgAHgaqyZP/hMA+uabb0aNGhW2dgMnhh5l2kaTpEjpDACFhYUGgyE7OzvAyBcYaZAKJYR9gxCiHPRDK5MK/hvmduMTl8sFiG/1a5IEQvXZV18vf+P16Jkml0C0TGy0K2VlZQ9Pnrru6GWaDve8IGdvRE9/nHBDC+sz45kjR458++23PN96B5MK4tg9e/YMuXf0mIl/aHMhKwaD8U9RUbO8Ez4QUF1d9fe//z0s6rQXceEIEUIKXRJ/44ORVgRzhbvuvW/XqdL2cITuxup3//WpW6ZV9hx0+PDhb775JuRNYKxW6/DRD419+A8ulztSOiDWs3379jVr1sRhIIWwQVHU8jYDaCAEPHfu3LmysrKwKNUudPyh0bKysp9//pnH0SyjhtraWoQQdMppj8p52s31G0OeP+iuL/vou+rd3x+76aab0tLSpK22xzTnwoULP/zww08nC123PU0e/Xek1HDVlv7rJ7n8650///xzbm7ulClTrr322kgp01Gpra3lCEUAwUbQp1//r6Ki4sMPP+zSpUs4NAs1HfmN0Gg0njx58qYhw//80utuMek82gOWZT0eT6wsmWk/vvvuux55fcOwdwWxDNvnrqqa2mt79sLvhcHj8Xhmz1805K77brj1jr++/DrF8JAc0Vsez3F3PMUSsv9eYJet3XDp0qVIKtNB2blzZ4BHWs3GHTt21NSEb/lFaAmHIwxP9gmbzXb8+PEjR45UV1e/tPiN/rffmd2r7x0PTDQ3Njgmrm6nRgOEsjSMf/BBtVp98uTJyGoijVBZ8IkZsx/545+YIU9AeGIyMW6kSVb1GTp/6coHxk7YuHHj5cuXw9Fu9BG8BSsrK99//x9HNQM9hMI55vWQaygRBHDbH1Wdrjl//vxPP/0kJNjrkIQ/h8/OnTuffX5uQIciHnEcAKxbt06nT4jF3RThcITtF7KZZdnS0tLS0lKXyzX5mefuevjxO+4f37P3dSvz3zh900zaanDODPSJpl3hnFb03PaEnjdUV1eXlpbGXEb14C3odrtnvfjy5//6wJaaKz6sa1DQDeUlZNa3+w4+PmPOyy+/vGDBgq+++urUqVMWiyWcakSWYCzIsuzLS/KnPjODI+TQ87Z20jAYXKb6OUvfuv2eB9Iys7QJifPnz3/vvfdOnDhRVlbWvlH1w0jYAt+bTCabzbZq1aqnp01HgXdVjgWldv1/PnFxkNGtlzIxZcGCBXa7vf30DC3hmCP0H7KZ47ji4uIbbrgBAHie37JlS3l5Ocdxw4cPP3nypMPhUKvVAwYMaGhosNvtxcXFarVaqVQqlKpPNm2z2h1V50+DSp+UlOSkPeyDy4mNcz2Pr4f1U0CdAABQcw4QgoZLwLFgqgSWBmsdol1gb0RuG4AMEQhoB9jqgaHAXIM4BkyXgefAUAYIQf1FQAC15wAAas4CANSeA4SgsQw4BsxVwFBgrUeUA5xGRDuA9SB7I3icYK1HLA3masSxYKxAPA+GUpfVMnbsWNAmpyUnqRVk0akTOp1OuAgymSx6QpM0R5QFv//++/Lycoqizp4vcTicRpMZsR7gue9+qQOEgOeRzQAIgLIjRyMAgNsOCiUCALcNEAcIwGVDchkAApcV8QwgBE4rou2AeHBZEOUAhMBpBtqJEA9OM/K4gefBYQKGAp4DhwlYGgEChxFxHgRyUKhBrmRI9WffHQLnNkKm5J0meVInGUd375nbJS3p4YcmHD52vLSqlkRcnx7dSJK8/fbbKYoym819+/YVwvIJBsrJyTEYDGfPnuU4bvr06YqrYZo7kgUPHDhgNptLSkpyc3MLCgoUCsWq1e9y/ccRHgoay4FjoLECsR4wVSKWBlPVlb7jcYOtDmjnlR5hNyC3/UpHI+WIdoDMDYCAZYFnwG1FCiVQdmQzAOUAWz14nGCrB9oN1jrkcYOlFhgKLDVX+hHrAeNlxDFgrEAcC43liOegsUzoqpzbzoxeQG5f4hm3hPx25YqPNvMOI7IZQKYYPW6CwWylKYqjXX+b9nRRUVF6enppaWlOTo5cLn/uuec6jAVLSkqGDh3K8zzP85s2bUpLS+vcuTP8dn+UcIJCxESEUEFBQVVVVV5e3u9+9zsAKCoqeunl+WabAyHkpmgATlyYPJ4DkgCe43mO58hlb65Z897fc3JyLC5P7+6drTa71enO7pw5buzoCyUl6enpW3d8+/qiV7Va7alTp0aOHOnVk+d5giAoimJZ1ps0+PTp0zqdrkePHl4buVyuM2fO/OlPf/KGBQ7GgkQYZq30er3BYNBoNG63OzMz02azNf02NzcXAI4dOwYANE2PuOc+hgeedirlMheLlCoNwVI8y5BKNeVyUxwvA0QSBMexSKHmPLRSo6VcTo0uweWw6ROTbRZTcmqaydCQntm5oba6U5dsQ11NeqdMU2NjcmqqzWrV63VOp1ujUdEeViknWR5IxIFcyTMemVLN0m6lRke5HFp9gsNmTUxOsZiMKWkZRkNdRmZWfXVlZnZ2Y31danons8mYnJxss9n1eq3LTamVSg/LyUnggCR5VqhQrlJ73G61Tuey23WJiVcqNDamZnTSKGT7vvufN1ht9IRsbhFRFrz73t+bTCZCJrdYbTqdDuRKh9mYkJjIETKX3arX6xEhdzmsep2eI0jK6dDptBwiKLdTq9VxCDxul1arZXnkoSmtRstwHOOhtRotw7Ecw6o1ag/D8Byn0ahpmuERr1apaNoDBKiUKoqmSIJQqlSU203KZHK5gqbccplcoVK6nU6FQiFXqlwOu1KpvPJBpZIrlC6rSafTIYXKYbVqtWqZXGk3NyYlJhIyudVs6n5ND4fVxAORmJTitJlvu+22H374QaFPVpDE3o5rQYZDNdWVXbK61tbVpWdkNjYaUtPTTY2NKWlpFpMpOSXVYjEnJiXZrNbEpCS73Z6g1zscTp1O63K5tVq1y01p1GqKptVKJeVhVAq5h+UUcpLhkIIElgcSEAeEHBBPyAjEIlJOcAzIlYj1EHIVYmhSqeI8tFylZmhKoVIzlFup0XrcLqF7qrV6yuXQ6BLcTrtWn+B02HX6RIfdmpCYZLdaEpNTbGZzYkqq1WxMTk0zGxtT0jLMDbVdunSuratPz+xsqKvr1LmLSk52JAtyHHfs2DEhAOyddw7PyemZmZkJAAghb6xgb8RgwREeO/6z3W6/pmt2nz69EULHfy64ePEiAhKAB4K4Gt73qoNAcPXz1QDAV/8DAECo2XwHAoKUkzK5Si0H3uFyA0GqFWR2dle7w84zjMPlvvfukWazubS0fMSIO+FqIGjhA8/zHMcplUpB1cPHfuickd6zZ0/viTQ0NFy4VHpg397QWBC1P1qt1u12I4ScTqdWq/X5duDAgQUFBWLr/PLLL3ft2iVBmaKionfeeUdCQYTQjBkzpBVcs2bN2bNnpZWNBkJiwXfeeaeoqEhC60uWLKmurpZQcO7cuU6nU2wpmqZnzZoloblopj36YHl5+bJly6Tps2fPni+++EJa2X/84x8///yztLIvv/yyxWKRVjaytGnBp59+uqGhQVSdW7du3bp1q6giDQ0NCxcuFFUEIbRq1aqLFy+KKnLgwIFPPvlEbEOSCcfQqP/I91lZWc8995xWqxVVZ0VFhVKplLBU12Kx1NXVbd++XWxBADh+/LhPGtsAKS4u/uSTT/wkNFi8ePHgwYMl1BweQmLBc+fO/fe//xWb1QEATp8+/d1334n9hQBAQUHBiRMnxAa84Hn+xIkTbe8j/i3xYEEfHA5HeXn5vn37JOhTW1tL0/T69esllC0pKUlLS0tLS5NQ9sSJE0ePHm3xvSHWLbhp06Zz586J2iYkRNJ/7733Ai9CUdS5c+eEwYPAOXv27KZNm7zjnIFQX1/vcrk++uijwIsEY8FwDI3Onj1bpVLl5+e/8sorLMuuWrWq6bcGgyFG11KGkAEDBnTq1CnSWrQKtmCbYAvGOtiCsU4wFgyHI7RYLI899lhhYaEQslnCOwEmsmALxjrYgrEOtmC7Eg5HiMFgMBhM1BK9y4UxGAwGgwkD2BFiMBgMJq7BjhCDwWAwcU1sO8KioiJvcJbA2bp16/XXX5+cnDxs2LCSkpJAigQT6E9Ccx0G/+c+ZMgQ4irTpk0ThP4vtZ9vJbTlRx6IMhhpv+0I9iZpd4woh+O4Pn36NJUEfoX9//59kGY4UU0Ecy5BEbYdiyHHYrHcdNNNYk+hoqJCr9cfPXrU5XKtWrVq8ODBgZR66aWXnn32WYqinn322Xnz5rV3cx0D/+fO83xqampVVZXdbrfb7cJmYdTWpW7tW2lttSZvszkMCuK3HaneJO2OEeWsXr36lltu8TmpAK9wm79/HyQYTlQTwZxLkMTqb4Ln+fHjx3/55Zdif9b79+9/6qmnhM8NDQ1paWmBlMrLyysuLkYIFRcX5+XltXdzHQP/515bW6vX62+66Sa9Xj9u3Lj6+npB7v9St/attLZak7fZHAYF8duOSG+SfMeIcvbt2yeEB2kqDPAKt/n790GC4UQ1Ecy5BEms/iaWLVs2Z84chJDknzXLstOmTZs+fXogB+t0OpfLhRByuVwJCQnt3VwHo8VzP3ny5IgRI06ePGk0GqdMmTJp0iRB7v9St2kIUW21Jg+8OQwS/9uOSG8K/o4RzficVIBXuM3fvw8SDCe2CcnnEiQx85vo3bu3dyx33759d955pzfPraiyArt37x44cOBLL73EMEwgrfsP9NcmYpuLXaRd6pqampSUFOGz/0vt/1uxbbUpD9LuHY/guxKKRG8Se8eIZpqbADU7KT9XuMXiqPV+EWC1gRBIE0jMuYSQmPxNzJ8/32em89ChQwGW5Xl+3rx5d9xxx/nz5wNvsVevXiUlJQghITdN4AWlNdcx8H/uBQUFR44cET43NjZ27txZ+Oz/Urf2rbS2WpO32RwGBfHbDn9vCuaOERP4OI8Ar3Cbv38fJBhObBNI6rkESUyuGl26dKnPVRs6dGiAZY8ePbply5Zt27ZlZWU5HI4AU1qPGTNmw4YNCKENGzaMGzcucFWlNdcxaO3cDxw4AABOp3PChAnFxcUej+f1118fP3688G2Ll1oo0tq3kttqTd5mcxgI4rcd/t4UzB0jFgnwCrf2+w+y2mCaCEmjUmgnBxs2xJ7C0qVLJVwBs9k8atSo7OzsMWPGiErjIq25jkFr5y584Hl+7dq1OTk56enpU6ZMsVqtwrctXmpv2dYMIa2t1uRtNodBQfy2I9ubOmQf9DmpAK9wa7//1pBgOLFNSD6XIMGxRjEYDAYT18Tk0CgGg8FgMKECO0IMBoPBxDXYEWIwGAwmrsGOEIPBYDBxDXaEGAwGg4lrsCPEYDAYTFyDHSEGg8Fg4hrsCDEYDAYT12BHiMFgMJi4BjtCDAaDwcQ12BFiMBgMJq7BjhCDwWAwcQ12hBgMBoOJa7AjxGAwGExcgx0hBoPBYOIa7AgxGAwGE9dgR4jBYDCYuAY7QgwGg8HENdgRYjAYDCauwY4Qg8FgMHENdoQYDAaDiWuwI8RgMBhMXBN5R1hSUuJ2u5tKeJ7nOK7NghzH8Twf5sNYlkUIhfkwDAaDwbQf0h3hkCFDiKtMmzZNEJrN5jFjxqSmpo4dO9ZsNvsRepk0aVJxcXFTicvlcrlcbSrgdDp9PGiLOBwOiqLaPMxms3k8njYPs1qtDMO0eZjFYmFZts3DTCZTIN4Xg8FgMO2HREeIEDp37lxVVZXdbrfb7atXrxbkK1asuOaaa2pra7t3775y5Uo/wiBhWfbEiRMhqQqDwWAw8YxER1hfX+/xeMaNG9elS5fJkyfbbDZBvmXLlhkzZqhUqhkzZmzevNmPMEi+//77seMnhKQqDAaDwcQzcmnF6urqbr755rfeeqt79+6zZs2aOXPmZ599BgDV1dXXXHMNAAivgMLBLQq9uFyuV1999dNPP/VKnE4nQsj/mGFlZSWPwGaztTkCabPZFApFm+OZwtAoTdNtHsbzvFKpbPMwAFAoFG0eJpfLZTKZVqtt82AMBoPBtAcSHeGAAQP27dsnfF6+fPl1110nfEYIEQQhfPAueGlR+KsGcvnAgQObugG5XA5teRGZTCYc2ab/UCgUAR4mEP7DZDIZSUZ+1RIGg8HEJxId4YkTJyiKGjx4MAAolUqVSiXIs7KyKisrc3Nzq6urs7Oz/Qi9KJXKBx98UKvVeiU8zyOEmkqao1KpCAK0Wq3/wwCAYRiFQtHmYTRNazQatVrt/zC3263Vatt8I3S5XIG85Gk0Gq1WKzh1DAaDwUQEiS8iTqdzwoQJxcXFHo/n9ddfHz9+vCAfM2bMhg0bEEIbNmwYN26cHyEGg8FgMNGAREc4dOjQRYsWjRkzJjs722w2r1ixQpAvXLjw9OnT3bp1O3v27IIFC/wIMRgMBoOJBiQOjRIEMX369OnTp/vIk5OTd+zYEYgwSMxmM+Px2Gw2nU4X2poxGAwGE1fE6hqNl199jaao8vLySCuCwWAwmNgmVh2h0+kEhGOyYDAYDCZYYtURYjAYDAYTErAjxGAwGExcgx0hBoPBYOKa2HaEgWRrwmAwGAzGD7HsCJXajRs3RloJDAaDwcQ2wTrCoqKipjv5JOQjlA4Ry14cg8FgMNFBUL7EarVOnTq1aRLdsOUjxGAwGAwmJEh3hAihqVOnzps3r6kwbPkIMRgMBoMJCRJDrAHAihUrcnJyJk6c2FQoIR+h2+1evXr12rVrvRIhH2Fb7SMAYBjGbrf7P87hcCgUijaX1TgcDpZl20xb6HA4AKDN7BMOh4MkyTazTzgcDqVSKZPJNBqNkHwKg8FgMGFG4s13//79O3fu3L17t4+8eerB5cuXKxSK22+/nSAIhJBMJhs4cGDTIo2Njfv37x86dGjTSgBAqKc1NAqZWkF+8803R48e9a9qILUBAM/zBEEEcpj/3IG5ubkfffQRcRX/tQV4GAaDwWDaD4mOcO/evQcPHvS+GBEEcejQoaFDhzZPPVhZWTlt2rRHHnkkZCpHMSUlJfn5+Xq9nqIovV7f5huhy+XS6/U4HyEGg8FEEImOcOnSpUuXLhU+C696wmch9WB+fn7T1INdu3b1eQvsqOBE8xgMBhNzhPjGjVMPYjAYDCa2CMECjaYLW9oj9SAGg8FgMO0HHsrDYDAYTFyDHSEGg8Fg4hrsCDEYDAYT12BHiMFgMJi4JgLRTN5e885/v9oanrZIgFdfmDl27NjwNIfBYDCYmCMCjvBM8fkTqr7Qf3QY2lIeeK+ysjIMDWEwGAwmRolQfMvMXPjdyDC0Q57+OgytYDAYDCZ2wXOEv8JxXJ8+fZpKdu7c2bdv3+Tk5L59+/5/e/caE1e57gH8XTPMDAMzAwMou9MibikwrWxaMPqBHmugSnKodKQaooE0Ia1JNc2u1lh3sO32WA7Emlg5hqSmlOyIzUFrwhma6j6tW7ECzdZdiaUtt9gqFJBrBzr3y3rPh2XnjHNZMw5zdf6/T/adh2e9wwf/mVmL9zl//ny0NgYAAOETfBB6DYlwzeANv9bW1rKystHRUecKy7J1dXXvvffe0tLSm2++2dDQEMXtAQBAmAQZhL5CIi5m8D7//PPHjx/n/nv37t3vvPMOIaS4uPjw4cOuZXa7vbOzs6KiwmAwSCSS9PT0KOwVAADCLMh7hM6Q0Ov1riHR3d2t1Wq5GbwajaalpSV0Ww2Zp59+urm5+eWXX7ZYLFqt9ujRo4SQ8vJytzKxWFxVVaXX6xUKBcMwfX19gTSnlOr1eoPBIBQK/U6f4CJWKBQmJydjHiEAQFQE+T9fXyHBP4M3RlRUVNTX18/Ozn7zzTclJSUqlYqnWCaT6fX61tbW/fv3f/vtt36b018LsPi3vQEAAAidVT0sw4VEU1PTFy2LqgAADIdJREFU/v37uRXPwbwxSCwWb9++vaenp6urq76+3lfZjz/++OqrrxJCUlNTd+/ePTw8HEhzgUAgl8tlMpk8AM4yfBwEAIiWIIPQV0hwg3kJIc7BvLHpmWeeOX369IULF2pqanzVqFSqU6dOffXVV5TSjz76KEFGKgIAJJogg9BXSHCDeSmlroN5Y9ATTzxx+fLliooKhULhq0YsFnd3dx84cCAzM7Orq+vkyZOR3CEAAERG8PcIuZC4efOmWq3u6Ojg1o8cOVJXV5eTk1NaWtrZ2Rm6fYZYcnLyxo0bPb8Xdbtd99hjj12+fDmC+wIAgEgL/taU15AIZDBvRkaG5PNO0cQ/g7504Ow3viVPFbkt2my2oaGhycnJysrKCOwBAABiWRSe0fjPvx7avLEwUlf790ceecRt6ezZsy+88MKJEyfEYnGktgEAADEqCkEokUh4ntWMgJ07d+7cuTOKGwAAgNiBs0YBACChIQgBACChIQgBACChIQgBACChIQgBACChBR+EWq22qKgoPT1969atY2Nj3GL8ziMEAIDEFOSfT0xMTNTX158/f37z5s1tbW0NDQ39/f3k7jzCTz755JVXXjl27FhLS4tUKv3444+vXbuWlJRkt9sJIW4HTFutVpFIxB3VHSCr1eo8AoZhmNj5c8CFhYVobwEAAH4bJrgZQL29vadPn+aO35yfn9+wYQOXAYWFhVqtVq1Wj4yMaDSa0dFRk8l0//3379mzJzs7e3Z2tr29/fXXX3dt1dLSUlJSsm3bNucKl3MSicTX1d9597+m7854SpWn/cehv/Bs1Ww2BzIa0GQyiUQiv1MgjEYjN0HQV8E999xTW1u7uLioUCj8XnR+fj4jI4PbHk9PAAAInyCD0MnhcOzbt08gELS1tRFCZDLZ/Py8VCo1mUzZ2dkrKyu+Fp3UanVubm5XV5dzxWAwUEplMpmvi26tePzqteuEEMIwmfeuGb/yL54drqysiEQiqVTK/0Z0Op1UKuVJX87S0pJMJvP7GfS3BqFMJvNbDAAA4bCqk2U+//zzgwcPVlZWNjU1cSte5xHyDylMSUlpaWlRKpXOFZFIRCmVy+W+risQ/v+2GYHA9We9FAsEIpEoJSXF79uRSqXJycn8NSzLyuVyv0HocDjS0tL8ZpvNZlMqlfgsCAAQRUEGIaW0sbGxv7+/q6uroKDAuc7NI8zPz3edR+h1MVSsJkNoGwIAQEIJ8qnRgYGB7u7unp4elUql1+v1ej237nUeYRiHFFJ2ZX4mlA0BACDBBBmEvb29o6OjSqVSfhe3fuTIkStXruTk5Fy7du3QoUM8i6GxqvubAAAAq35YZvVKS0vb29tLS0udK3q9nv8eYVrW3SduKCV2C/9bWF5eDuQe4e3btwO5R7i4uBjIPcKFhYVA7hHOzc1lZmbiHiEAQBTF5ckyRqMx2lsAAIDfibgMQgAAgFCJvyC0Wq2sg3VdcT6qAwAA8FvFXxBOTEyw1CUIBUl9fX3R2w4AAMS3+AtCd2LpqVMd0d4EAADEq/gPQtbxj6/7o70JAACIV/EfhJRlmVUdFAcAAIlsVUHocDjUarXritd5hBhSCAAAMSv4IGxtbS0rKxsdHXVd5OYRzszM3HfffceOHeNZBAAAiAXBf6lYXFycl5dXXV3tutjd3a3VaiUSyb59+zQaTUtLi69FJ7vd3tfX9+CDDzpXLBYLpdTX6S1Wq9VthVJqsVh87dNisbAs6/f0FovFIhAI/M4Htlqt3PYCKWNZNpAybh6hQBD/X1MDAMSh4IOwvLzcc3Fqaio3N5cQwn0E5Fl0MpvNH374YX19vXPFaDRSSn0Fg8lk8lw0GHzOoDAajdxcJ/63w13Uc0qU54UYhrHZbH7LApkG7BzzG8iMQwAACIcQP2YSxDxCmUx24sSJjIwM54pYLOY5a/TmzZtuKwzDuP64Gy6Q/J41yjBMIGeNchvzG1osywZy1qjdbucG8/KXAQBA+IT46zhu9CAhxHMeodti0D777LNVdgAAAHAKcRBGYB7h3Nyc24r5zrLdbl9lWwAASEwhDsIIzCO80HvxV/+mrMWw7PbwKgAAQIBWe4/Q7SGU9PT0c+fOudV4XQza8vKK+5JU8ZfGQ2e13aG6BAAAJI44e2R/bm5uZtr9uVPCCPz8mQIAAIAPcRaEs7OzXtctFgvPX1AAAAD4EmdB6J3d9o/er1/Y9+do7wMAAOLP7yIIWTsRMH//3/MLCwvR3goAAMSZ30UQEkIomZ+55Xn6GgAAAL84C8J3333X+wsOG0kSX7p0KbLbAQCAuBdnQfi30//t/QVKCCHT09OR3AwAAPwORCIIQzWP0GazUZ55DozgpYONtbW1er0+6EsAAECiiUQQhmQe4eTk5P1/fIB/hARrt575n7P3/XG9NCV1ZGQkuAsBAEBCYfzOJ1q9wsJCrVarVqtHRkY0Go3bcWibNm06ePBgbW0tIYRl2TNnzmRlZSkUCoPBMDQ0JJVKBwYGvu4fmJi8RQkhDEMoQ5xDAyklhBLCEMItUsIICOsgQpFQKExLTsr+wxq90cjabfnr11dWVl69erWmpmb9+vUXL17U6/W5ubnj4+NlZWVarXbbtm3r1q2TSCQTExMrKytFRUXEZW6Gc3qGTqeTyWRCodBt3flrZBiGYZilpaXl5eWVlZXi4mLXJlyZxWK5evXqc889t7S0lJmZKRQKhUIh5hECAERFJIJQJpPNz89LpVKTyZSdnb2y8qsz0vLz8wkh3HMuFovl3x59dOOGDUqlcmpqanR8XCIWT/88a7dYKCO4GzWEkF/yh/xyc/CXdYYwhFBCnYtUyDAOlkokIrudvSdTuXh7+aGSTWq1+tPP/s4wTFpa+s8/z5SWbB64dGnro4+qVCqBQHB9eNhmtW/a9CdngLkym80ikSgpKYl7lfvtMQzDzeB1hpnZbL5x44bJbC4tKSF3I9D5Izqd7vsrQxe/6jUYDEqlUigUKhQKzCMEAIiKEM8j9Ip/HqFcLm9vb8/KyuL+efPGjXPnzjkcjh07dvC3PXPmjFKpfPzxx/nLPvjgg7y8vC1btvCXvf/++w8//HBpaSl/2fHjx6uqqgoLC/nLmpub33rrrZycHP6yw4cPv/TSS5mZmfxlAAAQPpH4RJifn//pp5/m5+ePj49v3759bGzM9dUnn3xSp9O5Ds69desWpdRvivz0009isXjNmjX8ZT/88INcLr/33nv5y8bGxrKysngG/HKuX7++bt06hULBXzY0NPTAAw+kpqbylw0ODm7cuFEikbzxxhtlZWX8xQAAEA6RCMIDBw5IJJLm5ubGxka73f7222+7vjo/Pz84OBjuPcS4zZs3+41qAAAIh0gEoU6nq6ur+/7770tLSzs7O9PS0sJ9RQAAgABFIggBAABiFh7ZBwCAhIYgBACAhIYgBACAhBajQbhlyxbmrr1793oWBHh+KX8fh8OhVqv9tvUs82yr1WqLiorS09O3bt3q/PsQz25ey/y+WQAACJ9YDEJK6cjIyK1bt+7cuXPnzh2vo5cCOb+Uv09ra2tZWZnbeW+ebT3LPNtOTEzU19efPHlyZmZmx44dDQ0NXrt5LQvkzQIAQBjR2DMzMyOTyR566CGZTKbRaGZnZz1rCgoKhoeHKaXDw8MFBQVB9Pniiy/Onj3r9hvwbOtZ5tn2yy+/3LNnD/fq3NxcZmam125eywJ5swAAED6xGISDg4Pl5eWDg4OLi4u7du169tlnPWtSU1ONRiOl1Gg0yuXyoPu4BaGvtq5lPG3tdvvevXtffPFF/m6uZYFsEgAAwidWgtB5eqfb+vT0tFKp9KxPSUkxmUyUUoPBkJKS4re/rz5uV/TV1tdHZ9e2Fy5cKCkpee2112w2G083zzK/mwQAgPCJxKHbgXAdH/jdd9+ZzWbu7E2xWCyRSDzrVSrV5ORkfn7+1NTU2rVrvfYMpE9I2lJKGxsb+/v7u7q6CgoKfHXzWhbEJgEAIIRi8WEZg8FQU1MzPDxstVqPHj361FNPedZUV1d3dHRQSjs6OjQaTdB9QtJ2YGCgu7u7p6dHpVLp9Xq9Xu+1m9eyIDYJAAChFNXPo96xLNvW1paXl5eVlbVr167l5WXPmtu3b1dVVa1du7a6ulqn0wXdx+034Kuta5ln26amJq+/VbduXssC2SQAAIQPzhoFAICEFotfjQIAAEQMghAAABIaghAAABIaghAAABIaghAAABIaghAAABIaghAAABIaghAAABIaghAAABIaghAAABIaghAAABLa/wG2jEm26H1ojAAAAABJRU5ErkJggg==" + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "Y = convert(Matrix{Float64}, cg10k_trait[:, 3:15])\n", + "histogram(Y, layout = 13)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Pre-processing data for heritability analysis" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To prepare variance component model fitting, we form an instance of `VarianceComponentVariate`. The two variance components are $(2\\Phi, I)$." + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(:Y, :X, :V)" + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "using VarianceComponentModels, LinearAlgebra\n", + "\n", + "# form data as VarianceComponentVariate\n", + "cg10kdata = VarianceComponentVariate(Y, (2Φgrm, Matrix(1.0I, size(Y, 1), size(Y, 1))))\n", + "fieldnames(typeof(cg10kdata))" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "VarianceComponentVariate{Float64,2,Array{Float64,2},Array{Float64,2},Array{Float64,2}}([-1.81573 -0.94615 … -1.02853 -0.394049; -1.2444 0.10966 … 1.09065 0.0256616; … ; 0.886626 0.487408 … -0.636874 -0.439825; -1.24394 0.213697 … 0.299931 0.392809], Array{Float64}(6670,0), ([1.00547 0.00656224 … -0.000121918 -0.00555374; 0.00656224 0.99614 … 0.00176999 0.00683183; … ; -0.000121918 0.00176999 … 0.998578 0.000962983; -0.00555374 0.00683183 … 0.000962983 0.999599], [1.0 0.0 … 0.0 0.0; 0.0 1.0 … 0.0 0.0; … ; 0.0 0.0 … 1.0 0.0; 0.0 0.0 … 0.0 1.0]))" + ] + }, + "execution_count": 25, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cg10kdata" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Before fitting the variance component model, we pre-compute the eigen-decomposition of $2\\Phi_{\\text{GRM}}$, the rotated responses, and the constant part in log-likelihood, and store them as a `TwoVarCompVariateRotate` instance, which is re-used in various variane component estimation procedures." + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " 49.812646 seconds (1.74 M allocations: 1.080 GiB, 0.33% gc time)\n" + ] + }, + { + "data": { + "text/plain": [ + "(:Yrot, :Xrot, :eigval, :eigvec, :logdetV2)" + ] + }, + "execution_count": 26, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# pre-compute eigen-decomposition (~50 secs on my laptop)\n", + "@time cg10kdata_rotated = TwoVarCompVariateRotate(cg10kdata)\n", + "fieldnames(typeof(cg10kdata_rotated))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Save intermediate results\n", + "\n", + "We don't want to re-compute SnpArray and empirical kinship matrices again and again for heritibility analysis." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To load workspace" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "metadata": {}, + "outputs": [ + { + "data": { + "text/latex": [ + "\\begin{tabular}\n", + "{l | r | l}\n", + "name & size & summary \\\\\n", + "\\hline\n", + "Base & & Module \\\\\n", + "Core & & Module \\\\\n", + "Main & & Module \\\\\n", + "Plots & 2.988 MiB & Module \\\\\n", + "PyPlot & 782.799 KiB & Module \\\\\n", + "Y & 677.461 KiB & 6670×13 Array\\{Float64,2\\} \\\\\n", + "cg10k & 1022.983 MiB & 6670×630860 SnpArray \\\\\n", + "cg10k\\_trait & 1.605 MiB & 6670×15 DataFrame \\\\\n", + "cg10kdata & 679.508 MiB & VarianceComponentVariate\\{Float64,2,Array\\{Float64,2\\},Array\\{Float64,2\\},Array\\{Float64,2\\}\\} \\\\\n", + "cg10kdata\\_rotated & 340.136 MiB & TwoVarCompVariateRotate\\{Float64,Array\\{Float64,2\\},Array\\{Float64,2\\}\\} \\\\\n", + "maf\\_cg10k & 4.813 MiB & 630860-element Array\\{Float64,1\\} \\\\\n", + "missings\\_by\\_snp & 4.813 MiB & 1×630860 Array\\{Int64,2\\} \\\\\n", + "mp & 28.748 MiB & 6670×630860 SparseArrays.SparseMatrixCSC\\{Bool,Int32\\} \\\\\n", + "people & 8 bytes & Int64 \\\\\n", + "snps & 8 bytes & Int64 \\\\\n", + "startupfile & 57 bytes & String \\\\\n", + "Φgrm & 339.423 MiB & 6670×6670 Array\\{Float64,2\\} \\\\\n", + "\\end{tabular}\n" + ], + "text/markdown": [ + "| name | size | summary |\n", + "|:----------------- | ------------:|:-------------------------------------------------------------------------------------- |\n", + "| Base | | Module |\n", + "| Core | | Module |\n", + "| Main | | Module |\n", + "| Plots | 2.988 MiB | Module |\n", + "| PyPlot | 782.799 KiB | Module |\n", + "| Y | 677.461 KiB | 6670×13 Array{Float64,2} |\n", + "| cg10k | 1022.983 MiB | 6670×630860 SnpArray |\n", + "| cg10k_trait | 1.605 MiB | 6670×15 DataFrame |\n", + "| cg10kdata | 679.508 MiB | VarianceComponentVariate{Float64,2,Array{Float64,2},Array{Float64,2},Array{Float64,2}} |\n", + "| cg10kdata_rotated | 340.136 MiB | TwoVarCompVariateRotate{Float64,Array{Float64,2},Array{Float64,2}} |\n", + "| maf_cg10k | 4.813 MiB | 630860-element Array{Float64,1} |\n", + "| missings_by_snp | 4.813 MiB | 1×630860 Array{Int64,2} |\n", + "| mp | 28.748 MiB | 6670×630860 SparseArrays.SparseMatrixCSC{Bool,Int32} |\n", + "| people | 8 bytes | Int64 |\n", + "| snps | 8 bytes | Int64 |\n", + "| startupfile | 57 bytes | String |\n", + "| Φgrm | 339.423 MiB | 6670×6670 Array{Float64,2} |\n" + ], + "text/plain": [ + "name size summary \n", + "––––––––––––––––– –––––––––––– ––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––\n", + "Base Module \n", + "Core Module \n", + "Main Module \n", + "Plots 2.988 MiB Module \n", + "PyPlot 782.799 KiB Module \n", + "Y 677.461 KiB 6670×13 Array{Float64,2} \n", + "cg10k 1022.983 MiB 6670×630860 SnpArray \n", + "cg10k_trait 1.605 MiB 6670×15 DataFrame \n", + "cg10kdata 679.508 MiB VarianceComponentVariate{Float64,2,Array{Float64,2},Array{Float64,2},Array{Float64,2}}\n", + "cg10kdata_rotated 340.136 MiB TwoVarCompVariateRotate{Float64,Array{Float64,2},Array{Float64,2}} \n", + "maf_cg10k 4.813 MiB 630860-element Array{Float64,1} \n", + "missings_by_snp 4.813 MiB 1×630860 Array{Int64,2} \n", + "mp 28.748 MiB 6670×630860 SparseArrays.SparseMatrixCSC{Bool,Int32} \n", + "people 8 bytes Int64 \n", + "snps 8 bytes Int64 \n", + "startupfile 57 bytes String \n", + "Φgrm 339.423 MiB 6670×6670 Array{Float64,2} " + ] + }, + "execution_count": 40, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "#pkg\"add FileIO JLD2\"\n", + "\n", + "using JLD2\n", + "@save \"cg10k.jld2\"\n", + "varinfo()" + ] + }, + { + "cell_type": "code", + "execution_count": 45, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "┌ Warning: type SparseArrays.SparseMatrixCSC{Bool,Int32} does not exist in workspace; reconstructing\n", + "└ @ JLD2 /Users/juhyun-kim/.julia/packages/JLD2/KjBIK/src/data.jl:1153\n" + ] + }, + { + "data": { + "text/latex": [ + "\\begin{tabular}\n", + "{l | r | l}\n", + "name & size & summary \\\\\n", + "\\hline\n", + "Base & & Module \\\\\n", + "Core & & Module \\\\\n", + "Main & & Module \\\\\n", + "Plots & 2.988 MiB & Module \\\\\n", + "PyPlot & 782.799 KiB & Module \\\\\n", + "Y & 677.461 KiB & 6670×13 Array\\{Float64,2\\} \\\\\n", + "cg10k & 1022.983 MiB & 6670×630860 SnpArray \\\\\n", + "cg10k\\_trait & 1.605 MiB & 6670×15 DataFrame \\\\\n", + "cg10kdata & 679.508 MiB & VarianceComponentVariate\\{Float64,2,Array\\{Float64,2\\},Array\\{Float64,2\\},Array\\{Float64,2\\}\\} \\\\\n", + "cg10kdata\\_rotated & 340.136 MiB & TwoVarCompVariateRotate\\{Float64,Array\\{Float64,2\\},Array\\{Float64,2\\}\\} \\\\\n", + "maf\\_cg10k & 4.813 MiB & 630860-element Array\\{Float64,1\\} \\\\\n", + "missings\\_by\\_snp & 4.813 MiB & 1×630860 Array\\{Int64,2\\} \\\\\n", + "mp & 28.748 MiB & getfield(JLD2.ReconstructedTypes, Symbol(\"\\#\\#SparseArrays.SparseMatrixCSC\\{Bool,Int32\\}\\#386\")) \\\\\n", + "people & 8 bytes & Int64 \\\\\n", + "snps & 8 bytes & Int64 \\\\\n", + "startupfile & 57 bytes & String \\\\\n", + "Φgrm & 339.423 MiB & 6670×6670 Array\\{Float64,2\\} \\\\\n", + "\\end{tabular}\n" + ], + "text/markdown": [ + "| name | size | summary |\n", + "|:----------------- | ------------:|:------------------------------------------------------------------------------------------- |\n", + "| Base | | Module |\n", + "| Core | | Module |\n", + "| Main | | Module |\n", + "| Plots | 2.988 MiB | Module |\n", + "| PyPlot | 782.799 KiB | Module |\n", + "| Y | 677.461 KiB | 6670×13 Array{Float64,2} |\n", + "| cg10k | 1022.983 MiB | 6670×630860 SnpArray |\n", + "| cg10k_trait | 1.605 MiB | 6670×15 DataFrame |\n", + "| cg10kdata | 679.508 MiB | VarianceComponentVariate{Float64,2,Array{Float64,2},Array{Float64,2},Array{Float64,2}} |\n", + "| cg10kdata_rotated | 340.136 MiB | TwoVarCompVariateRotate{Float64,Array{Float64,2},Array{Float64,2}} |\n", + "| maf_cg10k | 4.813 MiB | 630860-element Array{Float64,1} |\n", + "| missings_by_snp | 4.813 MiB | 1×630860 Array{Int64,2} |\n", + "| mp | 28.748 MiB | getfield(JLD2.ReconstructedTypes, Symbol(\"##SparseArrays.SparseMatrixCSC{Bool,Int32}#386\")) |\n", + "| people | 8 bytes | Int64 |\n", + "| snps | 8 bytes | Int64 |\n", + "| startupfile | 57 bytes | String |\n", + "| Φgrm | 339.423 MiB | 6670×6670 Array{Float64,2} |\n" + ], + "text/plain": [ + "name size summary \n", + "––––––––––––––––– –––––––––––– –––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––\n", + "Base Module \n", + "Core Module \n", + "Main Module \n", + "Plots 2.988 MiB Module \n", + "PyPlot 782.799 KiB Module \n", + "Y 677.461 KiB 6670×13 Array{Float64,2} \n", + "cg10k 1022.983 MiB 6670×630860 SnpArray \n", + "cg10k_trait 1.605 MiB 6670×15 DataFrame \n", + "cg10kdata 679.508 MiB VarianceComponentVariate{Float64,2,Array{Float64,2},Array{Float64,2},Array{Float64,2}} \n", + "cg10kdata_rotated 340.136 MiB TwoVarCompVariateRotate{Float64,Array{Float64,2},Array{Float64,2}} \n", + "maf_cg10k 4.813 MiB 630860-element Array{Float64,1} \n", + "missings_by_snp 4.813 MiB 1×630860 Array{Int64,2} \n", + "mp 28.748 MiB getfield(JLD2.ReconstructedTypes, Symbol(\"##SparseArrays.SparseMatrixCSC{Bool,Int32}#386\"))\n", + "people 8 bytes Int64 \n", + "snps 8 bytes Int64 \n", + "startupfile 57 bytes String \n", + "Φgrm 339.423 MiB 6670×6670 Array{Float64,2} " + ] + }, + "execution_count": 45, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "using SnpArrays, JLD2, DataFrames, VarianceComponentModels, Plots\n", + "pyplot()\n", + "\n", + "@load \"cg10k.jld2\"\n", + "varinfo()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Heritability of single traits" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We use Fisher scoring algorithm to fit variance component model for each single trait." + ] + }, + { + "cell_type": "code", + "execution_count": 46, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Trait1\n", + "\n", + "******************************************************************************\n", + "This program contains Ipopt, a library for large-scale nonlinear optimization.\n", + " Ipopt is released as open source code under the Eclipse Public License (EPL).\n", + " For more information visit http://projects.coin-or.org/Ipopt\n", + "******************************************************************************\n", + "\n", + "(σ2a[trait], σ2e[trait]) = (0.2594760477653529, 0.7375938878381831)\n", + "Trait2\n", + "(σ2a[trait], σ2e[trait]) = (0.18588150448760332, 0.8137880281704619)\n", + "Trait3\n", + "(σ2a[trait], σ2e[trait]) = (0.31960937565352526, 0.6795300238388766)\n", + "Trait4\n", + "(σ2a[trait], σ2e[trait]) = (0.26578244697498915, 0.7304994606945259)\n", + "Trait5\n", + "(σ2a[trait], σ2e[trait]) = (0.28143198005767217, 0.7169747061764987)\n", + "Trait6\n", + "(σ2a[trait], σ2e[trait]) = (0.2830055131241748, 0.7168800753377107)\n", + "Trait7\n", + "(σ2a[trait], σ2e[trait]) = (0.2156542141311619, 0.7816910320891676)\n", + "Trait8\n", + "(σ2a[trait], σ2e[trait]) = (0.19408878271207824, 0.8058201577783562)\n", + "Trait9\n", + "(σ2a[trait], σ2e[trait]) = (0.24746236011763145, 0.7512222977091793)\n", + "Trait10\n", + "(σ2a[trait], σ2e[trait]) = (0.0992417256213392, 0.9007769787053657)\n", + "Trait11\n", + "(σ2a[trait], σ2e[trait]) = (0.1645726648140337, 0.8343110221526228)\n", + "Trait12\n", + "(σ2a[trait], σ2e[trait]) = (0.0822495659186408, 0.9166483378364408)\n", + "Trait13\n", + "(σ2a[trait], σ2e[trait]) = (0.05687679106195183, 0.9424058676223598)\n", + " 6.203222 seconds (12.80 M allocations: 621.516 MiB, 7.13% gc time)\n" + ] + } + ], + "source": [ + "# heritability from single trait analysis\n", + "hST = zeros(13)\n", + "# standard errors of estimated heritability\n", + "hST_se = zeros(13)\n", + "# additive genetic effects\n", + "σ2a = zeros(13)\n", + "# enviromental effects\n", + "σ2e = zeros(13)\n", + "\n", + "@time for trait in 1:13\n", + " println(names(cg10k_trait)[trait + 2])\n", + " # form data set for trait j\n", + " traitj_data = TwoVarCompVariateRotate(cg10kdata_rotated.Yrot[:, trait], cg10kdata_rotated.Xrot, \n", + " cg10kdata_rotated.eigval, cg10kdata_rotated.eigvec, cg10kdata_rotated.logdetV2)\n", + " # initialize model parameters\n", + " traitj_model = VarianceComponentModel(traitj_data)\n", + " # estimate variance components\n", + " _, _, _, Σcov, _, _ = mle_fs!(traitj_model, traitj_data; solver=:Ipopt, verbose=false)\n", + " σ2a[trait] = traitj_model.Σ[1][1]\n", + " σ2e[trait] = traitj_model.Σ[2][1]\n", + " @show σ2a[trait], σ2e[trait]\n", + " h, hse = heritability(traitj_model.Σ, Σcov)\n", + " hST[trait] = h[1]\n", + " hST_se[trait] = hse[1]\n", + "end" + ] + }, + { + "cell_type": "code", + "execution_count": 47, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "2×13 Array{Float64,2}:\n", + " 0.260239 0.185943 0.319885 … 0.164757 0.0823403 0.0569176\n", + " 0.0799434 0.08689 0.0739664 0.0887138 0.0944375 0.0953072" + ] + }, + "execution_count": 47, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# heritability and standard errors\n", + "[hST'; hST_se']" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Pairwise traits\n", + "\n", + "Joint analysis of multiple traits is subject to intensive research recently. Following code snippet does joint analysis of all pairs of traits, a total of 78 bivariate variane component models." + ] + }, + { + "cell_type": "code", + "execution_count": 49, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Trait1Trait2\n", + "(Σa[i, j], Σe[i, j]) = ([0.258529 0.173943; 0.173943 0.184525], [0.738519 0.58639; 0.58639 0.815122])\n", + "Trait1Trait3\n", + "(Σa[i, j], Σe[i, j]) = ([0.259934 -0.0139395; -0.0139395 0.320025], [0.737149 -0.120342; -0.120342 0.67913])\n", + "Trait1Trait4\n", + "(Σa[i, j], Σe[i, j]) = ([0.259307 0.222267; 0.222267 0.265791], [0.737759 0.600077; 0.600077 0.730492])\n", + "Trait1Trait5\n", + "(Σa[i, j], Σe[i, j]) = ([0.259264 -0.147125; -0.147125 0.282102], [0.737798 -0.254669; -0.254669 0.716327])\n", + "Trait1Trait6\n", + "(Σa[i, j], Σe[i, j]) = ([0.259171 -0.129519; -0.129519 0.283267], [0.73789 -0.231373; -0.231373 0.716628])\n", + "Trait1Trait7\n", + "(Σa[i, j], Σe[i, j]) = ([0.258784 -0.140357; -0.140357 0.21535], [0.738269 -0.197894; -0.197894 0.782002])\n", + "Trait1Trait8\n", + "(Σa[i, j], Σe[i, j]) = ([0.259467 -0.0336174; -0.0336174 0.194107], [0.737603 -0.12623; -0.12623 0.805802])\n", + "Trait1Trait9\n", + "(Σa[i, j], Σe[i, j]) = ([0.261524 -0.204467; -0.204467 0.246378], [0.735632 -0.308118; -0.308118 0.752288])\n", + "Trait1Trait10\n", + "(Σa[i, j], Σe[i, j]) = ([0.259326 -0.0987008; -0.0987008 0.0962217], [0.73774 -0.304854; -0.304854 0.90378])\n", + "Trait1Trait11\n", + "(Σa[i, j], Σe[i, j]) = ([0.259194 -0.137578; -0.137578 0.162813], [0.737868 -0.36076; -0.36076 0.836058])\n", + "Trait1Trait12\n", + "(Σa[i, j], Σe[i, j]) = ([0.26168 -0.145305; -0.145305 0.0806851], [0.735514 -0.0421192; -0.0421192 0.918294])\n", + "Trait1Trait13\n", + "(Σa[i, j], Σe[i, j]) = ([0.260776 -0.108277; -0.108277 0.0513909], [0.736359 -0.114756; -0.114756 0.947913])\n", + "Trait2Trait3\n", + "(Σa[i, j], Σe[i, j]) = ([0.185992 0.14505; 0.14505 0.321413], [0.813678 0.0987898; 0.0987898 0.677805])\n", + "Trait2Trait4\n", + "(Σa[i, j], Σe[i, j]) = ([0.18554 0.074367; 0.074367 0.26577], [0.814123 0.22144; 0.22144 0.730512])\n", + "Trait2Trait5\n", + "(Σa[i, j], Σe[i, j]) = ([0.185854 -0.0115021; -0.0115021 0.281443], [0.813815 -0.0368393; -0.0368393 0.716964])\n", + "Trait2Trait6\n", + "(Σa[i, j], Σe[i, j]) = ([0.185903 -0.00350261; -0.00350261 0.283064], [0.813767 -0.0207933; -0.0207933 0.716823])\n", + "Trait2Trait7\n", + "(Σa[i, j], Σe[i, j]) = ([0.185498 -0.0301226; -0.0301226 0.215413], [0.814164 -0.00124563; -0.00124563 0.781929])\n", + "Trait2Trait8\n", + "(Σa[i, j], Σe[i, j]) = ([0.186397 0.032688; 0.032688 0.194625], [0.813285 -0.0321045; -0.0321045 0.805301])\n", + "Trait2Trait9\n", + "(Σa[i, j], Σe[i, j]) = ([0.184426 -0.084907; -0.084907 0.246288], [0.815228 -0.0814978; -0.0814978 0.752373])\n", + "Trait2Trait10\n", + "(Σa[i, j], Σe[i, j]) = ([0.18604 -0.123261; -0.123261 0.0992567], [0.813659 -0.273285; -0.273285 0.90084])\n", + "Trait2Trait11\n", + "(Σa[i, j], Σe[i, j]) = ([0.184894 -0.116422; -0.116422 0.165892], [0.814757 -0.297755; -0.297755 0.833033])\n", + "Trait2Trait12\n", + "(Σa[i, j], Σe[i, j]) = ([0.185297 -0.0908872; -0.0908872 0.0814927], [0.814404 0.0456529; 0.0456529 0.917465])\n", + "Trait2Trait13\n", + "(Σa[i, j], Σe[i, j]) = ([0.185348 -0.07104; -0.07104 0.0546791], [0.814367 0.0740412; 0.0740412 0.944638])\n", + "Trait3Trait4\n", + "(Σa[i, j], Σe[i, j]) = ([0.319575 -0.15468; -0.15468 0.265006], [0.679563 -0.303309; -0.303309 0.731254])\n", + "Trait3Trait5\n", + "(Σa[i, j], Σe[i, j]) = ([0.320016 0.184736; 0.184736 0.282719], [0.679136 0.336276; 0.336276 0.715725])\n", + "Trait3Trait6\n", + "(Σa[i, j], Σe[i, j]) = ([0.320608 0.166929; 0.166929 0.285124], [0.678565 0.297633; 0.297633 0.714823])\n", + "Trait3Trait7\n", + "(Σa[i, j], Σe[i, j]) = ([0.319614 0.167317; 0.167317 0.215458], [0.679526 0.346897; 0.346897 0.781883])\n", + "Trait3Trait8\n", + "(Σa[i, j], Σe[i, j]) = ([0.321532 0.0573397; 0.0573397 0.197221], [0.677674 0.0445311; 0.0445311 0.802757])\n", + "Trait3Trait9\n", + "(Σa[i, j], Σe[i, j]) = ([0.319777 0.138268; 0.138268 0.246547], [0.679369 0.266317; 0.266317 0.752112])\n", + "Trait3Trait10\n", + "(Σa[i, j], Σe[i, j]) = ([0.319905 -0.0779601; -0.0779601 0.100192], [0.679245 -0.141562; -0.141562 0.899839])\n", + "Trait3Trait11\n", + "(Σa[i, j], Σe[i, j]) = ([0.318882 -0.0177352; -0.0177352 0.164446], [0.680233 -0.114434; -0.114434 0.834435])\n", + "Trait3Trait12\n", + "(Σa[i, j], Σe[i, j]) = ([0.321958 0.0844644; 0.0844644 0.0863388], [0.67727 0.034186; 0.034186 0.912603])\n", + "Trait3Trait13\n", + "(Σa[i, j], Σe[i, j]) = ([0.32405 0.109731; 0.109731 0.0613165], [0.675287 -0.00619989; -0.00619989 0.938011])\n", + "Trait4Trait5\n", + "(Σa[i, j], Σe[i, j]) = ([0.26593 -0.216116; -0.216116 0.283173], [0.730347 -0.376697; -0.376697 0.715289])\n", + "Trait4Trait6\n", + "(Σa[i, j], Σe[i, j]) = ([0.266395 -0.200793; -0.200793 0.284547], [0.729899 -0.346915; -0.346915 0.715387])\n", + "Trait4Trait7\n", + "(Σa[i, j], Σe[i, j]) = ([0.264731 -0.182912; -0.182912 0.21441], [0.731526 -0.326257; -0.326257 0.782928])\n", + "Trait4Trait8\n", + "(Σa[i, j], Σe[i, j]) = ([0.266914 -0.0976281; -0.0976281 0.196119], [0.729401 -0.150498; -0.150498 0.803836])\n", + "Trait4Trait9\n", + "(Σa[i, j], Σe[i, j]) = ([0.270267 -0.227463; -0.227463 0.247665], [0.726155 -0.415849; -0.415849 0.751008])\n", + "Trait4Trait10\n", + "(Σa[i, j], Σe[i, j]) = ([0.265756 -0.0339156; -0.0339156 0.0987618], [0.730536 -0.227666; -0.227666 0.901251])\n", + "Trait4Trait11\n", + "(Σa[i, j], Σe[i, j]) = ([0.265858 -0.0963336; -0.0963336 0.163013], [0.730427 -0.273145; -0.273145 0.835846])\n", + "Trait4Trait12\n", + "(Σa[i, j], Σe[i, j]) = ([0.268394 -0.141452; -0.141452 0.0796851], [0.72801 -0.0831954; -0.0831954 0.919263])\n", + "Trait4Trait13\n", + "(Σa[i, j], Σe[i, j]) = ([0.266334 -0.0971798; -0.0971798 0.0541478], [0.729966 -0.226082; -0.226082 0.945138])\n", + "Trait5Trait6\n", + "(Σa[i, j], Σe[i, j]) = ([0.281794 0.281033; 0.281033 0.282364], [0.716628 0.660607; 0.660607 0.717505])\n", + "Trait5Trait7\n", + "(Σa[i, j], Σe[i, j]) = ([0.281023 0.232223; 0.232223 0.211943], [0.717383 0.67449; 0.67449 0.785343])\n", + "Trait5Trait8\n", + "(Σa[i, j], Σe[i, j]) = ([0.281551 0.164011; 0.164011 0.192737], [0.716865 0.221187; 0.221187 0.807144])\n", + "Trait5Trait9\n", + "(Σa[i, j], Σe[i, j]) = ([0.284085 0.244479; 0.244479 0.240901], [0.714415 0.508795; 0.508795 0.757606])\n", + "Trait5Trait10\n", + "(Σa[i, j], Σe[i, j]) = ([0.281993 -0.0460624; -0.0460624 0.100628], [0.716433 -0.0574194; -0.0574194 0.899407])\n", + "Trait5Trait11\n", + "(Σa[i, j], Σe[i, j]) = ([0.280652 0.0199425; 0.0199425 0.163738], [0.717732 -0.0349079; -0.0349079 0.83513])\n", + "Trait5Trait12\n", + "(Σa[i, j], Σe[i, j]) = ([0.281652 0.0612882; 0.0612882 0.0820476], [0.716765 0.0533362; 0.0533362 0.916851])\n", + "Trait5Trait13\n", + "(Σa[i, j], Σe[i, j]) = ([0.282494 0.0696137; 0.0696137 0.0570685], [0.715955 0.0537392; 0.0537392 0.942222])\n", + "Trait6Trait7\n", + "(Σa[i, j], Σe[i, j]) = ([0.283018 0.220794; 0.220794 0.214128], [0.716868 0.58124; 0.58124 0.783191])\n", + "Trait6Trait8\n", + "(Σa[i, j], Σe[i, j]) = ([0.283018 0.183995; 0.183995 0.192405], [0.716869 0.436926; 0.436926 0.807474])\n", + "Trait6Trait9\n", + "(Σa[i, j], Σe[i, j]) = ([0.285057 0.234311; 0.234311 0.24283], [0.714902 0.477263; 0.477263 0.755728])\n", + "Trait6Trait10\n", + "(Σa[i, j], Σe[i, j]) = ([0.283699 -0.0433873; -0.0433873 0.101183], [0.716209 -0.0593852; -0.0593852 0.89886])\n", + "Trait6Trait11\n", + "(Σa[i, j], Σe[i, j]) = ([0.281626 0.027639; 0.027639 0.163178], [0.718219 -0.0520113; -0.0520113 0.83568])\n", + "Trait6Trait12\n", + "(Σa[i, j], Σe[i, j]) = ([0.283196 0.0569533; 0.0569533 0.0819944], [0.716699 0.0481832; 0.0481832 0.916905])\n", + "Trait6Trait13\n", + "(Σa[i, j], Σe[i, j]) = ([0.283862 0.0600111; 0.0600111 0.0571748], [0.716057 0.0544601; 0.0544601 0.942116])\n", + "Trait7Trait8\n", + "(Σa[i, j], Σe[i, j]) = ([0.214129 0.0883281; 0.0883281 0.192399], [0.78319 -0.0565873; -0.0565873 0.80748])\n", + "Trait7Trait9\n", + "(Σa[i, j], Σe[i, j]) = ([0.219059 0.217228; 0.217228 0.243722], [0.778422 0.463004; 0.463004 0.754868])\n", + "Trait7Trait10\n", + "(Σa[i, j], Σe[i, j]) = ([0.216477 -0.042081; -0.042081 0.101258], [0.78089 -0.0859952; -0.0859952 0.898786])\n", + "Trait7Trait11\n", + "(Σa[i, j], Σe[i, j]) = ([0.214308 0.0205244; 0.0205244 0.163203], [0.783006 -0.0479474; -0.0479474 0.835656])\n", + "Trait7Trait12\n", + "(Σa[i, j], Σe[i, j]) = ([0.215192 0.0752708; 0.0752708 0.0802075], [0.782157 0.0353123; 0.0353123 0.918686])\n", + "Trait7Trait13\n", + "(Σa[i, j], Σe[i, j]) = ([0.216198 0.0740251; 0.0740251 0.0547527], [0.781186 0.0399163; 0.0399163 0.944534])\n", + "Trait8Trait9\n", + "(Σa[i, j], Σe[i, j]) = ([0.194551 0.112282; 0.112282 0.246832], [0.805384 0.185461; 0.185461 0.751837])\n", + "Trait8Trait10\n", + "(Σa[i, j], Σe[i, j]) = ([0.194399 -0.0154035; -0.0154035 0.0995883], [0.805519 0.0117323; 0.0117323 0.900435])\n", + "Trait8Trait11\n", + "(Σa[i, j], Σe[i, j]) = ([0.193837 0.022072; 0.022072 0.164393], [0.806067 -0.0267843; -0.0267843 0.834489])\n", + "Trait8Trait12\n", + "(Σa[i, j], Σe[i, j]) = ([0.193914 -0.00259164; -0.00259164 0.0821237], [0.805992 0.0333257; 0.0333257 0.916773])\n", + "Trait8Trait13\n", + "(Σa[i, j], Σe[i, j]) = ([0.193913 0.00330925; 0.00330925 0.0569126], [0.805993 0.0386552; 0.0386552 0.94237])\n", + "Trait9Trait10\n", + "(Σa[i, j], Σe[i, j]) = ([0.246837 -0.00323396; -0.00323396 0.0989732], [0.751834 0.074992; 0.074992 0.901043])\n", + "Trait9Trait11\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(Σa[i, j], Σe[i, j]) = ([0.247395 0.0305136; 0.0305136 0.164575], [0.751288 0.153633; 0.153633 0.834308])\n", + "Trait9Trait12\n", + "(Σa[i, j], Σe[i, j]) = ([0.249928 0.0844216; 0.0844216 0.0880992], [0.748829 0.108018; 0.108018 0.910884])\n", + "Trait9Trait13\n", + "(Σa[i, j], Σe[i, j]) = ([0.248998 0.0925603; 0.0925603 0.0580327], [0.749747 0.0992651; 0.0992651 0.94131])\n", + "Trait10Trait11\n", + "(Σa[i, j], Σe[i, j]) = ([0.0923059 0.0990858; 0.0990858 0.164638], [0.907657 0.475503; 0.475503 0.834248])\n", + "Trait10Trait12\n", + "(Σa[i, j], Σe[i, j]) = ([0.0957712 0.0574496; 0.0574496 0.0785646], [0.904228 0.0843534; 0.0843534 0.920326])\n", + "Trait10Trait13\n", + "(Σa[i, j], Σe[i, j]) = ([0.100109 -0.0266483; -0.0266483 0.0578511], [0.899941 0.164674; 0.164674 0.941451])\n", + "Trait11Trait12\n", + "(Σa[i, j], Σe[i, j]) = ([0.163544 0.0571549; 0.0571549 0.0784378], [0.835325 0.145549; 0.145549 0.920432])\n", + "Trait11Trait13\n", + "(Σa[i, j], Σe[i, j]) = ([0.164571 -0.00169939; -0.00169939 0.0575331], [0.834326 0.200725; 0.200725 0.941755])\n", + "Trait12Trait13\n", + "(Σa[i, j], Σe[i, j]) = ([0.08398 0.0685417; 0.0685417 0.0559416], [0.91494 0.573206; 0.573206 0.943343])\n", + " 4.430316 seconds (4.62 M allocations: 310.482 MiB, 3.51% gc time)\n" + ] + } + ], + "source": [ + "# additive genetic effects (2x2 psd matrices) from bavariate trait analysis;\n", + "Σa = Array{Matrix{Float64}}(undef, 13, 13)\n", + "# environmental effects (2x2 psd matrices) from bavariate trait analysis;\n", + "Σe = Array{Matrix{Float64}}(undef, 13, 13)\n", + "\n", + "\n", + "@time for i in 1:13\n", + " for j in (i+1):13\n", + " println(names(cg10k_trait)[i + 2], names(cg10k_trait)[j + 2])\n", + " # form data set for (trait1, trait2)\n", + " traitij_data = TwoVarCompVariateRotate(cg10kdata_rotated.Yrot[:, [i;j]], cg10kdata_rotated.Xrot, \n", + " cg10kdata_rotated.eigval, cg10kdata_rotated.eigvec, cg10kdata_rotated.logdetV2)\n", + " # initialize model parameters\n", + " traitij_model = VarianceComponentModel(traitij_data)\n", + " # estimate variance components\n", + " mle_fs!(traitij_model, traitij_data; solver=:Ipopt, verbose=false)\n", + " Σa[i, j] = traitij_model.Σ[1]\n", + " Σe[i, j] = traitij_model.Σ[2]\n", + " @show Σa[i, j], Σe[i, j]\n", + " end\n", + "end" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 3-trait analysis\n", + "\n", + "Researchers want to jointly analyze traits 5-7. Our strategy is to try both Fisher scoring and MM algorithm with different starting point, and choose the best local optimum. We first form the data set and run Fisher scoring, which yields a final objective value -1.4700991+04." + ] + }, + { + "cell_type": "code", + "execution_count": 50, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "This is Ipopt version 3.12.10, running with linear solver mumps.\n", + "NOTE: Other linear solvers might be more efficient (see Ipopt documentation).\n", + "\n", + "Number of nonzeros in equality constraint Jacobian...: 0\n", + "Number of nonzeros in inequality constraint Jacobian.: 0\n", + "Number of nonzeros in Lagrangian Hessian.............: 78\n", + "\n", + "Total number of variables............................: 12\n", + " variables with only lower bounds: 0\n", + " variables with lower and upper bounds: 0\n", + " variables with only upper bounds: 0\n", + "Total number of equality constraints.................: 0\n", + "Total number of inequality constraints...............: 0\n", + " inequality constraints with only lower bounds: 0\n", + " inequality constraints with lower and upper bounds: 0\n", + " inequality constraints with only upper bounds: 0\n", + "\n", + "iter objective inf_pr inf_du lg(mu) ||d|| lg(rg) alpha_du alpha_pr ls\n", + " 0 3.0244169e+04 0.00e+00 1.00e+02 0.0 0.00e+00 - 0.00e+00 0.00e+00 0 \n", + " 5 1.6834042e+04 0.00e+00 4.07e+02 -11.0 3.66e-01 - 1.00e+00 1.00e+00f 1 MaxS\n", + " 10 1.4744248e+04 0.00e+00 1.12e+02 -11.0 2.36e-01 - 1.00e+00 1.00e+00f 1 MaxS\n", + " 15 1.4701415e+04 0.00e+00 1.25e+01 -11.0 1.08e-01 -4.5 1.00e+00 1.00e+00f 1 MaxS\n", + " 20 1.4700955e+04 0.00e+00 6.34e-01 -11.0 1.66e-04 -6.9 1.00e+00 1.00e+00f 1 MaxS\n", + " 25 1.4700954e+04 0.00e+00 2.63e-02 -11.0 7.01e-06 -9.2 1.00e+00 1.00e+00f 1 MaxS\n", + " 30 1.4700954e+04 0.00e+00 1.09e-03 -11.0 2.90e-07 -11.6 1.00e+00 1.00e+00f 1 MaxS\n", + " 35 1.4700954e+04 0.00e+00 4.49e-05 -11.0 1.20e-08 -14.0 1.00e+00 1.00e+00f 1 MaxS\n", + " 40 1.4700954e+04 0.00e+00 1.86e-06 -11.0 4.96e-10 -16.4 1.00e+00 1.00e+00f 1 MaxSA\n", + " 45 1.4700954e+04 0.00e+00 7.67e-08 -11.0 2.05e-11 -18.8 1.00e+00 1.00e+00h 1 MaxSA\n", + "iter objective inf_pr inf_du lg(mu) ||d|| lg(rg) alpha_du alpha_pr ls\n", + "\n", + "Number of Iterations....: 49\n", + "\n", + " (scaled) (unscaled)\n", + "Objective...............: 4.4662368766169095e+02 1.4700954216526397e+04\n", + "Dual infeasibility......: 5.8982229663448267e-09 1.9414444012378635e-07\n", + "Constraint violation....: 0.0000000000000000e+00 0.0000000000000000e+00\n", + "Complementarity.........: 0.0000000000000000e+00 0.0000000000000000e+00\n", + "Overall NLP error.......: 5.8982229663448267e-09 1.9414444012378635e-07\n", + "\n", + "\n", + "Number of objective function evaluations = 50\n", + "Number of objective gradient evaluations = 50\n", + "Number of equality constraint evaluations = 0\n", + "Number of inequality constraint evaluations = 0\n", + "Number of equality constraint Jacobian evaluations = 0\n", + "Number of inequality constraint Jacobian evaluations = 0\n", + "Number of Lagrangian Hessian evaluations = 49\n", + "Total CPU secs in IPOPT (w/o function evaluations) = 0.019\n", + "Total CPU secs in NLP function evaluations = 0.054\n", + "\n", + "EXIT: Optimal Solution Found.\n", + " 0.084479 seconds (46.62 k allocations: 5.057 MiB)\n" + ] + }, + { + "data": { + "text/plain": [ + "VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}(Array{Float64}(0,3), ([0.28137 0.280153 0.232512; 0.280153 0.284916 0.220363; 0.232512 0.220363 0.212921], [0.717042 0.661484 0.674207; 0.661484 0.714964 0.581648; 0.674207 0.581648 0.784373]), Array{Float64}(0,0), Char[], Float64[], -Inf, Inf)" + ] + }, + "execution_count": 50, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "traitidx = 5:7\n", + "# form data set\n", + "trait57_data = TwoVarCompVariateRotate(cg10kdata_rotated.Yrot[:, traitidx], cg10kdata_rotated.Xrot, \n", + " cg10kdata_rotated.eigval, cg10kdata_rotated.eigvec, cg10kdata_rotated.logdetV2)\n", + "# initialize model parameters\n", + "trait57_model = VarianceComponentModel(trait57_data)\n", + "# estimate variance components\n", + "@time mle_fs!(trait57_model, trait57_data; solver=:Ipopt, verbose=true)\n", + "trait57_model" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We then run the MM algorithm, starting from the Fisher scoring answer. MM finds an improved solution with objective value 8.955397e+03." + ] + }, + { + "cell_type": "code", + "execution_count": 51, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + " MM Algorithm\n", + " Iter Objective \n", + "-------- -------------\n", + " 0 -1.470095e+04\n", + " 1 -1.470095e+04\n", + "\n", + " 0.505115 seconds (1.01 M allocations: 50.370 MiB, 5.09% gc time)\n" + ] + }, + { + "data": { + "text/plain": [ + "VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}(Array{Float64}(0,3), ([0.28137 0.280153 0.232512; 0.280153 0.284916 0.220363; 0.232512 0.220363 0.212921], [0.717042 0.661484 0.674207; 0.661484 0.714964 0.581648; 0.674207 0.581648 0.784373]), Array{Float64}(0,0), Char[], Float64[], -Inf, Inf)" + ] + }, + "execution_count": 51, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# trait59_model contains the fitted model by Fisher scoring now\n", + "@time mle_mm!(trait57_model, trait57_data; verbose=true)\n", + "trait57_model" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Do another run of MM algorithm from default starting point. It leads to a slightly better local optimum -1.470104e+04, slighly worse than the Fisher scoring result. Follow up anlaysis should use the Fisher scoring result." + ] + }, + { + "cell_type": "code", + "execution_count": 52, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + " MM Algorithm\n", + " Iter Objective \n", + "-------- -------------\n", + " 0 -3.024417e+04\n", + " 1 -2.040172e+04\n", + " 2 -1.656053e+04\n", + " 3 -1.528554e+04\n", + " 4 -1.491023e+04\n", + " 5 -1.480677e+04\n", + " 6 -1.477849e+04\n", + " 7 -1.477005e+04\n", + " 8 -1.476675e+04\n", + " 9 -1.476478e+04\n", + " 10 -1.476318e+04\n", + " 20 -1.475020e+04\n", + " 30 -1.474023e+04\n", + " 40 -1.473255e+04\n", + " 50 -1.472662e+04\n", + " 60 -1.472201e+04\n", + " 70 -1.471839e+04\n", + " 80 -1.471554e+04\n", + " 90 -1.471326e+04\n", + " 100 -1.471142e+04\n", + " 110 -1.470993e+04\n", + " 120 -1.470871e+04\n", + " 130 -1.470770e+04\n", + " 140 -1.470686e+04\n", + " 150 -1.470616e+04\n", + " 160 -1.470557e+04\n", + " 170 -1.470506e+04\n", + " 180 -1.470463e+04\n", + " 190 -1.470426e+04\n", + " 200 -1.470394e+04\n", + " 210 -1.470366e+04\n", + " 220 -1.470342e+04\n", + " 230 -1.470321e+04\n", + " 240 -1.470302e+04\n", + " 250 -1.470285e+04\n", + " 260 -1.470270e+04\n", + " 270 -1.470257e+04\n", + " 280 -1.470245e+04\n", + " 290 -1.470234e+04\n", + " 300 -1.470225e+04\n", + " 310 -1.470216e+04\n", + " 320 -1.470208e+04\n", + " 330 -1.470201e+04\n", + " 340 -1.470195e+04\n", + " 350 -1.470189e+04\n", + " 360 -1.470183e+04\n", + " 370 -1.470178e+04\n", + " 380 -1.470173e+04\n", + " 390 -1.470169e+04\n", + " 400 -1.470165e+04\n", + " 410 -1.470162e+04\n", + " 420 -1.470158e+04\n", + " 430 -1.470155e+04\n", + " 440 -1.470152e+04\n", + " 450 -1.470149e+04\n", + " 460 -1.470147e+04\n", + " 470 -1.470144e+04\n", + " 480 -1.470142e+04\n", + " 490 -1.470140e+04\n", + " 500 -1.470138e+04\n", + " 510 -1.470136e+04\n", + " 520 -1.470134e+04\n", + " 530 -1.470132e+04\n", + " 540 -1.470131e+04\n", + " 550 -1.470129e+04\n", + " 560 -1.470128e+04\n", + " 570 -1.470127e+04\n", + " 580 -1.470125e+04\n", + " 590 -1.470124e+04\n", + " 600 -1.470123e+04\n", + " 610 -1.470122e+04\n", + " 620 -1.470121e+04\n", + " 630 -1.470120e+04\n", + " 640 -1.470119e+04\n", + " 650 -1.470118e+04\n", + " 660 -1.470117e+04\n", + " 670 -1.470116e+04\n", + " 680 -1.470116e+04\n", + " 690 -1.470115e+04\n", + " 700 -1.470114e+04\n", + " 710 -1.470113e+04\n", + " 720 -1.470113e+04\n", + " 730 -1.470112e+04\n", + " 740 -1.470112e+04\n", + " 750 -1.470111e+04\n", + " 760 -1.470110e+04\n", + " 770 -1.470110e+04\n", + " 780 -1.470109e+04\n", + " 790 -1.470109e+04\n", + " 800 -1.470108e+04\n", + " 810 -1.470108e+04\n", + " 820 -1.470108e+04\n", + " 830 -1.470107e+04\n", + " 840 -1.470107e+04\n", + " 850 -1.470106e+04\n", + " 860 -1.470106e+04\n", + " 870 -1.470106e+04\n", + " 880 -1.470105e+04\n", + " 890 -1.470105e+04\n", + " 900 -1.470105e+04\n", + " 910 -1.470104e+04\n", + " 920 -1.470104e+04\n", + " 930 -1.470104e+04\n", + " 940 -1.470103e+04\n", + " 950 -1.470103e+04\n", + " 960 -1.470103e+04\n", + " 970 -1.470103e+04\n", + " 980 -1.470102e+04\n", + " 990 -1.470102e+04\n", + " 1000 -1.470102e+04\n", + " 1010 -1.470102e+04\n", + " 1020 -1.470102e+04\n", + " 1030 -1.470101e+04\n", + " 1040 -1.470101e+04\n", + " 1050 -1.470101e+04\n", + " 1060 -1.470101e+04\n", + " 1070 -1.470101e+04\n", + " 1080 -1.470101e+04\n", + " 1090 -1.470100e+04\n", + " 1100 -1.470100e+04\n", + " 1110 -1.470100e+04\n", + "\n", + " 0.744170 seconds (134.31 k allocations: 13.430 MiB, 1.57% gc time)\n" + ] + }, + { + "data": { + "text/plain": [ + "VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}(Array{Float64}(0,3), ([0.281394 0.280169 0.232564; 0.280169 0.285001 0.220511; 0.232564 0.220511 0.213147], [0.717018 0.661467 0.674155; 0.661467 0.714877 0.581502; 0.674155 0.581502 0.784149]), Array{Float64}(0,0), Char[], Float64[], -Inf, Inf)" + ] + }, + "execution_count": 52, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# default starting point\n", + "trait57_model = VarianceComponentModel(trait57_data)\n", + "@time _, _, _, Σcov, = mle_mm!(trait57_model, trait57_data; verbose=true)\n", + "trait57_model" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Heritability from 3-variate estimate and their standard errors." + ] + }, + { + "cell_type": "code", + "execution_count": 53, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "2×3 Array{Float64,2}:\n", + " 0.281842 0.285036 0.213725 \n", + " 0.0777056 0.0772501 0.0840114" + ] + }, + "execution_count": 53, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "h, hse = heritability(trait57_model.Σ, Σcov)\n", + "[h'; hse']" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 13-trait joint analysis\n", + "\n", + "In some situations, such as studying the genetic covariance, we need to jointly analyze 13 traits. We first try the **Fisher scoring algorithm**." + ] + }, + { + "cell_type": "code", + "execution_count": 54, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "This is Ipopt version 3.12.10, running with linear solver mumps.\n", + "NOTE: Other linear solvers might be more efficient (see Ipopt documentation).\n", + "\n", + "Number of nonzeros in equality constraint Jacobian...: 0\n", + "Number of nonzeros in inequality constraint Jacobian.: 0\n", + "Number of nonzeros in Lagrangian Hessian.............: 16653\n", + "\n", + "Total number of variables............................: 182\n", + " variables with only lower bounds: 0\n", + " variables with lower and upper bounds: 0\n", + " variables with only upper bounds: 0\n", + "Total number of equality constraints.................: 0\n", + "Total number of inequality constraints...............: 0\n", + " inequality constraints with only lower bounds: 0\n", + " inequality constraints with lower and upper bounds: 0\n", + " inequality constraints with only upper bounds: 0\n", + "\n", + "iter objective inf_pr inf_du lg(mu) ||d|| lg(rg) alpha_du alpha_pr ls\n", + " 0 1.3111983e+05 0.00e+00 1.00e+02 0.0 0.00e+00 - 0.00e+00 0.00e+00 0 \n", + " 5 8.2228529e+04 0.00e+00 6.03e+02 -11.0 2.42e+00 - 1.00e+00 1.00e+00f 1 MaxS\n", + " 10 1.2570490e+05 0.00e+00 9.38e+02 -11.0 6.72e+01 -5.4 1.00e+00 1.00e+00h 1 MaxS\n" + ] + }, + { + "ename": "PosDefException", + "evalue": "PosDefException: matrix is not positive definite; Cholesky factorization failed.", + "output_type": "error", + "traceback": [ + "PosDefException: matrix is not positive definite; Cholesky factorization failed.", + "", + "Stacktrace:", + " [1] chkposdef at /Users/osx/buildbot/slave/package_osx64/build/usr/share/julia/stdlib/v1.1/LinearAlgebra/src/lapack.jl:50 [inlined]", + " [2] sygvd!(::Int64, ::Char, ::Char, ::Array{Float64,2}, ::Array{Float64,2}) at /Users/osx/buildbot/slave/package_osx64/build/usr/share/julia/stdlib/v1.1/LinearAlgebra/src/lapack.jl:5075", + " [3] eigen! at /Users/osx/buildbot/slave/package_osx64/build/usr/share/julia/stdlib/v1.1/LinearAlgebra/src/symmetric.jl:646 [inlined]", + " [4] eigen(::Symmetric{Float64,Array{Float64,2}}, ::Symmetric{Float64,Array{Float64,2}}) at /Users/osx/buildbot/slave/package_osx64/build/usr/share/julia/stdlib/v1.1/LinearAlgebra/src/eigen.jl:383", + " [5] TwoVarCompModelRotate(::VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}) at /Users/juhyun-kim/.julia/packages/VarianceComponentModels/Mh6LK/src/VarianceComponentModels.jl:119", + " [6] logpdf at /Users/juhyun-kim/.julia/packages/VarianceComponentModels/Mh6LK/src/two_variance_component.jl:62 [inlined]", + " [7] eval_f(::VarianceComponentModels.TwoVarCompOptProb{VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}},TwoVarCompVariateRotate{Float64,Array{Float64,2},Array{Float64,2}},Array{Float64,2},Array{Float64,1},VarianceComponentAuxData{Array{Float64,2},Array{Float64,1}}}, ::Array{Float64,1}) at /Users/juhyun-kim/.julia/packages/VarianceComponentModels/Mh6LK/src/two_variance_component.jl:731", + " [8] (::getfield(Ipopt, Symbol(\"#eval_f_cb#6\")){VarianceComponentModels.TwoVarCompOptProb{VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}},TwoVarCompVariateRotate{Float64,Array{Float64,2},Array{Float64,2}},Array{Float64,2},Array{Float64,1},VarianceComponentAuxData{Array{Float64,2},Array{Float64,1}}}})(::Array{Float64,1}) at /Users/juhyun-kim/.julia/packages/Ipopt/f6QJl/src/MPB_wrapper.jl:64", + " [9] eval_f_wrapper(::Int32, ::Ptr{Float64}, ::Int32, ::Ptr{Float64}, ::Ptr{Nothing}) at /Users/juhyun-kim/.julia/packages/Ipopt/f6QJl/src/Ipopt.jl:128", + " [10] solveProblem(::Ipopt.IpoptProblem) at /Users/juhyun-kim/.julia/packages/Ipopt/f6QJl/src/Ipopt.jl:346", + " [11] optimize!(::Ipopt.IpoptMathProgModel) at /Users/juhyun-kim/.julia/packages/Ipopt/f6QJl/src/MPB_wrapper.jl:141", + " [12] #mle_fs!#27(::Int64, ::Symbol, ::Symbol, ::Bool, ::Function, ::VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}, ::TwoVarCompVariateRotate{Float64,Array{Float64,2},Array{Float64,2}}) at /Users/juhyun-kim/.julia/packages/VarianceComponentModels/Mh6LK/src/two_variance_component.jl:949", + " [13] (::getfield(VarianceComponentModels, Symbol(\"#kw##mle_fs!\")))(::NamedTuple{(:solver, :verbose),Tuple{Symbol,Bool}}, ::typeof(mle_fs!), ::VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}, ::TwoVarCompVariateRotate{Float64,Array{Float64,2},Array{Float64,2}}) at ./none:0", + " [14] top-level scope at util.jl:156", + " [15] top-level scope at In[54]:3" + ] + } + ], + "source": [ + "# initialize model parameters\n", + "traitall_model = VarianceComponentModel(cg10kdata_rotated)\n", + "# estimate variance components using Fisher scoring algorithm\n", + "@time mle_fs!(traitall_model, cg10kdata_rotated; solver=:Ipopt, verbose=true)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "From the output we can see the Fisher scoring algorithm ran into some numerical issues. Let's try the **MM algorithm**." + ] + }, + { + "cell_type": "code", + "execution_count": 55, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + " MM Algorithm\n", + " Iter Objective \n", + "-------- -------------\n", + " 0 -1.311198e+05\n", + " 1 -8.001595e+04\n", + " 2 -5.806836e+04\n", + " 3 -4.926167e+04\n", + " 4 -4.611163e+04\n", + " 5 -4.511722e+04\n", + " 6 -4.482795e+04\n", + " 7 -4.474405e+04\n", + " 8 -4.471603e+04\n", + " 9 -4.470277e+04\n", + " 10 -4.469345e+04\n", + " 20 -4.462308e+04\n", + " 30 -4.456930e+04\n", + " 40 -4.452802e+04\n", + " 50 -4.449620e+04\n", + " 60 -4.447148e+04\n", + " 70 -4.445209e+04\n", + " 80 -4.443672e+04\n", + " 90 -4.442442e+04\n", + " 100 -4.441448e+04\n", + " 110 -4.440635e+04\n", + " 120 -4.439966e+04\n", + " 130 -4.439409e+04\n", + " 140 -4.438943e+04\n", + " 150 -4.438549e+04\n", + " 160 -4.438215e+04\n", + " 170 -4.437928e+04\n", + " 180 -4.437682e+04\n", + " 190 -4.437469e+04\n", + " 200 -4.437283e+04\n", + " 210 -4.437121e+04\n", + " 220 -4.436978e+04\n", + " 230 -4.436853e+04\n", + " 240 -4.436742e+04\n", + " 250 -4.436643e+04\n", + " 260 -4.436555e+04\n", + " 270 -4.436476e+04\n", + " 280 -4.436406e+04\n", + " 290 -4.436342e+04\n", + " 300 -4.436285e+04\n", + " 310 -4.436234e+04\n", + " 320 -4.436187e+04\n", + " 330 -4.436144e+04\n", + " 340 -4.436106e+04\n", + " 350 -4.436071e+04\n", + " 360 -4.436039e+04\n", + " 370 -4.436009e+04\n", + " 380 -4.435982e+04\n", + " 390 -4.435957e+04\n", + " 400 -4.435935e+04\n", + " 410 -4.435914e+04\n", + " 420 -4.435895e+04\n", + " 430 -4.435877e+04\n", + " 440 -4.435860e+04\n", + " 450 -4.435845e+04\n", + " 460 -4.435831e+04\n", + " 470 -4.435818e+04\n", + " 480 -4.435806e+04\n", + " 490 -4.435794e+04\n", + " 500 -4.435784e+04\n", + " 510 -4.435774e+04\n", + " 520 -4.435765e+04\n", + " 530 -4.435757e+04\n", + " 540 -4.435749e+04\n", + " 550 -4.435741e+04\n", + " 560 -4.435734e+04\n", + " 570 -4.435728e+04\n", + " 580 -4.435722e+04\n", + " 590 -4.435716e+04\n", + " 600 -4.435711e+04\n", + " 610 -4.435706e+04\n", + " 620 -4.435701e+04\n", + " 630 -4.435696e+04\n", + " 640 -4.435692e+04\n", + " 650 -4.435688e+04\n", + " 660 -4.435685e+04\n", + " 670 -4.435681e+04\n", + " 680 -4.435678e+04\n", + " 690 -4.435675e+04\n", + " 700 -4.435672e+04\n", + " 710 -4.435669e+04\n", + " 720 -4.435666e+04\n", + " 730 -4.435664e+04\n", + " 740 -4.435662e+04\n", + " 750 -4.435659e+04\n", + " 760 -4.435657e+04\n", + " 770 -4.435655e+04\n", + " 780 -4.435653e+04\n", + " 790 -4.435652e+04\n", + " 800 -4.435650e+04\n", + " 810 -4.435648e+04\n", + " 820 -4.435647e+04\n", + " 830 -4.435645e+04\n", + " 840 -4.435644e+04\n", + " 850 -4.435643e+04\n", + " 860 -4.435641e+04\n", + " 870 -4.435640e+04\n", + " 880 -4.435639e+04\n", + " 890 -4.435638e+04\n", + " 900 -4.435637e+04\n", + " 910 -4.435636e+04\n", + " 920 -4.435635e+04\n", + " 930 -4.435634e+04\n", + " 940 -4.435633e+04\n", + " 950 -4.435633e+04\n", + " 960 -4.435632e+04\n", + " 970 -4.435631e+04\n", + " 980 -4.435630e+04\n", + " 990 -4.435630e+04\n", + " 1000 -4.435629e+04\n", + " 1010 -4.435628e+04\n", + " 1020 -4.435628e+04\n", + " 1030 -4.435627e+04\n", + " 1040 -4.435627e+04\n", + " 1050 -4.435626e+04\n", + " 1060 -4.435626e+04\n", + " 1070 -4.435625e+04\n", + " 1080 -4.435625e+04\n", + "\n", + " 3.314963 seconds (131.35 k allocations: 65.449 MiB, 0.65% gc time)\n" + ] + }, + { + "data": { + "text/plain": [ + "(-44356.24416259489, VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}(Array{Float64}(0,13), ([0.272112 0.190023 … -0.128464 -0.0976418; 0.190023 0.216814 … -0.0687833 -0.04341; … ; -0.128464 -0.0687833 … 0.116994 0.0900933; -0.0976418 -0.04341 … 0.0900933 0.105876], [0.725183 0.570497 … -0.0589748 -0.125486; 0.570497 0.783023 … 0.0235685 0.0464638; … ; -0.0589748 0.0235685 … 0.882056 0.551829; -0.125486 0.0464638 … 0.551829 0.893642]), Array{Float64}(0,0), Char[], Float64[], -Inf, Inf), ([0.0111603 0.013107 … 0.0128915 0.0127587; 0.0131027 0.0151631 … 0.017153 0.0171359; … ; 0.0128908 0.017153 … 0.0174176 0.018212; 0.0127586 0.0171361 … 0.0182122 0.0188002], [0.0112235 0.0133041 … 0.0130038 0.0127778; 0.0133005 0.0158053 … 0.0178518 0.0177823; … ; 0.0130043 0.0178518 … 0.0179557 0.0187638; 0.0127775 0.0177823 … 0.0187637 0.0193477]), [0.000124552 7.23469e-5 … -3.6584e-7 -1.40474e-5; 7.23585e-5 0.00017168 … -2.04611e-5 -3.18804e-6; … ; -3.70686e-7 -2.04634e-5 … 0.000352082 -1.46096e-5; -1.4039e-5 -3.1795e-6 … -1.46073e-5 0.000374334], Array{Float64}(0,13), Array{Float64}(0,0))" + ] + }, + "execution_count": 55, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# reset model parameters\n", + "traitall_model = VarianceComponentModel(cg10kdata_rotated)\n", + "# estimate variance components using Fisher scoring algorithm\n", + "@time mle_mm!(traitall_model, cg10kdata_rotated; verbose=true)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "It converges after ~1000 iterations." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Save analysis results" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "#using JLD2, FileIO\n", + "#save(\"copd.jld2\")\n", + "#varinfo()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Julia 1.1.0", + "language": "julia", + "name": "julia-1.1" + }, + "language_info": { + "file_extension": ".jl", + "mimetype": "application/julia", + "name": "julia", + "version": "1.1.0" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From b22621c56f5e5d2f46dcaeb8faf641709e5a207a Mon Sep 17 00:00:00 2001 From: Juhyun Kim Date: Wed, 30 Jan 2019 16:20:22 -0800 Subject: [PATCH 52/56] update to v1.1 --- docs/mle_reml1.1.ipynb | 1306 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1306 insertions(+) create mode 100644 docs/mle_reml1.1.ipynb diff --git a/docs/mle_reml1.1.ipynb b/docs/mle_reml1.1.ipynb new file mode 100644 index 0000000..3a904ea --- /dev/null +++ b/docs/mle_reml1.1.ipynb @@ -0,0 +1,1306 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# MLE and REML" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Machine information" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Julia Version 1.1.0\n", + "Commit 80516ca202 (2019-01-21 21:24 UTC)\n", + "Platform Info:\n", + " OS: macOS (x86_64-apple-darwin14.5.0)\n", + " CPU: Intel(R) Core(TM) i5-6267U CPU @ 2.90GHz\n", + " WORD_SIZE: 64\n", + " LIBM: libopenlibm\n", + " LLVM: libLLVM-6.0.1 (ORCJIT, skylake)\n" + ] + } + ], + "source": [ + "versioninfo()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Demo data" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "For demonstration, we generate a random data set." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# generate data from a d-variate response variane component model\n", + "using Random, LinearAlgebra\n", + "Random.seed!(123)\n", + "n = 1000 # no. observations\n", + "d = 2 # dimension of responses\n", + "m = 2 # no. variance components\n", + "p = 2 # no. covariates\n", + "# n-by-p design matrix\n", + "X = randn(n, p)\n", + "# p-by-d mean component regression coefficient\n", + "B = ones(p, d) \n", + "# a tuple of m covariance matrices\n", + "V = ntuple(x -> zeros(n, n), m) \n", + "for i = 1:m-1\n", + " Vi = randn(n, 50)\n", + " copyto!(V[i], Vi * Vi')\n", + "end\n", + "copyto!(V[m], Matrix(I, n, n)) # last covarianec matrix is idendity\n", + "# a tuple of m d-by-d variance component parameters\n", + "Σ = ntuple(x -> zeros(d, d), m) \n", + "for i in 1:m\n", + " Σi = randn(d, d)\n", + " copyto!(Σ[i], Σi' * Σi)\n", + "end\n", + "# form overall nd-by-nd covariance matrix Ω\n", + "Ω = zeros(n * d, n * d)\n", + "for i = 1:m\n", + " Ω += kron(Σ[i], V[i])\n", + "end\n", + "Ωchol = cholesky(Ω)\n", + "# n-by-d responses\n", + "Y = X * B + reshape(Ωchol.L * randn(n*d), n, d);" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Maximum likelihood estimation (MLE)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To find the MLE of parameters $(B,\\Sigma_1,\\ldots,\\Sigma_m)$, we take 3 steps: \n", + "\n", + "**Step 1 (Construct data)**. Construct an instance of `VarianceComponentVariate`, which consists fields \n", + "\n", + "* `Y`: $n$-by-$d$ responses \n", + "* `X`: $n$-by-$p$ covariate matrix \n", + "* `V=(V[1],...,V[m])`: a tuple of $n$-by-$n$ covariance matrices. The last covariance matrix must be positive definite and usually is the identity matrix. " + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(:Y, :X, :V)" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "using VarianceComponentModels\n", + "vcdata = VarianceComponentVariate(Y, X, V)\n", + "fieldnames(typeof(vcdata))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In the absence of covariates $X$, we can simply initialize by `vcdata = VarianceComponentVariate(Y, V)`." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Step 2 (Construct a model)**. Construct an instance of `VarianceComponentModel`, which consists of fields \n", + "\n", + "* `B`: $n$-by-$p$ mean regression coefficients \n", + "* `Σ=(Σ[1],...,Σ[m])`: variane component parameters respectively. \n", + "\n", + "When constructed from a `VarianceComponentVariate` instance, the mean parameters $B$ are initialized to be zero and the tuple of variance component parameters $\\Sigma$ to be `(eye(d),...,eye(d))`." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(:B, :Σ, :A, :sense, :b, :lb, :ub)" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "vcmodel = VarianceComponentModel(vcdata)\n", + "fieldnames(typeof(vcmodel))" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}([0.0 0.0; 0.0 0.0], ([1.0 0.0; 0.0 1.0], [1.0 0.0; 0.0 1.0]), Array{Float64}(0,4), Char[], Float64[], -Inf, Inf)" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "vcmodel" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The remaining fields `A`, `sense`, `b`, `lb`, `ub` specify (optional) constraints on the mean parameters `B`:\n", + "\n", + "$A * \\text{vec}(B) \\,\\, =(\\text{or } \\ge \\text{or } \\le) \\,\\, b$\n", + "\n", + "$lb \\le \\text{vec}(B) \\le ub$\n", + "\n", + "`A` is an constraint matrix with $pd$ columns, `sense` is a vector of charaters taking values `'<'`, `'='` or `'>'`, and `lb` and `ub` are the lower and upper bounds for `vec(B)`. By default, `A`, `sense`, `b` are empty, `lb` is `-Inf`, and `ub` is `Inf`. If any constraits are non-trivial, final estimates of `B` are enforced to satisfy them." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "When a better initial guess is available, we can initialize by calling `vcmodel=VarianceComponentModel(B0, Σ0)` directly." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Step 3 (Fit model)**. Call optmization routine `fit_mle!`. The keywork `algo` dictates the optimization algorithm: `:MM` (minorization-maximization algorithm) or `:FS` (Fisher scoring algorithm)." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + " MM Algorithm\n", + " Iter Objective \n", + "-------- -------------\n", + " 0 -6.253551e+03\n", + "\n", + "******************************************************************************\n", + "This program contains Ipopt, a library for large-scale nonlinear optimization.\n", + " Ipopt is released as open source code under the Eclipse Public License (EPL).\n", + " For more information visit http://projects.coin-or.org/Ipopt\n", + "******************************************************************************\n", + "\n", + " 1 -3.881454e+03\n", + " 2 -3.853179e+03\n", + " 3 -3.846525e+03\n", + " 4 -3.844906e+03\n", + " 5 -3.844506e+03\n", + " 6 -3.844406e+03\n", + " 7 -3.844381e+03\n", + " 8 -3.844375e+03\n", + " 9 -3.844374e+03\n", + " 10 -3.844373e+03\n", + "\n", + " 5.031460 seconds (11.29 M allocations: 568.015 MiB, 4.78% gc time)\n" + ] + } + ], + "source": [ + "vcmodel_mle = deepcopy(vcmodel)\n", + "@time logl, vcmodel_mle, Σse, Σcov, Bse, Bcov = fit_mle!(vcmodel_mle, vcdata; algo = :MM);" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The output of `fit_mle!` contains \n", + "\n", + "* final log-likelihood " + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "-3844.3731814180887" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "logl" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "* fitted model" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(:B, :Σ, :A, :sense, :b, :lb, :ub)" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "fieldnames(typeof(vcmodel_mle))" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}([1.092 1.04727; 0.955346 1.01632], ([0.380637 -0.305465; -0.305465 4.51938], [1.84009 0.265569; 0.265569 2.17275]), Array{Float64}(0,4), Char[], Float64[], -Inf, Inf)" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "vcmodel_mle" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "* standard errors of the estimated varianec component parameters" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "([0.0765136 0.263047; 0.263047 0.904332], [0.0844292 0.0917441; 0.0917441 0.0996927])" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "Σse" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "* covariance matrix of the variance component parameters estimates" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "8×8 Array{Float64,2}:\n", + " 0.00585433 -0.00467019 -0.00467019 … -1.07903e-6 -1.557e-7 \n", + " -0.00467019 0.0691937 0.00372555 -1.557e-7 -1.27444e-6 \n", + " -0.00467019 0.00372555 0.0691937 -8.83212e-6 -1.27444e-6 \n", + " 0.00372555 -0.055198 -0.055198 -1.27444e-6 -1.04316e-5 \n", + " -7.4779e-6 -1.07903e-6 -1.07903e-6 0.00102878 0.000148477\n", + " -1.07903e-6 -8.83212e-6 -1.557e-7 … 0.000148477 0.00121477 \n", + " -1.07903e-6 -1.557e-7 -8.83212e-6 0.00841698 0.00121477 \n", + " -1.557e-7 -1.27444e-6 -1.27444e-6 0.00121477 0.00993864 " + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "Σcov" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "* standard errors of the estimated mean parameters" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "2×2 Array{Float64,2}:\n", + " 0.0425562 0.0483834\n", + " 0.0430596 0.0492809" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "Bse" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "* covariance matrix of the mean parameter estimates" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "4×4 Array{Float64,2}:\n", + " 0.00181103 -1.96485e-5 0.000243441 -4.38252e-6 \n", + " -1.96485e-5 0.00185413 -4.38252e-6 0.000246407\n", + " 0.000243441 -4.38252e-6 0.00234096 -5.73331e-6 \n", + " -4.38252e-6 0.000246407 -5.73331e-6 0.00242861 " + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "Bcov" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Restricted maximum likelihood estimation (REML)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "[REML (restricted maximum likelihood estimation)](https://en.wikipedia.org/wiki/Restricted_maximum_likelihood) is a popular alternative to the MLE. To find the REML of a variane component model, we replace the above step 3 by " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Step 3**. Call optmization routine `fit_reml!`." + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + " MM Algorithm\n", + " Iter Objective \n", + "-------- -------------\n", + " 0 -4.215053e+03\n", + " 1 -3.925799e+03\n", + " 2 -3.865114e+03\n", + " 3 -3.851105e+03\n", + " 4 -3.847732e+03\n", + " 5 -3.846903e+03\n", + " 6 -3.846698e+03\n", + " 7 -3.846647e+03\n", + " 8 -3.846634e+03\n", + " 9 -3.846631e+03\n", + " 10 -3.846630e+03\n", + "\n", + " 0.726373 seconds (388.90 k allocations: 82.673 MiB, 13.22% gc time)\n" + ] + } + ], + "source": [ + "vcmodel_reml = deepcopy(vcmodel)\n", + "@time logl, vcmodel_reml, Σse, Σcov, Bse, Bcov = fit_reml!(vcmodel_reml, vcdata; algo = :MM);" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The output of `fit_reml!` contains" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "* the final log-likelihood at REML estimate" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "-3844.3777179025055" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "logl" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "* REML estimates" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(:B, :Σ, :A, :sense, :b, :lb, :ub)" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "fieldnames(typeof(vcmodel_reml))" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}([1.092 1.04727; 0.955345 1.01632], ([0.380594 -0.305485; -0.305485 4.51994], [1.84285 0.261963; 0.261963 2.17842]), Array{Float64}(0,4), Char[], Float64[], -Inf, Inf)" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "vcmodel_reml" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "* standard errors of the estimated variance component parameters" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "([0.0765055 0.26305; 0.26305 0.904446], [0.0845559 0.0919325; 0.0919325 0.0999526])" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "Σse" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "* covariance matrix of the variance component parameters estimates" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "8×8 Array{Float64,2}:\n", + " 0.0058531 -0.00467005 -0.00467005 … -1.06597e-6 -1.51499e-7 \n", + " -0.00467005 0.0691951 0.00372613 -1.51499e-7 -1.26041e-6 \n", + " -0.00467005 0.00372613 0.0691951 -8.86843e-6 -1.26041e-6 \n", + " 0.00372613 -0.0552092 -0.0552092 -1.26041e-6 -1.0486e-5 \n", + " -7.50035e-6 -1.06597e-6 -1.06597e-6 0.00101633 0.000144472\n", + " -1.06597e-6 -8.86843e-6 -1.51499e-7 … 0.000144472 0.0012014 \n", + " -1.06597e-6 -1.51499e-7 -8.86843e-6 0.00845158 0.0012014 \n", + " -1.51499e-7 -1.26041e-6 -1.26041e-6 0.0012014 0.00999052 " + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "Σcov" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "* standard errors of the estimated mean parameters" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "2×2 Array{Float64,2}:\n", + " 0.0425881 0.0484485\n", + " 0.0430919 0.0493475" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "Bse" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "* covariance matrix of the mean parameter estimates" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "4×4 Array{Float64,2}:\n", + " 0.00181375 -1.96783e-5 0.000239868 -4.34611e-6 \n", + " -1.96783e-5 0.00185691 -4.34611e-6 0.000242745\n", + " 0.000239868 -4.34611e-6 0.00234726 -5.73082e-6 \n", + " -4.34611e-6 0.000242745 -5.73082e-6 0.00243518 " + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "Bcov" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Optimization algorithms" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Finding the MLE or REML of variance component models is a non-trivial nonlinear optimization problem. The main complications are the non-convexity of objective function and the positive semi-definiteness constraint of variane component parameters $\\Sigma_1,\\ldots,\\Sigma_m$. In specific applications, users should try different algorithms with different starting points in order to find a better solution. Here are some tips for efficient computation. " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In general the optimization algorithm needs to invert the $nd$ by $nd$ overall covariance matrix $\\Omega = \\Sigma_1 \\otimes V_1 + \\cdots + \\Sigma_m \\otimes V_m$ in each iteration. Inverting a matrix is an expensive operation with $O(n^3 d^3)$ floating operations. When there are only **two** varianec components ($m=2$), this tedious task can be avoided by taking one (generalized) eigendecomposion of $(V_1, V_2)$ and rotating data $(Y, X)$ by the eigen-vectors. " + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(:Yrot, :Xrot, :eigval, :eigvec, :logdetV2)" + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "vcdatarot = TwoVarCompVariateRotate(vcdata)\n", + "fieldnames(typeof(vcdatarot))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Two optimization algorithms are implemented: [Fisher scoring](https://books.google.com/books?id=QYqeYTftPNwC&lpg=PP1&pg=PA142#v=onepage&q&f=false) (`mle_fs!`) and the [minorization-maximization (MM) algorithm](http://arxiv.org/abs/1509.07426) (`mle_mm!`). Both take the rotated data as input. These two functions give finer control of the optimization algorithms. Generally speaking, MM algorithm is more stable while Fisher scoring (if it converges) yields more accurate answer." + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + " MM Algorithm\n", + " Iter Objective \n", + "-------- -------------\n", + " 0 -6.253551e+03\n", + " 1 -3.881454e+03\n", + " 2 -3.853179e+03\n", + " 3 -3.846525e+03\n", + " 4 -3.844906e+03\n", + " 5 -3.844506e+03\n", + " 6 -3.844406e+03\n", + " 7 -3.844381e+03\n", + " 8 -3.844375e+03\n", + " 9 -3.844374e+03\n", + " 10 -3.844373e+03\n", + "\n", + " 0.055578 seconds (21.91 k allocations: 1.394 MiB)\n" + ] + } + ], + "source": [ + "vcmodel_mm = deepcopy(vcmodel)\n", + "@time mle_mm!(vcmodel_mm, vcdatarot; maxiter=10000, funtol=1e-8, verbose = true);" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "2×2 Array{Float64,2}:\n", + " 1.092 1.04727\n", + " 0.955346 1.01632" + ] + }, + "execution_count": 26, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# MM estimates\n", + "vcmodel_mm.B" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "([0.380637 -0.305465; -0.305465 4.51938], [1.84009 0.265569; 0.265569 2.17275])" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# MM estimates\n", + "vcmodel_mm.Σ" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Fisher scoring (`mle_fs!`) uses either [Ipopt.jl](https://github.com/JuliaOpt/Ipopt.jl) (keyword `solver=:Ipopt`) or [KNITRO.jl](https://github.com/JuliaOpt/KNITRO.jl) (keyword `solver=:Knitro`) as the backend solver. Ipopt is open source and installation of [Ipopt.jl](https://github.com/JuliaOpt/Ipopt.jl) package alone is sufficient." + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "This is Ipopt version 3.12.10, running with linear solver mumps.\n", + "NOTE: Other linear solvers might be more efficient (see Ipopt documentation).\n", + "\n", + "Number of nonzeros in equality constraint Jacobian...: 0\n", + "Number of nonzeros in inequality constraint Jacobian.: 0\n", + "Number of nonzeros in Lagrangian Hessian.............: 21\n", + "\n", + "Total number of variables............................: 6\n", + " variables with only lower bounds: 0\n", + " variables with lower and upper bounds: 0\n", + " variables with only upper bounds: 0\n", + "Total number of equality constraints.................: 0\n", + "Total number of inequality constraints...............: 0\n", + " inequality constraints with only lower bounds: 0\n", + " inequality constraints with lower and upper bounds: 0\n", + " inequality constraints with only upper bounds: 0\n", + "\n", + "iter objective inf_pr inf_du lg(mu) ||d|| lg(rg) alpha_du alpha_pr ls\n", + " 0 4.2109423e+03 0.00e+00 1.00e+02 0.0 0.00e+00 - 0.00e+00 0.00e+00 0 \n", + " 5 3.8445586e+03 0.00e+00 7.87e-01 -11.0 4.94e-02 - 1.00e+00 1.00e+00f 1 MaxS\n", + " 10 3.8443870e+03 0.00e+00 2.25e-01 -11.0 1.38e-02 - 1.00e+00 1.00e+00f 1 MaxS\n", + " 15 3.8443742e+03 0.00e+00 6.23e-02 -11.0 3.78e-03 - 1.00e+00 1.00e+00f 1 MaxS\n", + " 20 3.8443733e+03 0.00e+00 1.70e-02 -11.0 1.03e-03 - 1.00e+00 1.00e+00f 1 MaxS\n", + " 25 3.8443732e+03 0.00e+00 4.61e-03 -11.0 2.79e-04 - 1.00e+00 1.00e+00f 1 MaxS\n", + " 30 3.8443732e+03 0.00e+00 1.25e-03 -11.0 7.56e-05 - 1.00e+00 1.00e+00f 1 MaxS\n", + " 35 3.8443732e+03 0.00e+00 3.39e-04 -11.0 2.05e-05 - 1.00e+00 1.00e+00f 1 MaxS\n", + " 40 3.8443732e+03 0.00e+00 9.19e-05 -11.0 5.55e-06 - 1.00e+00 1.00e+00f 1 MaxS\n", + " 45 3.8443732e+03 0.00e+00 2.49e-05 -11.0 1.51e-06 - 1.00e+00 1.00e+00f 1 MaxS\n", + "iter objective inf_pr inf_du lg(mu) ||d|| lg(rg) alpha_du alpha_pr ls\n", + " 50 3.8443732e+03 0.00e+00 6.76e-06 -11.0 4.08e-07 - 1.00e+00 1.00e+00f 1 MaxSA\n", + " 55 3.8443732e+03 0.00e+00 1.83e-06 -11.0 1.11e-07 - 1.00e+00 1.00e+00f 1 MaxSA\n", + " 60 3.8443732e+03 0.00e+00 4.97e-07 -11.0 3.00e-08 - 1.00e+00 1.00e+00h 1 MaxSA\n", + "\n", + "Number of Iterations....: 63\n", + "\n", + " (scaled) (unscaled)\n", + "Objective...............: 3.4496886481728791e+02 3.8443731733053728e+03\n", + "Dual infeasibility......: 2.2693631660531264e-07 2.5290047206674095e-06\n", + "Constraint violation....: 0.0000000000000000e+00 0.0000000000000000e+00\n", + "Complementarity.........: 0.0000000000000000e+00 0.0000000000000000e+00\n", + "Overall NLP error.......: 2.2693631660531264e-07 2.5290047206674095e-06\n", + "\n", + "\n", + "Number of objective function evaluations = 64\n", + "Number of objective gradient evaluations = 64\n", + "Number of equality constraint evaluations = 0\n", + "Number of inequality constraint evaluations = 0\n", + "Number of equality constraint Jacobian evaluations = 0\n", + "Number of inequality constraint Jacobian evaluations = 0\n", + "Number of Lagrangian Hessian evaluations = 63\n", + "Total CPU secs in IPOPT (w/o function evaluations) = 1.739\n", + "Total CPU secs in NLP function evaluations = 0.293\n", + "\n", + "EXIT: Solved To Acceptable Level.\n", + " 2.745554 seconds (4.30 M allocations: 210.935 MiB, 2.63% gc time)\n" + ] + } + ], + "source": [ + "# Fisher scoring using Ipopt\n", + "vcmodel_ipopt = deepcopy(vcmodel)\n", + "@time mle_fs!(vcmodel_ipopt, vcdatarot; solver=:Ipopt, maxiter=1000, verbose=true);" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "2×2 Array{Float64,2}:\n", + " 1.092 1.04727\n", + " 0.955346 1.01632" + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Ipopt estimates\n", + "vcmodel_ipopt.B" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "([0.380552 -0.305594; -0.305594 4.52106], [1.84008 0.265385; 0.265385 2.17287])" + ] + }, + "execution_count": 30, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Ipopt estimates\n", + "vcmodel_ipopt.Σ" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Knitro is a commercial software and users need to follow instructions at [KNITRO.jl](https://github.com/JuliaOpt/KNITRO.jl) for proper functioning. Following code invokes Knitro as the backend optimization solver.\n", + "```julia\n", + "using KNITRO\n", + "\n", + "# Fisher scoring using Knitro\n", + "vcmodel_knitro = deepcopy(vcmodel)\n", + "@time mle_fs!(vcmodel_knitro, vcdatarot; solver=:Knitro, maxiter=1000, verbose=true);\n", + "\n", + "# Knitro estimates\n", + "vcmodel_knitro.B\n", + "\n", + "# Knitro estimates\n", + "vcmodel_knitro.Σ\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Starting point\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Here are a few strategies for successful optimization. " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "* For $d>1$ (multivariate response), initialize $B, \\Sigma$ from univariate estimates. \n", + "* Use REML estimate as starting point for MLE. \n", + "* When there are only $m=2$ variance components, pre-compute `TwoVarCompVariateRotate` and use it for optimization." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Constrained estimation of `B`\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Many applications invoke constraints on the mean parameters `B`. For demonstration, we enforce `B[1,1]=B[1,2]` and all entries of `B` are within [0, 2]." + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}([0.0 0.0; 0.0 0.0], ([1.0 0.0; 0.0 1.0], [1.0 0.0; 0.0 1.0]), [1.0 0.0 -1.0 0.0], '=', 0.0, 0.0, 2.0)" + ] + }, + "execution_count": 31, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# set up constraints on B\n", + "vcmodel_constr = deepcopy(vcmodel)\n", + "vcmodel_constr.A = [1.0 0.0 -1.0 0.0]\n", + "vcmodel_constr.sense = '='\n", + "vcmodel_constr.b = 0.0\n", + "vcmodel_constr.lb = 0.0\n", + "vcmodel_constr.ub = 2.0\n", + "vcmodel_constr" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We first try the MM algorithm." + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + " MM Algorithm\n", + " Iter Objective \n", + "-------- -------------\n", + " 0 -6.253551e+03\n", + " 1 -3.881820e+03\n", + " 2 -3.853477e+03\n", + " 3 -3.846807e+03\n", + " 4 -3.845184e+03\n", + " 5 -3.844783e+03\n", + " 6 -3.844683e+03\n", + " 7 -3.844658e+03\n", + " 8 -3.844652e+03\n", + " 9 -3.844650e+03\n", + " 10 -3.844650e+03\n", + "\n", + " 0.185885 seconds (179.51 k allocations: 9.295 MiB)\n" + ] + } + ], + "source": [ + "# MM algorithm for constrained estimation of B\n", + "@time mle_mm!(vcmodel_constr, vcdatarot; maxiter=10000, funtol=1e-8, verbose = true);" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(:B, :Σ, :A, :sense, :b, :lb, :ub)" + ] + }, + "execution_count": 33, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "fieldnames(typeof(vcmodel_constr))" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "2×2 Array{Float64,2}:\n", + " 1.07177 1.07177\n", + " 0.955683 1.01591" + ] + }, + "execution_count": 34, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "vcmodel_constr.B" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "([0.380624 -0.305498; -0.305498 4.51948], [1.84051 0.265065; 0.265065 2.17336])" + ] + }, + "execution_count": 35, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "vcmodel_constr.Σ" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now let's try Fisher scoring." + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "This is Ipopt version 3.12.10, running with linear solver mumps.\n", + "NOTE: Other linear solvers might be more efficient (see Ipopt documentation).\n", + "\n", + "Number of nonzeros in equality constraint Jacobian...: 0\n", + "Number of nonzeros in inequality constraint Jacobian.: 0\n", + "Number of nonzeros in Lagrangian Hessian.............: 21\n", + "\n", + "Total number of variables............................: 6\n", + " variables with only lower bounds: 0\n", + " variables with lower and upper bounds: 0\n", + " variables with only upper bounds: 0\n", + "Total number of equality constraints.................: 0\n", + "Total number of inequality constraints...............: 0\n", + " inequality constraints with only lower bounds: 0\n", + " inequality constraints with lower and upper bounds: 0\n", + " inequality constraints with only upper bounds: 0\n", + "\n", + "iter objective inf_pr inf_du lg(mu) ||d|| lg(rg) alpha_du alpha_pr ls\n", + " 0 4.2114270e+03 0.00e+00 1.00e+02 0.0 0.00e+00 - 0.00e+00 0.00e+00 0 \n", + " 5 3.8448353e+03 0.00e+00 7.87e-01 -11.0 4.94e-02 - 1.00e+00 1.00e+00f 1 MaxS\n", + " 10 3.8446636e+03 0.00e+00 2.25e-01 -11.0 1.38e-02 - 1.00e+00 1.00e+00f 1 MaxS\n", + " 15 3.8446509e+03 0.00e+00 6.23e-02 -11.0 3.78e-03 - 1.00e+00 1.00e+00f 1 MaxS\n", + " 20 3.8446499e+03 0.00e+00 1.70e-02 -11.0 1.03e-03 - 1.00e+00 1.00e+00f 1 MaxS\n", + " 25 3.8446498e+03 0.00e+00 4.61e-03 -11.0 2.79e-04 - 1.00e+00 1.00e+00f 1 MaxS\n", + " 30 3.8446498e+03 0.00e+00 1.25e-03 -11.0 7.56e-05 - 1.00e+00 1.00e+00f 1 MaxS\n", + " 35 3.8446498e+03 0.00e+00 3.39e-04 -11.0 2.05e-05 - 1.00e+00 1.00e+00f 1 MaxS\n", + " 40 3.8446498e+03 0.00e+00 9.19e-05 -11.0 5.56e-06 - 1.00e+00 1.00e+00f 1 MaxS\n", + " 45 3.8446498e+03 0.00e+00 2.49e-05 -11.0 1.51e-06 - 1.00e+00 1.00e+00f 1 MaxS\n", + "iter objective inf_pr inf_du lg(mu) ||d|| lg(rg) alpha_du alpha_pr ls\n", + " 50 3.8446498e+03 0.00e+00 6.76e-06 -11.0 4.08e-07 - 1.00e+00 1.00e+00f 1 MaxSA\n", + " 55 3.8446498e+03 0.00e+00 1.83e-06 -11.0 1.11e-07 - 1.00e+00 1.00e+00f 1 MaxSA\n", + " 60 3.8446498e+03 0.00e+00 4.97e-07 -11.0 3.00e-08 - 1.00e+00 1.00e+00f 1 MaxSA\n", + "\n", + "Number of Iterations....: 63\n", + "\n", + " (scaled) (unscaled)\n", + "Objective...............: 3.4484507551949685e+02 3.8446498170293398e+03\n", + "Dual infeasibility......: 2.2694405475622814e-07 2.5301808856629548e-06\n", + "Constraint violation....: 0.0000000000000000e+00 0.0000000000000000e+00\n", + "Complementarity.........: 0.0000000000000000e+00 0.0000000000000000e+00\n", + "Overall NLP error.......: 2.2694405475622814e-07 2.5301808856629548e-06\n", + "\n", + "\n", + "Number of objective function evaluations = 64\n", + "Number of objective gradient evaluations = 64\n", + "Number of equality constraint evaluations = 0\n", + "Number of inequality constraint evaluations = 0\n", + "Number of equality constraint Jacobian evaluations = 0\n", + "Number of inequality constraint Jacobian evaluations = 0\n", + "Number of Lagrangian Hessian evaluations = 63\n", + "Total CPU secs in IPOPT (w/o function evaluations) = 0.028\n", + "Total CPU secs in NLP function evaluations = 0.634\n", + "\n", + "EXIT: Solved To Acceptable Level.\n", + " 0.760983 seconds (102.63 k allocations: 8.135 MiB)\n" + ] + } + ], + "source": [ + "# Fisher scoring using Ipopt for constrained estimation of B\n", + "vcmodel_constr = deepcopy(vcmodel)\n", + "vcmodel_constr.A = [1.0 0.0 -1.0 0.0]\n", + "vcmodel_constr.sense = '='\n", + "vcmodel_constr.b = 0.0\n", + "vcmodel_constr.lb = 0.0\n", + "vcmodel_constr.ub = 2.0\n", + "vcmodel_constr\n", + "@time mle_fs!(vcmodel_constr, vcdatarot; solver=:Ipopt, maxiter=1000, verbose=true);" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "2×2 Array{Float64,2}:\n", + " 1.07177 1.07177\n", + " 0.955683 1.01591" + ] + }, + "execution_count": 37, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "vcmodel_constr.B" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "([0.380539 -0.305626; -0.305626 4.52116], [1.8405 0.264881; 0.264881 2.17348])" + ] + }, + "execution_count": 38, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "vcmodel_constr.Σ" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Julia 1.1.0", + "language": "julia", + "name": "julia-1.1" + }, + "language_info": { + "file_extension": ".jl", + "mimetype": "application/julia", + "name": "julia", + "version": "1.1.0" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From c94017aba2eede3f42d424dfe30e67ffee66c9cd Mon Sep 17 00:00:00 2001 From: Juhyun Kim Date: Wed, 30 Jan 2019 16:21:15 -0800 Subject: [PATCH 53/56] rename figures --- ...heritability_hist1.png => fig1_heritability.png} | Bin ...heritability_hist2.png => fig2_heritability.png} | Bin 2 files changed, 0 insertions(+), 0 deletions(-) rename docs/src/man/{heritability_hist1.png => fig1_heritability.png} (100%) rename docs/src/man/{heritability_hist2.png => fig2_heritability.png} (100%) diff --git a/docs/src/man/heritability_hist1.png b/docs/src/man/fig1_heritability.png similarity index 100% rename from docs/src/man/heritability_hist1.png rename to docs/src/man/fig1_heritability.png diff --git a/docs/src/man/heritability_hist2.png b/docs/src/man/fig2_heritability.png similarity index 100% rename from docs/src/man/heritability_hist2.png rename to docs/src/man/fig2_heritability.png From 729a9ec524eaf49c1dda88d2f61598d433d93809 Mon Sep 17 00:00:00 2001 From: Juhyun Kim Date: Wed, 30 Jan 2019 16:22:00 -0800 Subject: [PATCH 54/56] update to v1.1 --- docs/src/man/heritability.md | 201 +++++++++++++++-------------------- docs/src/man/mle_reml.md | 50 ++++----- 2 files changed, 111 insertions(+), 140 deletions(-) diff --git a/docs/src/man/heritability.md b/docs/src/man/heritability.md index d53af40..dd42f40 100644 --- a/docs/src/man/heritability.md +++ b/docs/src/man/heritability.md @@ -1,6 +1,7 @@ # Heritability Analysis + As an application of the variance component model, this note demonstrates the workflow for heritability analysis in genetics, using a sample data set `cg10k` with **6,670** individuals and **630,860** SNPs. Person IDs and phenotype names are masked for privacy. `cg10k.bed`, `cg10k.bim`, and `cg10k.fam` is a set of Plink files in binary format. `cg10k_traits.txt` contains 13 phenotypes of the 6,670 individuals. @@ -21,14 +22,14 @@ Machine information: versioninfo() ``` - Julia Version 0.7.0 - Commit a4cb80f3ed (2018-08-08 06:46 UTC) + Julia Version 1.1.0 + Commit 80516ca202 (2019-01-21 21:24 UTC) Platform Info: OS: macOS (x86_64-apple-darwin14.5.0) CPU: Intel(R) Core(TM) i5-6267U CPU @ 2.90GHz WORD_SIZE: 64 LIBM: libopenlibm - LLVM: libLLVM-6.0.0 (ORCJIT, skylake) + LLVM: libLLVM-6.0.1 (ORCJIT, skylake) ## Read in binary SNP data @@ -50,7 +51,7 @@ using SnpArrays @time cg10k = SnpArray("cg10k.bed") ``` - 0.080738 seconds (70.17 k allocations: 23.096 MiB, 12.02% gc time) + 0.030779 seconds (54 allocations: 19.467 MiB, 28.08% gc time) @@ -110,7 +111,7 @@ mp = missingpos(cg10k) - 6670×630860 SparseMatrixCSC{Bool,Int32} with 5524131 stored entries: + 6670×630860 SparseArrays.SparseMatrixCSC{Bool,Int32} with 5524131 stored entries: [5688 , 1] = true [6445 , 1] = true [109 , 3] = true @@ -210,8 +211,10 @@ Statistics.quantile(maf_cg10k, [0.0 .25 .5 .75 1.0]), mean(maf_cg10k) ```julia -# Pkg.add("Plots") -# Pkg.add("PyPlot") +#using Pkg +#pkg "add Plots" +#pkg"add PyPlot" + using Plots gr(size=(600,500), html_output_format=:png) histogram(maf_cg10k, xlab = "Minor Allele Frequency (MAF)", label = "MAF") @@ -220,7 +223,7 @@ histogram(maf_cg10k, xlab = "Minor Allele Frequency (MAF)", label = "MAF") -![png](heritability_hist1.png) +![png](fig1_heritability.png) @@ -262,7 +265,7 @@ Random.seed!(123) @time Φgrm = grm(cg10k; method = :GRM) ``` - 657.345223 seconds (176.95 k allocations: 361.775 MiB, 0.19% gc time) + 568.176660 seconds (2.91 M allocations: 494.981 MiB, 0.40% gc time) @@ -344,7 +347,7 @@ histogram(Y, layout = 13) -![png](heritability_hist2.png) +![png](fig2_heritability.png) @@ -389,7 +392,7 @@ Before fitting the variance component model, we pre-compute the eigen-decomposit fieldnames(typeof(cg10kdata_rotated)) ``` - 51.732018 seconds (29 allocations: 1021.427 MiB, 0.98% gc time) + 49.812646 seconds (1.74 M allocations: 1.080 GiB, 0.33% gc time) @@ -409,8 +412,8 @@ To load workspace ```julia #pkg"add FileIO JLD2" -using JLD2, FileIO -save("cg10k.jld2") +using JLD2 +@save "cg10k.jld2" varinfo() ``` @@ -422,15 +425,16 @@ varinfo() | Base | | Module | | Core | | Module | | Main | | Module | -| Plots | 22.448 MiB | Module | +| Plots | 2.988 MiB | Module | +| PyPlot | 782.799 KiB | Module | | Y | 677.461 KiB | 6670×13 Array{Float64,2} | | cg10k | 1022.983 MiB | 6670×630860 SnpArray | -| cg10k_trait | 1.386 MiB | 6670×15 DataFrame | +| cg10k_trait | 1.605 MiB | 6670×15 DataFrame | | cg10kdata | 679.508 MiB | VarianceComponentVariate{Float64,2,Array{Float64,2},Array{Float64,2},Array{Float64,2}} | | cg10kdata_rotated | 340.136 MiB | TwoVarCompVariateRotate{Float64,Array{Float64,2},Array{Float64,2}} | | maf_cg10k | 4.813 MiB | 630860-element Array{Float64,1} | | missings_by_snp | 4.813 MiB | 1×630860 Array{Int64,2} | -| mp | 28.748 MiB | 6670×630860 SparseMatrixCSC{Bool,Int32} | +| mp | 28.748 MiB | 6670×630860 SparseArrays.SparseMatrixCSC{Bool,Int32} | | people | 8 bytes | Int64 | | snps | 8 bytes | Int64 | | startupfile | 57 bytes | String | @@ -441,33 +445,39 @@ varinfo() ```julia -#using SnpArrays, JLD2, DataFrames, VarianceComponentModels, Plots -#pyplot() -load("cg10k.jld") +using SnpArrays, JLD2, DataFrames, VarianceComponentModels, Plots +pyplot() + +@load "cg10k.jld2" varinfo() ``` + ┌ Warning: type SparseArrays.SparseMatrixCSC{Bool,Int32} does not exist in workspace; reconstructing + └ @ JLD2 /Users/juhyun-kim/.julia/packages/JLD2/KjBIK/src/data.jl:1153 + -| name | size | summary | -|:----------------- | ------------:|:-------------------------------------------------------------------------------------- | -| Base | | Module | -| Core | | Module | -| Main | | Module | -| Plots | 22.448 MiB | Module | -| Y | 677.461 KiB | 6670×13 Array{Float64,2} | -| cg10k | 1022.983 MiB | 6670×630860 SnpArray | -| cg10k_trait | 1.449 MiB | 6670×15 DataFrame | -| cg10kdata | 679.508 MiB | VarianceComponentVariate{Float64,2,Array{Float64,2},Array{Float64,2},Array{Float64,2}} | -| cg10kdata_rotated | 340.136 MiB | TwoVarCompVariateRotate{Float64,Array{Float64,2},Array{Float64,2}} | -| maf_cg10k | 4.813 MiB | 630860-element Array{Float64,1} | -| missings_by_snp | 4.813 MiB | 1×630860 Array{Int64,2} | -| mp | 28.748 MiB | 6670×630860 SparseMatrixCSC{Bool,Int32} | -| people | 8 bytes | Int64 | -| snps | 8 bytes | Int64 | -| startupfile | 57 bytes | String | -| Φgrm | 339.423 MiB | 6670×6670 Array{Float64,2} | + +| name | size | summary | +|:----------------- | ------------:|:------------------------------------------------------------------------------------------- | +| Base | | Module | +| Core | | Module | +| Main | | Module | +| Plots | 2.988 MiB | Module | +| PyPlot | 782.799 KiB | Module | +| Y | 677.461 KiB | 6670×13 Array{Float64,2} | +| cg10k | 1022.983 MiB | 6670×630860 SnpArray | +| cg10k_trait | 1.605 MiB | 6670×15 DataFrame | +| cg10kdata | 679.508 MiB | VarianceComponentVariate{Float64,2,Array{Float64,2},Array{Float64,2},Array{Float64,2}} | +| cg10kdata_rotated | 340.136 MiB | TwoVarCompVariateRotate{Float64,Array{Float64,2},Array{Float64,2}} | +| maf_cg10k | 4.813 MiB | 630860-element Array{Float64,1} | +| missings_by_snp | 4.813 MiB | 1×630860 Array{Int64,2} | +| mp | 28.748 MiB | getfield(JLD2.ReconstructedTypes, Symbol("##SparseArrays.SparseMatrixCSC{Bool,Int32}#386")) | +| people | 8 bytes | Int64 | +| snps | 8 bytes | Int64 | +| startupfile | 57 bytes | String | +| Φgrm | 339.423 MiB | 6670×6670 Array{Float64,2} | @@ -513,32 +523,32 @@ end For more information visit http://projects.coin-or.org/Ipopt ****************************************************************************** - (σ2a[trait], σ2e[trait]) = (0.25947604776534905, 0.7375938878381869) + (σ2a[trait], σ2e[trait]) = (0.2594760477653529, 0.7375938878381831) Trait2 - (σ2a[trait], σ2e[trait]) = (0.18588150448759697, 0.8137880281704691) + (σ2a[trait], σ2e[trait]) = (0.18588150448760332, 0.8137880281704619) Trait3 - (σ2a[trait], σ2e[trait]) = (0.3196093756535387, 0.6795300238388639) + (σ2a[trait], σ2e[trait]) = (0.31960937565352526, 0.6795300238388766) Trait4 - (σ2a[trait], σ2e[trait]) = (0.2657824469749975, 0.7304994606945179) + (σ2a[trait], σ2e[trait]) = (0.26578244697498915, 0.7304994606945259) Trait5 - (σ2a[trait], σ2e[trait]) = (0.28143198005766734, 0.7169747061765026) + (σ2a[trait], σ2e[trait]) = (0.28143198005767217, 0.7169747061764987) Trait6 - (σ2a[trait], σ2e[trait]) = (0.28300551312417344, 0.7168800753377119) + (σ2a[trait], σ2e[trait]) = (0.2830055131241748, 0.7168800753377107) Trait7 - (σ2a[trait], σ2e[trait]) = (0.21565421413116753, 0.7816910320891622) + (σ2a[trait], σ2e[trait]) = (0.2156542141311619, 0.7816910320891676) Trait8 - (σ2a[trait], σ2e[trait]) = (0.19408878271207622, 0.8058201577783586) + (σ2a[trait], σ2e[trait]) = (0.19408878271207824, 0.8058201577783562) Trait9 - (σ2a[trait], σ2e[trait]) = (0.24746236011764494, 0.7512222977091667) + (σ2a[trait], σ2e[trait]) = (0.24746236011763145, 0.7512222977091793) Trait10 - (σ2a[trait], σ2e[trait]) = (0.0992417256213372, 0.9007769787053679) + (σ2a[trait], σ2e[trait]) = (0.0992417256213392, 0.9007769787053657) Trait11 - (σ2a[trait], σ2e[trait]) = (0.16457266481403124, 0.8343110221526249) + (σ2a[trait], σ2e[trait]) = (0.1645726648140337, 0.8343110221526228) Trait12 - (σ2a[trait], σ2e[trait]) = (0.08224956591863997, 0.9166483378364418) + (σ2a[trait], σ2e[trait]) = (0.0822495659186408, 0.9166483378364408) Trait13 - (σ2a[trait], σ2e[trait]) = (0.05687679106196685, 0.9424058676223442) - 6.559467 seconds (12.64 M allocations: 611.362 MiB, 11.95% gc time) + (σ2a[trait], σ2e[trait]) = (0.05687679106195183, 0.9424058676223598) + 6.203222 seconds (12.80 M allocations: 621.516 MiB, 7.13% gc time) @@ -741,7 +751,7 @@ end (Σa[i, j], Σe[i, j]) = ([0.164571 -0.00169939; -0.00169939 0.0575331], [0.834326 0.200725; 0.200725 0.941755]) Trait12Trait13 (Σa[i, j], Σe[i, j]) = ([0.08398 0.0685417; 0.0685417 0.0559416], [0.91494 0.573206; 0.573206 0.943343]) - 4.258555 seconds (4.55 M allocations: 306.219 MiB, 3.55% gc time) + 4.430316 seconds (4.62 M allocations: 310.482 MiB, 3.51% gc time) ## 3-trait analysis @@ -787,18 +797,18 @@ trait57_model 25 1.4700954e+04 0.00e+00 2.63e-02 -11.0 7.01e-06 -9.2 1.00e+00 1.00e+00f 1 MaxS 30 1.4700954e+04 0.00e+00 1.09e-03 -11.0 2.90e-07 -11.6 1.00e+00 1.00e+00f 1 MaxS 35 1.4700954e+04 0.00e+00 4.49e-05 -11.0 1.20e-08 -14.0 1.00e+00 1.00e+00f 1 MaxS - 40 1.4700954e+04 0.00e+00 1.86e-06 -11.0 4.96e-10 -16.4 1.00e+00 1.00e+00h 1 MaxSA + 40 1.4700954e+04 0.00e+00 1.86e-06 -11.0 4.96e-10 -16.4 1.00e+00 1.00e+00f 1 MaxSA 45 1.4700954e+04 0.00e+00 7.67e-08 -11.0 2.05e-11 -18.8 1.00e+00 1.00e+00h 1 MaxSA iter objective inf_pr inf_du lg(mu) ||d|| lg(rg) alpha_du alpha_pr ls Number of Iterations....: 49 (scaled) (unscaled) - Objective...............: 4.4662368766169078e+02 1.4700954216526392e+04 - Dual infeasibility......: 5.8683114125458765e-09 1.9315987885869595e-07 + Objective...............: 4.4662368766169095e+02 1.4700954216526397e+04 + Dual infeasibility......: 5.8982229663448267e-09 1.9414444012378635e-07 Constraint violation....: 0.0000000000000000e+00 0.0000000000000000e+00 Complementarity.........: 0.0000000000000000e+00 0.0000000000000000e+00 - Overall NLP error.......: 5.8683114125458765e-09 1.9315987885869595e-07 + Overall NLP error.......: 5.8982229663448267e-09 1.9414444012378635e-07 Number of objective function evaluations = 50 @@ -808,11 +818,11 @@ trait57_model Number of equality constraint Jacobian evaluations = 0 Number of inequality constraint Jacobian evaluations = 0 Number of Lagrangian Hessian evaluations = 49 - Total CPU secs in IPOPT (w/o function evaluations) = 0.022 - Total CPU secs in NLP function evaluations = 0.059 + Total CPU secs in IPOPT (w/o function evaluations) = 0.019 + Total CPU secs in NLP function evaluations = 0.054 EXIT: Optimal Solution Found. - 0.098447 seconds (46.49 k allocations: 5.053 MiB) + 0.084479 seconds (46.62 k allocations: 5.057 MiB) @@ -838,7 +848,7 @@ trait57_model 0 -1.470095e+04 1 -1.470095e+04 - 0.548238 seconds (1.04 M allocations: 51.897 MiB, 5.51% gc time) + 0.505115 seconds (1.01 M allocations: 50.370 MiB, 5.09% gc time) @@ -984,7 +994,7 @@ trait57_model 1100 -1.470100e+04 1110 -1.470100e+04 - 0.794444 seconds (135.15 k allocations: 13.553 MiB, 1.62% gc time) + 0.744170 seconds (134.31 k allocations: 13.430 MiB, 1.57% gc time) @@ -1053,13 +1063,13 @@ traitall_model = VarianceComponentModel(cg10kdata_rotated) Stacktrace: - [1] chkposdef at /Users/osx/buildbot/slave/package_osx64/build/usr/share/julia/stdlib/v0.7/LinearAlgebra/src/lapack.jl:50 [inlined] + [1] chkposdef at /Users/osx/buildbot/slave/package_osx64/build/usr/share/julia/stdlib/v1.1/LinearAlgebra/src/lapack.jl:50 [inlined] - [2] sygvd!(::Int64, ::Char, ::Char, ::Array{Float64,2}, ::Array{Float64,2}) at /Users/osx/buildbot/slave/package_osx64/build/usr/share/julia/stdlib/v0.7/LinearAlgebra/src/lapack.jl:5075 + [2] sygvd!(::Int64, ::Char, ::Char, ::Array{Float64,2}, ::Array{Float64,2}) at /Users/osx/buildbot/slave/package_osx64/build/usr/share/julia/stdlib/v1.1/LinearAlgebra/src/lapack.jl:5075 - [3] eigen! at /Users/osx/buildbot/slave/package_osx64/build/usr/share/julia/stdlib/v0.7/LinearAlgebra/src/symmetric.jl:638 [inlined] + [3] eigen! at /Users/osx/buildbot/slave/package_osx64/build/usr/share/julia/stdlib/v1.1/LinearAlgebra/src/symmetric.jl:646 [inlined] - [4] eigen(::Symmetric{Float64,Array{Float64,2}}, ::Symmetric{Float64,Array{Float64,2}}) at /Users/osx/buildbot/slave/package_osx64/build/usr/share/julia/stdlib/v0.7/LinearAlgebra/src/eigen.jl:383 + [4] eigen(::Symmetric{Float64,Array{Float64,2}}, ::Symmetric{Float64,Array{Float64,2}}) at /Users/osx/buildbot/slave/package_osx64/build/usr/share/julia/stdlib/v1.1/LinearAlgebra/src/eigen.jl:383 [5] TwoVarCompModelRotate(::VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}) at /Users/juhyun-kim/.julia/packages/VarianceComponentModels/Mh6LK/src/VarianceComponentModels.jl:119 @@ -1067,23 +1077,21 @@ traitall_model = VarianceComponentModel(cg10kdata_rotated) [7] eval_f(::VarianceComponentModels.TwoVarCompOptProb{VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}},TwoVarCompVariateRotate{Float64,Array{Float64,2},Array{Float64,2}},Array{Float64,2},Array{Float64,1},VarianceComponentAuxData{Array{Float64,2},Array{Float64,1}}}, ::Array{Float64,1}) at /Users/juhyun-kim/.julia/packages/VarianceComponentModels/Mh6LK/src/two_variance_component.jl:731 - [8] (::getfield(Ipopt, Symbol("#eval_f_cb#6")){VarianceComponentModels.TwoVarCompOptProb{VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}},TwoVarCompVariateRotate{Float64,Array{Float64,2},Array{Float64,2}},Array{Float64,2},Array{Float64,1},VarianceComponentAuxData{Array{Float64,2},Array{Float64,1}}}})(::Array{Float64,1}) at /Users/juhyun-kim/.julia/packages/Ipopt/vosfb/src/MPB_wrapper.jl:64 - - [9] eval_f_wrapper(::Int32, ::Ptr{Float64}, ::Int32, ::Ptr{Float64}, ::Ptr{Nothing}) at /Users/juhyun-kim/.julia/packages/Ipopt/vosfb/src/Ipopt.jl:128 + [8] (::getfield(Ipopt, Symbol("#eval_f_cb#6")){VarianceComponentModels.TwoVarCompOptProb{VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}},TwoVarCompVariateRotate{Float64,Array{Float64,2},Array{Float64,2}},Array{Float64,2},Array{Float64,1},VarianceComponentAuxData{Array{Float64,2},Array{Float64,1}}}})(::Array{Float64,1}) at /Users/juhyun-kim/.julia/packages/Ipopt/f6QJl/src/MPB_wrapper.jl:64 - [10] solveProblem(::Ipopt.IpoptProblem) at /Users/juhyun-kim/.julia/packages/Ipopt/vosfb/src/Ipopt.jl:346 + [9] eval_f_wrapper(::Int32, ::Ptr{Float64}, ::Int32, ::Ptr{Float64}, ::Ptr{Nothing}) at /Users/juhyun-kim/.julia/packages/Ipopt/f6QJl/src/Ipopt.jl:128 - [11] optimize!(::Ipopt.IpoptMathProgModel) at /Users/juhyun-kim/.julia/packages/Ipopt/vosfb/src/MPB_wrapper.jl:141 + [10] solveProblem(::Ipopt.IpoptProblem) at /Users/juhyun-kim/.julia/packages/Ipopt/f6QJl/src/Ipopt.jl:346 - [12] macro expansion at ./simdloop.jl:67 [inlined] + [11] optimize!(::Ipopt.IpoptMathProgModel) at /Users/juhyun-kim/.julia/packages/Ipopt/f6QJl/src/MPB_wrapper.jl:141 - [13] #mle_fs!#27(::Int64, ::Symbol, ::Symbol, ::Bool, ::Function, ::VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}, ::TwoVarCompVariateRotate{Float64,Array{Float64,2},Array{Float64,2}}) at /Users/juhyun-kim/.julia/packages/VarianceComponentModels/Mh6LK/src/two_variance_component.jl:942 + [12] #mle_fs!#27(::Int64, ::Symbol, ::Symbol, ::Bool, ::Function, ::VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}, ::TwoVarCompVariateRotate{Float64,Array{Float64,2},Array{Float64,2}}) at /Users/juhyun-kim/.julia/packages/VarianceComponentModels/Mh6LK/src/two_variance_component.jl:949 - [14] (::getfield(VarianceComponentModels, Symbol("#kw##mle_fs!")))(::NamedTuple{(:solver, :verbose),Tuple{Symbol,Bool}}, ::typeof(mle_fs!), ::VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}, ::TwoVarCompVariateRotate{Float64,Array{Float64,2},Array{Float64,2}}) at ./none:0 + [13] (::getfield(VarianceComponentModels, Symbol("#kw##mle_fs!")))(::NamedTuple{(:solver, :verbose),Tuple{Symbol,Bool}}, ::typeof(mle_fs!), ::VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}, ::TwoVarCompVariateRotate{Float64,Array{Float64,2},Array{Float64,2}}) at ./none:0 - [15] top-level scope at util.jl:156 + [14] top-level scope at util.jl:156 - [16] top-level scope at In[92]:3 + [15] top-level scope at In[54]:3 From the output we can see the Fisher scoring algorithm ran into some numerical issues. Let's try the **MM algorithm**. @@ -1219,13 +1227,13 @@ traitall_model = VarianceComponentModel(cg10kdata_rotated) 1070 -4.435625e+04 1080 -4.435625e+04 - 3.423491 seconds (131.35 k allocations: 65.559 MiB, 0.86% gc time) + 3.314963 seconds (131.35 k allocations: 65.449 MiB, 0.65% gc time) - (-44356.24416253091, VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}(Array{Float64}(0,13), ([0.272112 0.190023 … -0.128464 -0.0976418; 0.190023 0.216814 … -0.0687833 -0.04341; … ; -0.128464 -0.0687833 … 0.116994 0.0900933; -0.0976418 -0.04341 … 0.0900933 0.105876], [0.725183 0.570497 … -0.0589748 -0.125486; 0.570497 0.783023 … 0.0235685 0.0464638; … ; -0.0589748 0.0235685 … 0.882056 0.551829; -0.125486 0.0464638 … 0.551829 0.893642]), Array{Float64}(0,0), Char[], Float64[], -Inf, Inf), ([0.0111615 0.0131072 … 0.0128913 0.0127584; 0.0131071 0.0151656 … 0.017153 0.0171365; … ; 0.0128911 0.017153 … 0.0174174 0.018212; 0.0127593 0.0171355 … 0.0182119 0.0188], [0.0112229 0.0133031 … 0.013004 0.0127777; 0.0133033 0.015806 … 0.0178516 0.0177826; … ; 0.0130042 0.0178517 … 0.0179556 0.0187634; 0.0127776 0.0177822 … 0.0187633 0.0193475]), [0.00012458 7.242e-5 … -3.70019e-7 -1.40553e-5; 7.23929e-5 0.000171795 … -2.04581e-5 -3.19854e-6; … ; -3.8356e-7 -2.04665e-5 … 0.000352066 -1.46068e-5; -1.40476e-5 -3.19013e-6 … -1.46022e-5 0.000374325], Array{Float64}(0,13), Array{Float64}(0,0)) + (-44356.24416259489, VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}(Array{Float64}(0,13), ([0.272112 0.190023 … -0.128464 -0.0976418; 0.190023 0.216814 … -0.0687833 -0.04341; … ; -0.128464 -0.0687833 … 0.116994 0.0900933; -0.0976418 -0.04341 … 0.0900933 0.105876], [0.725183 0.570497 … -0.0589748 -0.125486; 0.570497 0.783023 … 0.0235685 0.0464638; … ; -0.0589748 0.0235685 … 0.882056 0.551829; -0.125486 0.0464638 … 0.551829 0.893642]), Array{Float64}(0,0), Char[], Float64[], -Inf, Inf), ([0.0111603 0.013107 … 0.0128915 0.0127587; 0.0131027 0.0151631 … 0.017153 0.0171359; … ; 0.0128908 0.017153 … 0.0174176 0.018212; 0.0127586 0.0171361 … 0.0182122 0.0188002], [0.0112235 0.0133041 … 0.0130038 0.0127778; 0.0133005 0.0158053 … 0.0178518 0.0177823; … ; 0.0130043 0.0178518 … 0.0179557 0.0187638; 0.0127775 0.0177823 … 0.0187637 0.0193477]), [0.000124552 7.23469e-5 … -3.6584e-7 -1.40474e-5; 7.23585e-5 0.00017168 … -2.04611e-5 -3.18804e-6; … ; -3.70686e-7 -2.04634e-5 … 0.000352082 -1.46096e-5; -1.4039e-5 -3.1795e-6 … -1.46073e-5 0.000374334], Array{Float64}(0,13), Array{Float64}(0,0)) @@ -1239,42 +1247,3 @@ It converges after ~1000 iterations. #save("copd.jld2") #varinfo() ``` - - - - -| name | size | summary | -|:----------------- | ------------:|:-------------------------------------------------------------------------------------- | -| Base | | Module | -| Core | | Module | -| Main | | Module | -| Plots | 22.448 MiB | Module | -| Y | 677.461 KiB | 6670×13 Array{Float64,2} | -| _ | 240 bytes | Tuple{Array{Float64,2},Array{Float64,2}} | -| cg10k | 1022.983 MiB | 6670×630860 SnpArray | -| cg10k_trait | 1.443 MiB | 6670×15 DataFrame | -| cg10kdata | 679.508 MiB | VarianceComponentVariate{Float64,2,Array{Float64,2},Array{Float64,2},Array{Float64,2}} | -| cg10kdata_rotated | 340.136 MiB | TwoVarCompVariateRotate{Float64,Array{Float64,2},Array{Float64,2}} | -| h | 64 bytes | 3-element Array{Float64,1} | -| hST | 144 bytes | 13-element Array{Float64,1} | -| hST_se | 144 bytes | 13-element Array{Float64,1} | -| hse | 64 bytes | 3-element Array{Float64,1} | -| maf_cg10k | 4.813 MiB | 630860-element Array{Float64,1} | -| missings_by_snp | 4.813 MiB | 1×630860 Array{Int64,2} | -| mp | 28.748 MiB | 6670×630860 SparseMatrixCSC{Bool,Int32} | -| people | 8 bytes | Int64 | -| snps | 8 bytes | Int64 | -| startupfile | 57 bytes | String | -| trait57_data | 339.627 MiB | TwoVarCompVariateRotate{Float64,Array{Float64,2},Array{Float64,2}} | -| trait57_model | 472 bytes | VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}} | -| traitall_model | 2.961 KiB | VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}} | -| traitidx | 16 bytes | 3-element UnitRange{Int64} | -| Σa | 6.844 KiB | 13×13 Array{Array{Float64,2},2} | -| Σcov | 2.570 KiB | 18×18 Array{Float64,2} | -| Σe | 6.844 KiB | 13×13 Array{Array{Float64,2},2} | -| Φgrm | 339.423 MiB | 6670×6670 Array{Float64,2} | -| σ2a | 144 bytes | 13-element Array{Float64,1} | -| σ2e | 144 bytes | 13-element Array{Float64,1} | - - - diff --git a/docs/src/man/mle_reml.md b/docs/src/man/mle_reml.md index d31d94b..d54c46f 100644 --- a/docs/src/man/mle_reml.md +++ b/docs/src/man/mle_reml.md @@ -8,14 +8,14 @@ Machine information versioninfo() ``` - Julia Version 0.7.0 - Commit a4cb80f3ed (2018-08-08 06:46 UTC) + Julia Version 1.1.0 + Commit 80516ca202 (2019-01-21 21:24 UTC) Platform Info: OS: macOS (x86_64-apple-darwin14.5.0) CPU: Intel(R) Core(TM) i5-6267U CPU @ 2.90GHz WORD_SIZE: 64 LIBM: libopenlibm - LLVM: libLLVM-6.0.0 (ORCJIT, skylake) + LLVM: libLLVM-6.0.1 (ORCJIT, skylake) ## Demo data @@ -157,7 +157,7 @@ vcmodel_mle = deepcopy(vcmodel) 9 -3.844374e+03 10 -3.844373e+03 - 4.546981 seconds (11.23 M allocations: 566.109 MiB, 5.23% gc time) + 5.031460 seconds (11.29 M allocations: 568.015 MiB, 4.78% gc time) The output of `fit_mle!` contains @@ -172,7 +172,7 @@ logl - -3844.3731814180883 + -3844.3731814180887 @@ -300,7 +300,7 @@ vcmodel_reml = deepcopy(vcmodel) 9 -3.846631e+03 10 -3.846630e+03 - 0.443964 seconds (8.09 k allocations: 62.532 MiB, 2.39% gc time) + 0.726373 seconds (388.90 k allocations: 82.673 MiB, 13.22% gc time) The output of `fit_reml!` contains @@ -315,7 +315,7 @@ logl - -3844.3777179025046 + -3844.3777179025055 @@ -345,7 +345,7 @@ vcmodel_reml -* standard errors of the estimated varianec component parameters +* standard errors of the estimated variance component parameters ```julia @@ -458,7 +458,7 @@ vcmodel_mm = deepcopy(vcmodel) 9 -3.844374e+03 10 -3.844373e+03 - 0.042187 seconds (21.56 k allocations: 1.366 MiB) + 0.055578 seconds (21.91 k allocations: 1.394 MiB) @@ -529,16 +529,16 @@ vcmodel_ipopt = deepcopy(vcmodel) iter objective inf_pr inf_du lg(mu) ||d|| lg(rg) alpha_du alpha_pr ls 50 3.8443732e+03 0.00e+00 6.76e-06 -11.0 4.08e-07 - 1.00e+00 1.00e+00f 1 MaxSA 55 3.8443732e+03 0.00e+00 1.83e-06 -11.0 1.11e-07 - 1.00e+00 1.00e+00f 1 MaxSA - 60 3.8443732e+03 0.00e+00 4.97e-07 -11.0 3.00e-08 - 1.00e+00 1.00e+00f 1 MaxSA + 60 3.8443732e+03 0.00e+00 4.97e-07 -11.0 3.00e-08 - 1.00e+00 1.00e+00h 1 MaxSA Number of Iterations....: 63 (scaled) (unscaled) - Objective...............: 3.4496886481728779e+02 3.8443731733053728e+03 - Dual infeasibility......: 2.2693631701157965e-07 2.5290047251948971e-06 + Objective...............: 3.4496886481728791e+02 3.8443731733053728e+03 + Dual infeasibility......: 2.2693631660531264e-07 2.5290047206674095e-06 Constraint violation....: 0.0000000000000000e+00 0.0000000000000000e+00 Complementarity.........: 0.0000000000000000e+00 0.0000000000000000e+00 - Overall NLP error.......: 2.2693631701157965e-07 2.5290047251948971e-06 + Overall NLP error.......: 2.2693631660531264e-07 2.5290047206674095e-06 Number of objective function evaluations = 64 @@ -548,11 +548,11 @@ vcmodel_ipopt = deepcopy(vcmodel) Number of equality constraint Jacobian evaluations = 0 Number of inequality constraint Jacobian evaluations = 0 Number of Lagrangian Hessian evaluations = 63 - Total CPU secs in IPOPT (w/o function evaluations) = 1.488 - Total CPU secs in NLP function evaluations = 0.375 + Total CPU secs in IPOPT (w/o function evaluations) = 1.739 + Total CPU secs in NLP function evaluations = 0.293 EXIT: Solved To Acceptable Level. - 2.486697 seconds (4.13 M allocations: 201.947 MiB, 3.03% gc time) + 2.745554 seconds (4.30 M allocations: 210.935 MiB, 2.63% gc time) @@ -600,6 +600,7 @@ vcmodel_knitro.Σ ## Starting point + Here are a few strategies for successful optimization. * For $d>1$ (multivariate response), initialize $B, \Sigma$ from univariate estimates. @@ -609,6 +610,7 @@ Here are a few strategies for successful optimization. ## Constrained estimation of `B` + Many applications invoke constraints on the mean parameters `B`. For demonstration, we enforce `B[1,1]=B[1,2]` and all entries of `B` are within [0, 2]. @@ -654,7 +656,7 @@ We first try the MM algorithm. 9 -3.844650e+03 10 -3.844650e+03 - 0.170236 seconds (170.93 k allocations: 8.918 MiB) + 0.185885 seconds (179.51 k allocations: 9.295 MiB) @@ -740,17 +742,17 @@ vcmodel_constr 45 3.8446498e+03 0.00e+00 2.49e-05 -11.0 1.51e-06 - 1.00e+00 1.00e+00f 1 MaxS iter objective inf_pr inf_du lg(mu) ||d|| lg(rg) alpha_du alpha_pr ls 50 3.8446498e+03 0.00e+00 6.76e-06 -11.0 4.08e-07 - 1.00e+00 1.00e+00f 1 MaxSA - 55 3.8446498e+03 0.00e+00 1.83e-06 -11.0 1.11e-07 - 1.00e+00 1.00e+00h 1 MaxSA + 55 3.8446498e+03 0.00e+00 1.83e-06 -11.0 1.11e-07 - 1.00e+00 1.00e+00f 1 MaxSA 60 3.8446498e+03 0.00e+00 4.97e-07 -11.0 3.00e-08 - 1.00e+00 1.00e+00f 1 MaxSA Number of Iterations....: 63 (scaled) (unscaled) - Objective...............: 3.4484507551949679e+02 3.8446498170293380e+03 - Dual infeasibility......: 2.2694405212011240e-07 2.5301808562731130e-06 + Objective...............: 3.4484507551949685e+02 3.8446498170293398e+03 + Dual infeasibility......: 2.2694405475622814e-07 2.5301808856629548e-06 Constraint violation....: 0.0000000000000000e+00 0.0000000000000000e+00 Complementarity.........: 0.0000000000000000e+00 0.0000000000000000e+00 - Overall NLP error.......: 2.2694405212011240e-07 2.5301808562731130e-06 + Overall NLP error.......: 2.2694405475622814e-07 2.5301808856629548e-06 Number of objective function evaluations = 64 @@ -760,11 +762,11 @@ vcmodel_constr Number of equality constraint Jacobian evaluations = 0 Number of inequality constraint Jacobian evaluations = 0 Number of Lagrangian Hessian evaluations = 63 - Total CPU secs in IPOPT (w/o function evaluations) = 0.024 - Total CPU secs in NLP function evaluations = 0.625 + Total CPU secs in IPOPT (w/o function evaluations) = 0.028 + Total CPU secs in NLP function evaluations = 0.634 EXIT: Solved To Acceptable Level. - 0.729483 seconds (102.83 k allocations: 8.150 MiB, 0.88% gc time) + 0.760983 seconds (102.63 k allocations: 8.135 MiB) From d8009ea0cb87562050b88b98bb06c833f4cd497d Mon Sep 17 00:00:00 2001 From: Juhyun Kim Date: Wed, 30 Jan 2019 16:24:13 -0800 Subject: [PATCH 55/56] delete v.7 ipynb --- docs/heritability.ipynb | 1971 --------------------------------------- docs/mle_reml.ipynb | 1312 -------------------------- 2 files changed, 3283 deletions(-) delete mode 100644 docs/heritability.ipynb delete mode 100644 docs/mle_reml.ipynb diff --git a/docs/heritability.ipynb b/docs/heritability.ipynb deleted file mode 100644 index d0aa5c7..0000000 --- a/docs/heritability.ipynb +++ /dev/null @@ -1,1971 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Heritability Analysis" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "As an application of the variance component model, this note demonstrates the workflow for heritability analysis in genetics, using a sample data set `cg10k` with **6,670** individuals and **630,860** SNPs. Person IDs and phenotype names are masked for privacy. `cg10k.bed`, `cg10k.bim`, and `cg10k.fam` is a set of Plink files in binary format. `cg10k_traits.txt` contains 13 phenotypes of the 6,670 individuals." - ] - }, - { - "cell_type": "code", - "execution_count": 37, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "cg10k.bed\n", - "cg10k.bim\n", - "cg10k.fam\n", - "cg10k_traits.txt\n" - ] - } - ], - "source": [ - ";ls cg10k.bed cg10k.bim cg10k.fam cg10k_traits.txt" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Machine information:" - ] - }, - { - "cell_type": "code", - "execution_count": 38, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Julia Version 0.7.0\n", - "Commit a4cb80f3ed (2018-08-08 06:46 UTC)\n", - "Platform Info:\n", - " OS: macOS (x86_64-apple-darwin14.5.0)\n", - " CPU: Intel(R) Core(TM) i5-6267U CPU @ 2.90GHz\n", - " WORD_SIZE: 64\n", - " LIBM: libopenlibm\n", - " LLVM: libLLVM-6.0.0 (ORCJIT, skylake)\n" - ] - } - ], - "source": [ - "versioninfo()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Read in binary SNP data" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We will use the [`SnpArrays.jl`](https://github.com/OpenMendel/SnpArrays.jl) package to read in binary SNP data and compute the empirical kinship matrix. The package has not yet been registered and must be installed using the repository location. Start julia and use the `]` key to switch to the package manager REPL\n", - "```julia\n", - "(v0.7) pkg> add https://github.com/OpenMendel/SnpArrays.jl.git#juliav0.7\n", - "```\n", - "Use the backspace key to return to the Julia REPL." - ] - }, - { - "cell_type": "code", - "execution_count": 39, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "using SnpArrays" - ] - }, - { - "cell_type": "code", - "execution_count": 40, - "metadata": { - "scrolled": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " 0.080738 seconds (70.17 k allocations: 23.096 MiB, 12.02% gc time)\n" - ] - }, - { - "data": { - "text/plain": [ - "6670×630860 SnpArray:\n", - " 0x02 0x02 0x03 0x03 0x02 0x03 … 0x02 0x03 0x03 0x03 0x03 0x03\n", - " 0x03 0x03 0x02 0x03 0x02 0x03 0x03 0x02 0x02 0x03 0x02 0x01\n", - " 0x03 0x03 0x02 0x03 0x02 0x03 0x03 0x03 0x03 0x03 0x03 0x03\n", - " 0x03 0x03 0x03 0x03 0x03 0x03 0x03 0x02 0x02 0x02 0x02 0x03\n", - " 0x03 0x03 0x03 0x03 0x03 0x03 0x03 0x03 0x03 0x03 0x03 0x02\n", - " 0x02 0x02 0x03 0x03 0x02 0x03 … 0x03 0x03 0x03 0x03 0x03 0x03\n", - " 0x00 0x00 0x03 0x03 0x00 0x03 0x03 0x02 0x03 0x03 0x03 0x03\n", - " 0x03 0x03 0x03 0x03 0x03 0x02 0x03 0x03 0x03 0x03 0x03 0x02\n", - " 0x03 0x03 0x02 0x03 0x02 0x03 0x02 0x02 0x03 0x03 0x03 0x03\n", - " 0x03 0x03 0x03 0x00 0x03 0x03 0x02 0x02 0x03 0x03 0x02 0x03\n", - " 0x03 0x03 0x02 0x03 0x02 0x02 … 0x02 0x03 0x03 0x03 0x03 0x03\n", - " 0x02 0x02 0x03 0x03 0x02 0x03 0x02 0x03 0x03 0x03 0x03 0x02\n", - " 0x03 0x03 0x03 0x00 0x03 0x03 0x03 0x03 0x03 0x03 0x03 0x02\n", - " ⋮ ⋮ ⋱ ⋮ \n", - " 0x02 0x02 0x03 0x03 0x02 0x03 0x03 0x02 0x03 0x03 0x02 0x02\n", - " 0x02 0x02 0x02 0x03 0x00 0x02 0x02 0x03 0x03 0x03 0x02 0x03\n", - " 0x03 0x03 0x02 0x02 0x02 0x03 … 0x03 0x02 0x03 0x03 0x02 0x03\n", - " 0x02 0x02 0x03 0x03 0x02 0x03 0x03 0x02 0x03 0x03 0x03 0x02\n", - " 0x03 0x03 0x03 0x02 0x03 0x02 0x03 0x03 0x03 0x03 0x02 0x03\n", - " 0x03 0x03 0x01 0x02 0x02 0x01 0x03 0x00 0x00 0x02 0x00 0x02\n", - " 0x03 0x03 0x03 0x02 0x03 0x03 0x03 0x03 0x03 0x03 0x03 0x02\n", - " 0x03 0x03 0x02 0x02 0x02 0x03 … 0x00 0x03 0x03 0x03 0x03 0x03\n", - " 0x03 0x03 0x03 0x02 0x02 0x03 0x02 0x02 0x02 0x02 0x02 0x03\n", - " 0x03 0x03 0x03 0x02 0x03 0x02 0x02 0x03 0x02 0x03 0x03 0x02\n", - " 0x02 0x02 0x03 0x03 0x02 0x03 0x03 0x03 0x01 0x03 0x03 0x03\n", - " 0x03 0x03 0x03 0x03 0x03 0x02 0x03 0x00 0x03 0x03 0x03 0x03" - ] - }, - "execution_count": 40, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# read in genotype data from Plink binary file (~50 secs on my laptop)\n", - "@time cg10k = SnpArray(\"cg10k.bed\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Summary statistics of SNP data" - ] - }, - { - "cell_type": "code", - "execution_count": 41, - "metadata": { - "scrolled": true - }, - "outputs": [ - { - "data": { - "text/plain": [ - "(6670, 630860)" - ] - }, - "execution_count": 41, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "people, snps = size(cg10k)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The positions of the missing data are evaluated by" - ] - }, - { - "cell_type": "code", - "execution_count": 42, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "6670×630860 SparseMatrixCSC{Bool,Int32} with 5524131 stored entries:\n", - " [5688 , 1] = true\n", - " [6445 , 1] = true\n", - " [109 , 3] = true\n", - " [189 , 3] = true\n", - " [216 , 3] = true\n", - " [254 , 3] = true\n", - " [331 , 3] = true\n", - " [522 , 3] = true\n", - " [525 , 3] = true\n", - " [597 , 3] = true\n", - " [619 , 3] = true\n", - " [672 , 3] = true\n", - " ⋮\n", - " [4929 , 630860] = true\n", - " [5594 , 630860] = true\n", - " [5650 , 630860] = true\n", - " [5780 , 630860] = true\n", - " [5854 , 630860] = true\n", - " [5867 , 630860] = true\n", - " [6084 , 630860] = true\n", - " [6175 , 630860] = true\n", - " [6178 , 630860] = true\n", - " [6239 , 630860] = true\n", - " [6478 , 630860] = true\n", - " [6511 , 630860] = true" - ] - }, - "execution_count": 42, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "mp = missingpos(cg10k)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The number of missing data values in each column can be evaluated as" - ] - }, - { - "cell_type": "code", - "execution_count": 43, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "1×630860 Array{Int64,2}:\n", - " 2 0 132 77 0 27 2 2 6 27 2 … 6 4 5 11 0 0 4 29 0 5 43" - ] - }, - "execution_count": 43, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "missings_by_snp = sum(mp, dims=1)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Minor allele frequencies (MAF) for each SNP." - ] - }, - { - "cell_type": "code", - "execution_count": 44, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "630860-element Array{Float64,1}:\n", - " 0.1699160167966407 \n", - " 0.17098950524737633 \n", - " 0.11402569593147749 \n", - " 0.2686940694676172 \n", - " 0.21926536731634183 \n", - " 0.23934969140448592 \n", - " 0.19061187762447507 \n", - " 0.20200959808038388 \n", - " 0.027160864345738278\n", - " 0.2997139846454915 \n", - " 0.24625074985003004 \n", - " 0.05555555555555558 \n", - " 0.3659067046647667 \n", - " ⋮ \n", - " 0.22547254725472543 \n", - " 0.4035864345738295 \n", - " 0.20799579957995795 \n", - " 0.44801200300075017 \n", - " 0.2954647845021775 \n", - " 0.14265367316341826 \n", - " 0.1709145427286357 \n", - " 0.2814281428142814 \n", - " 0.06113537117903933 \n", - " 0.052473763118440764\n", - " 0.13930982745686427 \n", - " 0.1324128564961521 " - ] - }, - "execution_count": 44, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "maf_cg10k = maf(cg10k)" - ] - }, - { - "cell_type": "code", - "execution_count": 45, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "([0.00841726 0.124063 … 0.364253 0.5], 0.24536516625042462)" - ] - }, - "execution_count": 45, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# 5 number summary and average MAF (minor allele frequencies)\n", - "using Statistics\n", - "Statistics.quantile(maf_cg10k, [0.0 .25 .5 .75 1.0]), mean(maf_cg10k)" - ] - }, - { - "cell_type": "code", - "execution_count": 46, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlgAAAH0CAIAAABuMsSDAAAABmJLR0QA/wD/AP+gvaeTAAAgAElEQVR4nO3deXxTdb7/8W/SJd2g0MpaoQwglEVEATdQRNSrYCmI1xkEkRFnUFyBhyOOiAsMUJhRAcfLD6UjOoNAK6UoDIvQCoIbiqAIVBSklJaldG+znZzfH7nmdpKT0CU9Sfp9PR/8Eb6fnHM++eacvJuTzaCqqgAAQFbGQDcAAEAgEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpEYQAAKkRhAAAqRGEAACpteQgPHXqVGlpqX/Xabfb+VI6IYTD4VAUJdBdBAWbzRboFoKCoigOhyPQXQSeqqp2uz3QXQSFEHq0bMlB+Pzzz3/00Uf+XWdZWRl7uRDCbDZXVVUFuougcOHChUC3EBQqKystFkuguwg8m81WXl4e6C6CwsWLF0Plb6OWHIQAAFwSQQgAkBpBCACQGkEIAJAaQQgAkBpBCACQGkEIAJAaQQgAkBpBCACQGkEIAJBaeKAbQOCZzWZvX44VHh4eGxurcz8AoCeCUBZnzpzJz8/3HDebzQ89NqPk7BnPkupQBlx19f7P9jR/dwAQMAShLIbflXbOFmEMi3AbV6y1lYUF4g2tn+n48VNL3lw9mgOAwCEIZVFccLLqhYOiVTv3wtl8Me+6QHQEAEGBN8sAAKRGEKIxvv322w5JXYxeLF26NNANAkB9cWoUjfH1118br/2tOuENz5Ip65lQ+VlqABAEIZrEwBkFACGPBzIAgNR4RggAQWThwoXr168PdBd+oChKWFhYIxbs3bv32rVr/d6PDyEfhAcOHBg2bFh1dXWgG2mJqkryj/8U1/Yyz4rdYlYGjde/I6DFKygouOuuu/77v/870I0ERn5+/oIFC3TeaGgHYXFx8csvv1xTUxPoRlqosjP2mETrrO0apX8+3pi/9ADUw+WXX3711VcHuovAMBoD8IJdCL9GaDabp02btmzZskA30pIZjGEi7jKNf+GmQLcGAP4RwkH4xBNPzJgxo2vXroFuBAAQwkL41GhmZubbb7/tvGwwGGpra6OiogLbEoQQQrFt+mjL6aJiz4pBiD8+PPWKK67QvykA8CaEg7CsrMx5wWAw8Anu4GH/+avcVh1zj7fyLEV9/s7tI28lCAEEFf+fGlUUJSUlRbM0dOhQw68eeeSRhq6ntLQ0NTU1ISFhzJgxpaVav5aAINFnhLjrT57/TO27BbozAHDn52eES5cuXbNmzbFjxzxLqqoePXr09OnT8fHxQojwcF+b1lxPenp6cnJyVlbWrFmzFi9evHDhQteaNVdis9l27NjheuLocu2111555ZX1v1F11dbWRkRERES4/5hRkMjds+/NlW9plsxms5831ogn4apqsVha2Lt8a2trW9gtapza2lpFUTg3Y7Vam7hL2O12P/YTilRVbcQEmkymxn1sUfg9CAcMGNCjR4/U1FTP0tmzZ61Wa1pa2rFjx0aOHLly5UrnS3p2u90Viq7LmuvJzs7OyckxmUyPP/54WlqaKwi9URTl5MmTnlPTpUsXb89ZL8lms9lstsYtq4PZL7z0ffvholNvjZryoe7taLDb7cE8gY0Q5LuEbmw2m9FoZCpsv2r0GhwOh9vIa0uX/TMrp2l91ZdRiBeeeWrMmDH6bE6TqqqNmMCIiIhgCcIRI0Z4KxUXFw8ZMuTVV1/t2rXrjBkznnrqqffff18IMW3atFGjRo0fPz49Pd1iscydO9fbegoLC5OTk4UQycnJRUVFl2wmKirqD3/4wwMPPND42+PBZrPFx8cH7TNCY1iY6Hur6HWzRu3dS5yLbjBDIxYxxMbGOk8JtBg1NTUt7BY1jqqqJpMpOjo60I0EmNVqNRqNTdklIiMj3UYOHT76jamvuOruprVWv63nvXH69GkdNuRDEyewEfR7s8zAgQN37drlvLxo0aJ+/fo5Ly9ZsuTOO+/MyckpKSnZsGGDjzWoqmowGJwXFEVp7oYBIFh07CX63KrDdsIOZmuOGwyGCRMmrFmzxjUyceLENWvWuE6G2+32pKSkhISEH374wflA7VzK7VlacJ741e9zhN98882+ffuclyMjI02m//1EdkJCwrhx49atWzdlyhTXoKbOnTsXFBQIIQoLC5OSkpq7YQCAS15ensVicV62Wq15eXl1q5988slll1124cKFw4cP1x23/yfdum0QPZ4R5uXl3XLLLdXV1ffee29eXl6PHj3mzZs3duxYZ3XFihW5ubn5+fljx46NjY0dNWqUt/WkpqZmZGQsWLAgIyMjLS1Nh87hXw6bedU/3tmZm+dZMhrE7Gef5RwjELSGDRu2Y8eOu+++Wwixa9euoUOHZmZmuqrr169/8MEHT5w4kZmZ2b9//8C12Rh6BOGIESNUVR02bNiLL76YmppaXl4+atSo5cuXO6tdunTZuHFjTEzMli1bNN9u6jJ37tyJEyd26dLlmmuuee+993ToHP5VU3xqfZsewqLxMlLk9r9Nf/RRghAIWvfcc88HH3zgDMLs7Oxx48a5gtBms23cuPGbb7758ccfH3300Zdeesl1djQkNEsQur2F2vlfg8Ewffr06dOnu1159OjRzgudOnXq1KmTj/W0adNm8+bN/m8Xehpyn+h3u+dwxKcr9e8FQP2NHj161qxZNpstLCxs69atixcvdpV27do1YMCApKSkjh07lpeXHz582PWksG4ivvXWWw8//LDefddDCH+zDFoSi0MMGHKD5hfPt2kV99OxH/RvCUBdrVq1GjRoUF5eXkxMTP/+/euev1m/fv3HH3/syry6Z0dD4qOlBCGCglJdXjbz3yK+k3vBUl254DrXiXQ3HTp0uO+++5q9OQBCiF/PjsbFxY0bN841aLVas7Ozf/nlF+dPIGzbtu3pp58OrbOjBCGCRnxnkXC5+2B5sU1x/OnDo55Xd1Sc+435F4IQ0M2YMWPmzJkTExPzpz/9yTW4Y8eOrl27un4IaPjw4adOnap7djT4EYQIcqpQhfm+1zUqJ75SNz+tez+AvBISEvr27WuxWNq3b+8aXL9+fd1vAYuKiho5cmRovXeUIAQAXILrpb7t27e7Da5evdrtyps2bXJbKsgRhAhZ5srTxee79blKszhhfNrC+a/o3BHQHBISE0y73o345XMdtmX/+SshBuiwoaBCECJkXSwwOwy/3JehUTr44Y8nTurdD9A8Frz0wtX9Gvk7AQ1317XXXqvXtoIFQYhQFmESXbT+ei34VlhO6t0M0DxMJtOkSZMC3UVLpt93jQIAEIQIQgCA1AhCAIDUeI0QLdMvPx9ftmyZZmno0KGDBg3SuR8AQYsgREv085cHTp777qN8z4rj+GfzamsJQgAuBCFaIsVmSL7act9rnpXwDX/Wvx0AwYwgBIAgEh0dnZmZefSoxvfryuDChQv6b5QgBIAgMn/+/LfeeivQXfhBZWVlbGys5m+r+dCzZ8+631yqD4IQAIJIdHT0k08+Gegu/ODcuXOJiYlhYWGBbuTSCEJIx2az1dTUaJZMJlNIHLcA/IggDD2Kohw/flyzZLFYdG4m5Kjnfp770qaX5y/wLDls1uzsDWPGjNG/KwABRBCGntf+vuKFl+ZFxMZ7lqpKz4vb9O8olKhlZ9SxL9vvfMazFL/yHv37ARBwBGHoOXToO/PoF8zD/+hZCnu2m6J/QwAQyviKNQCA1AhCAIDUODUK/C/FUpO+9O/vrc/2LEUYxet/TW/fvr3+XQFobgQh8L/MxSf2tRstIq7yLEVtfP6Vuc8ThECLRBACdfS5VVx1t+dw+PZF+vcCQB+8RggAkBpBCACQGkEIAJAarxECl6Yq9ry8vPx8jV/6LSsru//++/VvCYC/EITApdWazU+/tjo8Ks6jolYe3EkQAiGNIAQuTbWZqye+Jdr3cC8oNuNjGl/6CiCEEIRAU50+fVpzPDIyko8eAsGPIASawKE4TLG9r7nes6LabXGxsWPHaHwqUQhx45BBU6Y82MzNAagXghBoAtUhaitrlp7XKO3PMv/ribcu/kajdOrA6eJ/E4RAkCAIgeZiiI5XRz6uUfhqvbjwoe7tANDG5wgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFLjc4RAACh2e3l5uWYpKirKZDLp3A8gM4IQ0N2Fkzs/3tE+qatnRbFa5s7589y5cz1LFRUV27dv97bKO+64o3Xr1v5sEpAGQQjorrzI0Hu4dfoHGqXsuVu2f1xSVuFZ2Z736clKR+RlXTxL5u8+/uaLff369fN7p4AMCEIgmPy494vI6C8K2nlWjCcLHPf9zTx4vGep9fyBzd8Z0GIRhECQ6XG9+K+ZnsPGT1c59G8GkADvGgUASI0gBABIjVOjQMiz2mwTpj4aHR3tWYoKE3k7thoMBv27AkIFQQiEPEtFyXe9ZohEjc9jiKWpS5Ys0QzCxMTEhx56qNmbA4IeQQi0CD1vFJ37aowbjHNyz3oOq9UX2xd+RhACgiAEWjiHYrtngTB4vBug+Jj4x72BaAgIOgRhkCorK3v4iZnm2lrP0sGDh8T1V+rfEgC0SARhkHr33Xc3HSy0Xf+AZ8lY8aX+/QBAS0UQBi9jxyvEtb/1HDdsW6J/MwDQUhGEgKSs5tr33ntPs9S1a9fhw4fr3A8QKAQhIKXS0yVlFdPf2uZZsZcUDP1NAkEIeRCEgJTsNmGKrZr8D43SgRzHyX/p3hAQMHzFGgBAagQhAEBqBCEAP1jxj/diYmIMXnz99deBbhDwitcIAfjB7j2fiqnviOvv9yzFp1+nfz9A/fGMEAAgNZ4RAmgAVVVVVdWseF1Ese/evbuwsFCzevfddxuN/EWOQCIIAdTXnj17brllhEN1aNRMsWLiaM2lqivKnl/5QWSrtp6lih8+7ZPSx6AVhL+5vNOHGzKb1i9QLwQhgP9UW3Hg+x/6DRnqWblQdFq9c5YYN9+zFP7iALuX9ak2c+29f6tNvkajNi3mh1teEuGR7uPlZ4s3PduwtoHGIggB/Keio5Xh8T+MXKBRWv+MQfu8aBP0uF5ERLsPlvzi560A3hGEANwZYtqInjdqFKLjde8FaHYEIYDgY60pr7UmdOqiWbxj5K1r/7la547QghGEAIJPdalis5bO/kqj9P2/d+554+b/StVc7sH7xk2d+lDz9oYWhyAEEJQMRpFwucZ4+dmLNuOeXlM1St/mXPX94ebuCy0PQQggxBjiEsRVWh/VOJsvxFnd20HI43OsAACpEYQAAKkRhAAAqfEaIQD4044dO6qqqjRL/fr1u+GGG3TuB5dEEAKANrvdbrFYNEtGozE62uMLcYT47LPPpj49W/S5VWNthUcmjxxEEAYhghCAFGw2m+b4x3l7Vry9SgiDZ2nv51+Wnyv0LKiqGh8ff+FskeciFoslvF23ion/T2NLu948eWLb2rVrNdsYPHhwz549fd2AhqioqPD2rDQyMvKyyy7z14ZaBoIQQMu3YtXq6X+cajBopJ3DFCdGPi7aa4XQmSzxtwIR08Z9vLzItqjhvzZ88uvcn499sTzbs2I78fVfn5/hryCsqam58tphFy5c8Cw57LY+vXt988Vev2yoxSAIAbR8ebt3qw+8qQ77vWfJ+Eyyo9/toofWGcvV0/zZhLVGpNxaOelNz0rUuif9uB2z2Xz2zGnLq1ofqTy+z7Lzz37cVstAEAJoKSrO/SN37foPNmpUqqrEmIY/h5OM1Wr18qvLIjw8PCwsTOd+dEMQAmgpio/W9r6terTGM56wN9L0bye0fPXVVzffervNUutZUhVlyZLFM2fO1L8rfRCEAFqQ6HjRrrvnsCHMz491DsWel5fnOX7o0CGHQ/HvtvTx7bffGoeMV+5f4VmKzHpG/370RBACQAPVlFVbbOOeeNGzYr5w2hadoE8XP/744xdffOE5XlVVFaJhHCgEIQA0kN0qhCh76mON0vbXwvZn6tPFiLvSyi7rY4iIchtXzVU2m12fHs6fP5+Tk6NZqqysfOihh+LjQ+DHnAlCAAgw1Wb+1/oPvv7uB89SmEH8aebTV1xxhWepsuxi9aOvi/iO7oXzP4tD13jblmK3FxVpfAKyvLzcyxtlfJn4h8f2nCg3JiR5lmxfrE9LSyMIAQCXZj/57Wften5W2duzFJ23/P777tUMwsYoO3P8+PGe/TVi0lpdLgaNb+j6KioqzCOeEP3/y7MU84PWM+agRBACQBDodZMY8ajnsOm7Df7cStVFtW1SzdyvNUoZv/eWB46Lp2fPefv5F1/xLFlFuBjgzwYDgiAEgOCl2m27du0qLi72LNns2l8a5/8eSk7Z73zWdssjnqWweYP16aFZEYQAELyqS4qXfLA7MvdHz1JtrVm/PiKiNL5qTgjN72gNOQQhAAQv1W6xjJpjSblFo7ZfM5nQYPwwLwBAajwjDKSKioppTz9bUHDKs1RUcFJJuln/lgBANgRhIH3xxRcffvJZ9eiXNWon08Ma8aEeAEADEYQBFt7qMjFglEZh72rdewEAGfEaIQBAagQhAEBqBCEAQGoEIQBAagQhAEBqBCEAQGoEIQBAanyO0G9KSko0vyFeCBEREdGrVy+d+wEA1AdB6DdDb7/7zIUyg9H9Sbaq2CNtVReKCwPSFQDAN4LQb07mH7YsPCGiW7sXys60WXJjIDoCAFwarxECAKRGEAIApEYQAgCkRhACAKTGm2X0oNhtH3zwgef4wYMHFcWufz8AABeCsPlVlVTVWh5a8i/PiuVMvt0Up39HAAAXgrD5KTZhCKt4eJ1G6d9Lwg5t0r0hAMD/4TVCAIDUCEIAgNQIQgCA1AhCAIDUCEIAgNQIQgCA1EI+CA8cOBAbGxvoLgAAoSq0g7C4uPjll1+uqakJdCMAgFAVwkFoNpunTZu2bNmyQDcCAAhhIRyETzzxxIwZM7p27RroRgAAISyEgzAzM3PEiBEGg0EIYTAYzGZzoDsCAISeEP6u0bKyMucFg8GgqmpgmwEAhCj/PyNUFCUlJUWzVFpampqampCQMGbMmNLS0oaup0GLAwBQH35+Rrh06dI1a9YcO3ZMs5qenp6cnJyVlTVr1qzFixcvXLiwQevxtri3p4O1tbVvvPHGRx995DY+fvz422+/vQG3qo6ysjKHwxEREaFRa9yz0tB9KtuYzvW7td7uDR8dNENz/l2ljrOnyyJ6rtLn0Rncu3IQ8HnKzXtJVcvLy3V70hIXF6f9yFwPfg7CAQMG9OjRIzU1VbOanZ2dk5NjMpkef/zxtLQ0Z5LZ7fbw8P9tw3VZcz2ai/sQERExYMCAG264wW38iiuuiI6ObtwNjI6Ojo6O9jLdhsassVEL4ZIMXibWoOtDGPduSGjM3aTvjhRgBm+H06VERUU1+sG2oYzGxp/g9HMQjhgxwke1sLAwOTlZCJGcnFxUVOQcnDZt2qhRo8aPH5+enm6xWObOnettPZqL+xAeHn7zzTc/8MADjbgh3phMpqioKO0glO1Br5GPHsErqJsTQs8GG7GlZmjOn6ts7CM5Lsn7zBoMzgdMHZtpJF3fLKOqqvMvC1VVFUVxDi5ZsuTOO+/MyckpKSnZsGFDQxcHAKApdP34ROfOnQsKCoQQhYWFSUlJzsGEhIRx48atW7duypQpJpOpoYsDANAUegRhXl6e80JqampGRoaqqhkZGWlpac7BFStW5Obm5ufnL1iwYMuWLT7Wo7k4AABNoUcQul7wmzt37qFDh7p06XL48OE5c+Y4B7t06bJx48bk5OQtW7bExMT4WI/m4gAANEWzvEbo9l5b13/btGmzefNmtyuPHj3aeaFTp06dOnXysR7NxQEAaIoQ/oo1AACajiAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASI0gBABIjSAEAEiNIAQASC2Eg9DhcEyZMmXw4MEpKSmZmZmBbgcAEJJCOAh37Nhhs9n279+fmZk5ffr0QLcDAAhJIRyEV1111aJFi8xmc0FBQadOnQLdDgAgJIUHuoHG69ixoxDi7rvv3rx58/bt2wPdDgAgJIXwM0JFURwOR05OTlZW1iOPPBLodgAAISmEg/DNN9989tlnw8LCevToUVNTE+h2AAAhyc9BOHToUMOvPJ+llZaWpqamJiQkjBkzprS01PeqFEVJSUnxsfjkyZN/+umnAQMGTJgwYeXKlf69IQAASfjzNUJVVY8ePXr69On4+HghRHi4+8rT09OTk5OzsrJmzZq1ePHihQsXelvV0qVL16xZc+zYMd+Lb9iwwUc/VVVVM2bMeP75593Gn3766UmTJjXstv3q4sWLFoslIiJCo6aq3pfzXvKxUMP5aqFxK2xkzZ/LNI63qfB1J/m/O69rVBuzMR1nr+Hbaobm/LlKn/Md+F1Zv7vWJ29t+Nxdfe3kJSUlcXFxTeyqnuLj400mU+OW9WcQnj171mq1pqWlHTt2bOTIkStXroyKiqp7hezs7JycHJPJ9Pjjj6elpTmD0G63uyLTdXnAgAE9evRITU295OI+xMbGzp49e+zYsW7jCQkJzqhuBFVV4+PjtYPQYPC+nPeSj4UazlcLjVthI2v+XKZxvE2FwceB6//uvK7R0Ji7SsfZ03Fb3vmzB5/z3chd2Y/pFQzTLby34XN39bWTt23bNjExsYld1ZPR2PgTnP4MwuLi4iFDhrz66qtdu3adMWPGU0899f7779e9QmFhYXJyshAiOTm5qKjIOTht2rRRo0aNHz8+PT3dYrHMnTtXCDFixAjP9Wsu7oPBYOjQoUPPnj2bftNcwn7lx3UiSATJg1GIYvbgyWg0hsSjpT+DcODAgbt27XJeXrRoUb9+/dyuoKqq8y8LVVUVRXEOLlmy5M4778zJySkpKfF9qlNzcQAAmsKfb5b55ptv9u3b57wcGRnpebq2c+fOBQUFQojCwsKkpCTnYEJCwrhx49atWzdlyhTfZ3g1FwcAoCn8GYTV1dXjxo07cuSI1WqdN2+e68W5vLw854XU1NSMjAxVVTMyMtLS0pyDK1asyM3Nzc/PX7BgwZYtW3ysX3NxAACawp9BOGzYsBdffDE1NTUpKam0tDQ9Pd057nrBb+7cuYcOHerSpcvhw4fnzJnjHOzSpcvGjRuTk5O3bNkSExPjY/2aiwMA0BT+fI3QYDBMnz7d8/uvXW+9bdOmzebNm92qo0ePdl7o1KmT21eGur1nV3NxAACaIoS/WQYAgKYjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSCw90AyFm45btGe/+02gweJZsNpv+/QAAmoggbJj0v712POU+0aGXRi3vY93bAQA0FUHYcD1uEN2vC3QTAAD/4DVCAIDUQjsIS0tLU1NTExISxowZU1paGuh2AAChJ7SDMD09PTk5uaioqGvXrosXLw50OwCA0BParxFmZ2fn5OSYTKbHH388LS1t4cKFdauqqpaUlJw6dcptqfj4+Li4uEZuUlVF5QVRdka7Wl4sLBWAbDcAABFESURBVFXug5XnhXBoL2KuFHabdslWKyzV2iWHImrKNEuqKkRVidf2Ks4KxeOtrRXnhapqL1Jbripe2rPWCDVSu6TYRW25dkl1iOqLXtpTRcVZYfT4y6zinBBe2qspE4rdW3uqrVZ7ihSrqK30Mns+2hOi4qyIjPbYkFkIob1I9UWhKtolS7VqM2uX7BZhqfJ6v/tq75yIaaNdKisSnu9zri7x3l6lard4b097t1Qdiqgp9dpe5XmvpbIiERHlPlh1Xqg+jhqrl6PG6+yp3o8aoTp8HTXlxcJa4z5Y6eOoqRDejhpbrbCavLTn46hRvbanClFxVqgO9/GKs16Pmtpyr+1Za1WjwctRY/N21Pg4qK12x4BB1xoMGk+3WsXGFJz8SWNtTWA0Gg1a7+evD4Oqqv7tRk9xcXHnz5+Pjo6ura3t0KFDRUVF3eq4ceNyc3Ojo90fvGbOnPnggw82bovj7pvw/eEfhNZsV1VUxLVq7VlSHY6aqqrY1q09F1FsVqvVFh0b61mymWsdwmiKMnmWLLW1xrDwiMgIz1JtTW1kZGRYeJhnqbqyMiYuznNHUVVHTWVlbOt4jfbsNqvF6q09RTVERXs8fglhqa0xhkVotmeuqY3w1l5VVUxMrMHo2Z5aXVEeF6/xEK/YbRazJUbrbxqbxexwCJNWe9baGhEWHhkZqdFebW14RGS4dnvVMTExnu0JIarKSuPatPUcdyh2c61Zsz271WJXHFEee6YQwlpTLcIiIk1e2guPCI/Q+OO1promKirKGKbxiFNVVhoX39Zzt3QoirmmJqZVK+327EpUTIxGe7U1whjurb2w8PCICI373Vd75eVxreM12nMotdXVsa00jhq7zWq32bXbM9eoIkzzqPHRXm1NTaQpKkyrveqKilivB7WXo8Zqtdp0PKhj47SOGp8HtdkaHafVnsXsUIUpSvOoqTV4ac9cY46IjPDSXlVMrMZBLYSIi4468NUXnuNNER8fbzJpzG19hPYzQlVVnY/sqqoqiuJWjYuLW758+QMPPODHLeZkrYuPj9c8nKRSU1Njs9ni4zWONNkUFRV16tQp0F0EXllZmclk8vy7UzZWq7WysjIxMTHQjQTeuXPnEhMTw8I0MjLYhPZrhJ07dy4oKBBCFBYWJiUlBbodAEDoCe0gTE1NzcjIUFU1IyMjLS0t0O0AAEJPaAfh3LlzDx061KVLl8OHD8+ZMyfQ7QAAQk9ov0bYpk2bzZs3B7oLAEAIC+1nhPr7y1/+cvz48UB3EXi5ubnvvPNOoLsIPEVRHn300UB3ERRWrly5d+/eQHcReIcPH16yZEmguwgKzz777IULFwLdRb0QhA2zZ8+ekpKSQHcReD///PO3334b6C4Cz+FwcE7Caf/+/Z6f2ZXQ+fPn+YPAaceOHdXV1YHuol4IQgCA1AhCAIDUCEIAgNQIQgCA1EL74xO+Wa3WTZs2Ob96xl+Ki4vffffd3bt3+3Gdoeizzz47c+bMggULAt1IgCmK4nA4mAchxNGjRxVFKS4uDnQjAfbjjz8WFBSwSwghKisr33zzzYSEBH02d++99/bq1atxy4b2l277dv78+VdffbXR30euyeFwGD1/IUE+zt3Gv3MbotglnNglXNglnHSeh0mTJvXt27dxy7bkIAQA4JL4swUAIDWCEAAgNYIQACA1ghAAIDWC0JfS0tLU1NSEhIQxY8aUlpY2qNrCXPLGKoqSkpKif2N68j0JOTk5/fv3b9Omzc0335yfnx+QDnXjeyq2bt3at2/fNm3a9O3bd/v27QHpUB/1eRD4/vvvY2NjdW5Mf76nYujQoYZfPfLIIwHp0AeC0Jf09PTk5OSioqKuXbsuXry4QdUWxveNXbp06Y033njs2LGA9KYbH5Nw6tSpSZMmvfXWW0VFRWPGjPn9738fqCb14WMqHA7HxIkTly9ffvHixVdeeaVlT8UlHwTKy8unTJlSU1Ojf2868zEVqqoePXr09OnTlZWVlZWVr7/+eqCa9EqFd7169Tpy5IiqqkeOHOnVq1eDqi2M7xu7a9euDz/8sMXvTj4mITc39+GHH3ZePnfuXGJiYgD605GPqbBYLJs3b3Y4HBUVFZs2berbt2+AetSD7+PC4XCMHTs2MzOzxR8aqs+pKCoqiouLGzRoUFxcXFpa2tmzZwPUo1ct/+5pitjY2JqaGlVVa2pqWrVq1aBqC1OfG9vij/b6TILdbn/kkUemT5+ub2t6u+RUVFZWCiEMBsPevXt1704/vudh4cKFs2bNUiU4NFSfU3HgwIERI0YcOHCgpKRk8uTJv/vd7wLUo1ecGvVFVVXnN2WoqqooSoOqLYxUN9abS07Cxx9/PGTIkPj4+KVLl+rena4uORVxcXFVVVXz589/6qmndO9OPz7mITc3d+vWrQsXLgxQa3rzMRUDBw7ctWvXwIEDExISFi1atG3btgD16BVB6Evnzp2dX1VaWFiYlJTUoGoLI9WN9cbHJKiq+txzz73yyitr165dtGhReHhL/hZf4XMqTp48+cwzzwghYmNjp06deuTIkcC0qAsf87Bz585PPvkkMjLSGQ8Gg+HTTz8NTJe68DEV33zzzb59+5yXIyMjTSZTAPrziSD0JTU1NSMjQ1XVjIyMtLQ052BeXp6Pakvleyok4WMS9u3bl52dvWnTps6dO1dVVVVVVQWy0ebnYyo6d+68atWqTz75RFXVdevWXX311YFstJn5mIf58+e7zrwJIVRVHTZsWABbbW4+pqK6unrcuHFHjhyxWq3z5s0bO3ZsIBvVpMPp19BVWlo6atSopKSk1NTUsrIy56Br0jSrLZXvqdD8b8vjYxLmz58v1ZHle3/Iy8u75ppr2rZte8MNNzjfQNFS1ee40BxpeXxMhcPh+Pvf/96jR4/LLrts8uTJ5eXlAe1UA1+6DQCQGqdGAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgBAFIjCAEAUiMIAQBSIwgRpAwGg8lkKisrcxsvKyszmUzOHzt1Xu3o0aPN3Uz//v0NBsOqVas8m3RtvZ6dNKJhgxcNWklosVqtQ4YMOXfunKj3nuDi487yNo1nz54dPHiw1WptthuEoNbCf0cbIc1qtW7YsOGhhx6qO5idna3zA9bhw4cPHz4cGxubmZk5depUPTftsnz58ssvvzwgmw6I5cuX33bbbe3bt3f+t/57gu87y9s0dujQ4dZbb33jjTdmzpzpvxuB0BHoH0QEtAkhrr766pEjR7qN33777c4fPXf+t7KyUlGUZu3khRdeiI+P/8tf/hIWFnbhwgW3Jl0/PFv3sg/1vFoTFwlpNputS5cuJ06ccP63nnuCUz3vLE8//fRT165d7Xa7f24DQgqnRhG87r///l27dhUVFblGzp07t3PnzgkTJrhG4uLijMZm3I1VVV23bt299957//33K4qSnZ3dfNuC07Zt2y6//PJu3bq5RuqzJ4im3Vndu3fv1KnTtm3bmtw+Qg9BiOB1xx13tG3bdt26da6RrKys1q1b33HHHa4Rt1fpvvrqq9/+9rcJCQndu3evu6DD4Xj99df79esXFxc3aNCgrKysumv46quvUlNTr7vuOs8eDh48mJ+fP2HChG7dug0ePDgzM7M+nauqumLFij59+sTExFx99dWrV69WVbXRV/PNrX8f6/zyyy9HjhzZtm3b7t27/+EPf/j88881X2o9evRo3RfefKzQx4Srqrps2bK+ffu2bt36pptu+vTTT4UQc+bM+c1vfuNa/NSpU0ajMScnx+0Wbdu2bfjw4XVH6rMniMbeWS7Dhw/fvn17gxZBCxGYJ6LApQghjhw58sc//nHIkCGuwWHDhk2dOvXIkSOuXVf858nJ6667bsWKFfv27Zs8eXJkZGRVVZWztGTJkri4uMWLF2/ZsmX27NlhYWEffviha6nBgwdPnTr1X//6l2cbs2fP7tixo/OM2aJFi8LCws6fP+/WpOfld999t2/fvqtXr/73v//t3Nz//M//NOhqblOxefPmI//JbDZr9u9tnfv37w8LCxs7duyaNWsyMzMnTJiQmJioOY11p/eSt8XbhC9dujQ+Pn7ZsmVZWVnjxo0LCws7ePDg999/L4T4/PPPnddZsGBBhw4drFar2+295pprsrKy3Cb5kntCfe4sH9Ooquq6desGDRrkOf9o8QhCBCnnw19eXp4QIj8/X1XVU6dOCSE+/vhjH0H42muvOS9XVFS4Sg6HIzEx0flUxmn27Nk33XSTa6lnnnlGsweHw9G9e/cnn3zS+d/jx48LIVauXOnWpOfla6655vjx466rzZw5c9iwYQ26mttUePruu+80+/e2zrvuumvy5Ml1V+t8L4lnV27p4vu2eJvwjh07rlmzxllSFOWuu+567733VFUdMGDAjBkznNdJSUnRnPl27drt3bu37s2vz55QnzvLxzSqqrp79+727dt79oMWjyBEkHI+/CmKkpSU9PLLL6uqumTJkg4dOthsNh9BuH//frc1qKp69uxZIURJSYmrtHv37sTERNfVdu/erdnDl19+6fnQefvtt3tuwu1ybGys21IdOnRo0NU8p8LHRNXt39s627dvn5ubW3fBXbt21ScIfd8WzQk/f/68EMLtvSpOCxcuTEpKUhTFObc//PCD53XCw8MPHTrkttpL7gkNurM0HTx4MDw83McV0FLx8QkENaPROGHChDVr1rzwwgtr16697777wsN97bSej9reVqsoiuu/7dq107zaunXr2rVr98knn7heM1u9evWSJUvOnz/vbRGnmJiYPXv2REdH+26jnle7pLrNeFun1Wp1+8hdWFiY5tpqamrq36TmhNvtdm/r/93vfvfcc8/t3bs3MzPzhhtu6NOnj+d12rRpU1VV5TZ4yT2h0XeWS2VlZdu2betzTbQwvFkGwe7+++8/duzY+vXrv/76a7d3CdZTu3btEhMTt2zZ4hrZvHlz//79fS/lcDjWr1+flpbWp0+flF89+OCD9Xk7Yr9+/QoLC52L9O7de/ny5atXr2701RrE2zqvvPLKd955p+4116xZU/e/rvz77LPPmthkhw4d2rZtu3PnTud/HQ7H4MGDFyxYIITo1q3bjTfe+M9//vP999/39qHMyy+/vO4bRF187AlNubNcioqKpPqwJv5PoJ+SAtpEnRecevfu3bZt2+TkZIfDof7nuTvh5eSk238XL17cqlWrv/71r5pvltE8Y7Z3714hxObNm+sOOhyOnj173nbbbb63npOT06pVq9deey07O3vy5MlCCM3N+bia5lT4nijf69yxY4cQ4p577lm7dm1WVtbEiRPj4+Nd05icnDxy5MitW7e+8847bh/Oq+dtcfvvggULEhIS3njjja1btz744IORkZGuV+OWL19uNBpjYmIqKio0b9H06dP//Oc/e67Wx57Q0DtL03PPPffYY4/5uAJaKoIQQaruw9Yrr7wihHj22Wed/21EECqK8uqrrzo/AzBw4EDPNyV6NvDkk0+2atWq7rsKnWbNmmU0Gs+dO+d766tWrUpJSYmJiRk0aNAHH3zgbXPeruZtKupT9bbOrVu3Xn/99XFxcd26dZs0adKePXtc07hz586UlJS4uLiRI0f+/PPPbn8i1/O2uE34okWLevbsGRMTM2TIkG3btrmuVlxcbDQap0yZ4u0WZWdn133TUH32hIbeWZqGDRu2ceNGH1dAS2VQG/65JQAtwNGjR/v06aP/I8CJEye6d+++e/fum266SfMKNpstOTn5yy+/1PNE5alTp66//vpffvklIiJCt40iSPAaIQCd2O32ysrKefPmpaSkDBs2zNvVIiIipk+f/vbbb+vZ26pVqx577DFSUE7/Hz0kdvMCJ2OgAAAAAElFTkSuQmCC" - }, - "execution_count": 46, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# Pkg.add(\"Plots\")\n", - "# Pkg.add(\"PyPlot\")\n", - "using Plots\n", - "gr(size=(600,500), html_output_format=:png)\n", - "histogram(maf_cg10k, xlab = \"Minor Allele Frequency (MAF)\", label = \"MAF\")" - ] - }, - { - "cell_type": "code", - "execution_count": 47, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0.0013128198764010824" - ] - }, - "execution_count": 47, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# proportion of missing genotypes\n", - "sum(missings_by_snp) / length(cg10k)" - ] - }, - { - "cell_type": "code", - "execution_count": 48, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0.07228069619249913" - ] - }, - "execution_count": 48, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# proportion of rare SNPs with maf < 0.05\n", - "count(!iszero, maf_cg10k .< 0.05) / length(maf_cg10k)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Empirical kinship matrix\n", - "\n", - "We estimate empirical kinship based on all SNPs by the genetic relation matrix (GRM). Missing genotypes are imputed on the fly by drawing according to the minor allele frequencies." - ] - }, - { - "cell_type": "code", - "execution_count": 49, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "657.345223 seconds (176.95 k allocations: 361.775 MiB, 0.19% gc time)\n" - ] - }, - { - "data": { - "text/plain": [ - "6670×6670 Array{Float64,2}:\n", - " 0.502735 0.00328112 -6.79435e-5 … -6.09591e-5 -0.00277687 \n", - " 0.00328112 0.49807 -0.00195548 0.000884994 0.00341591 \n", - " -6.79435e-5 -0.00195548 0.492348 0.000198191 -0.000337529\n", - " 0.00087876 -0.00322605 -0.00192088 -0.00235314 -0.00124267 \n", - " -5.03227e-5 -0.00352498 0.00184588 0.00219109 -0.00163189 \n", - " 0.00203199 0.000597697 0.00251237 … 0.00088428 2.21226e-5 \n", - " 0.000560862 0.00244517 -0.00183233 0.00120214 -0.00120686 \n", - " -0.000656949 0.00322249 -0.00101472 0.00355832 -0.000240444\n", - " -0.00103874 -0.00125164 -0.000599731 0.00176492 0.00176928 \n", - " -0.00137058 0.00209596 0.000146711 -0.0014453 -0.00103066 \n", - " -0.00209312 0.000140721 -0.000442031 … -0.000214963 -0.00107391 \n", - " 0.000933587 0.00168842 0.00185731 -0.000787908 -0.00311063 \n", - " 0.000334572 -0.000886623 0.00304182 0.000752285 -0.00123334 \n", - " ⋮ ⋱ \n", - " 0.00298109 0.00121739 0.00102822 9.66935e-6 0.00306953 \n", - " -0.00209728 0.00271452 -0.00182325 -0.00108139 0.00366901 \n", - " 0.000549425 -0.00244419 -0.00301368 … -0.000631961 0.00215641 \n", - " -0.00423362 -0.00208073 -0.00107904 -0.000619315 -0.000593852\n", - " -0.00326697 -0.000769552 0.00310511 0.000520658 -0.000113441\n", - " 0.000430563 -0.0020236 0.00265425 -0.00635493 -0.00520252 \n", - " 0.00218746 0.000798767 -0.00105684 -0.000918245 -0.00061484 \n", - " -0.00230525 -0.000101149 0.000117936 … 0.000879829 -0.00233479 \n", - " -0.00201305 0.00233864 -0.00134496 0.00197044 -0.000486275\n", - " -0.000990534 -0.000924159 -9.12302e-5 0.00122311 -0.00298296 \n", - " -6.09591e-5 0.000884994 0.000198191 0.499289 0.000481492\n", - " -0.00277687 0.00341591 -0.000337529 0.000481492 0.499799 " - ] - }, - "execution_count": 49, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "## GRM using SNPs with maf > 0.01 (default) (~10 mins on my laptop)\n", - "using Random \n", - "Random.seed!(123)\n", - "@time Φgrm = grm(cg10k; method = :GRM)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Phenotypes" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Read in the phenotype data and compute descriptive statistics." - ] - }, - { - "cell_type": "code", - "execution_count": 63, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "

6,670 rows × 13 columns

Trait1Trait2Trait3Trait4Trait5Trait6Trait7Trait8Trait9Trait10Trait11Trait12Trait13
Float64⍰Float64⍰Float64⍰Float64⍰Float64⍰Float64⍰Float64⍰Float64⍰Float64⍰Float64⍰Float64⍰Float64⍰Float64⍰
1-1.81573-0.946151.11363-2.098670.7444170.001391720.934732-1.226771.11608-0.4436280.824466-1.02853-0.394049
2-1.24440.109660.467119-1.621311.056680.9789471.000150.3248741.162322.692273.082641.090650.0256616
31.455671.538671.094030.586655-0.327965-0.303377-0.0334355-0.464463-0.33194-0.486839-1.10649-1.42016-0.687463
4-0.7688090.5134910.244263-1.31741.193941.173441.087370.5360230.8027590.2341590.394175-0.7673660.0635386
5-0.264415-0.34824-0.02390650.004739161.256191.203891.298010.3101140.626160.8992890.5499680.5406880.179675
6-1.37617-1.471920.29118-0.803111-0.26424-0.260573-0.165372-0.2192571.04702-0.09858160.9473930.5940150.245407
70.100942-0.191616-0.5674210.378571-0.246656-0.6088110.189081-1.27078-0.4524760.7025630.3326360.002691650.317117
8-0.3198181.357740.81869-1.155660.6344840.2914620.933324-0.7410830.647478-0.9708780.2208610.852512-0.225905
9-0.2883340.5660830.254958-0.6525780.6689220.9783090.1228631.479090.06721320.07959040.1675320.2469160.539933
10-1.1576-0.781199-0.595808-1.005550.7898290.5710580.951304-0.2959630.990420.5613090.7331-1.73468-1.35278
110.7405691.408740.734690.0208323-0.337441-0.458304-0.142583-0.580392-0.684685-0.00785381-0.712244-0.313346-0.345419
12-0.6758920.2798930.267916-1.041040.9107420.8660281.074140.03817510.766355-0.340118-0.8090140.548522-0.0201829
13-0.79541-0.699990.39913-0.5104761.515521.287431.537720.1339891.020260.499019-0.369483-1.10153-0.598132
14-0.193483-0.286021-0.6914940.01315821.523371.401061.531150.3330661.043720.163207-0.422884-0.383528-0.489222
150.1512462.091852.038-1.124751.665571.625361.587510.6358520.8425780.450762-1.39479-0.5609840.28935
16-0.4646090.3612771.23277-0.8260341.434751.744520.2110972.648161.025110.1197570.0596832-0.631232-0.207879
17-0.732977-0.5262230.616579-0.554480.9474850.9368330.9725170.2902511.012850.516207-0.03006890.8787320.450255
18-0.1673260.1753270.287468-0.4026530.5511820.5222050.4368380.2995650.58311-0.704416-0.73081-1.95141-0.933505
191.411591.787220.8439760.481278-0.0887674-0.4995780.304196-1.23884-0.153476-0.8704860.0955473-0.983708-0.356345
20-1.42997-0.4901470.27273-1.61030.9907880.7116881.18858-0.3712291.24703-0.03891620.8834962.589883.3354
21-0.1472470.1232840.617549-0.1871310.2564380.177950.412612-0.2448090.09476250.723017-0.6839480.0873751-0.26221
22-0.187113-0.270777-1.015570.06028510.272420.869133-0.6575192.32389-0.9999361.446720.971158-0.358748-0.439658
23-1.82434-0.933481.29474-1.945450.3358470.3592020.513653-0.07319771.571391.533291.820772.22741.50063
24-2.29344-2.491620.40384-2.364881.410531.422441.170240.8447671.790270.648182-0.0857231-1.02790.491288
25-0.4341360.7408820.699576-1.024060.7595290.9566560.63330.7707340.8249891.842871.91046-0.5023170.13267
26-2.1921-2.494660.354855-1.931560.9419790.9789170.894860.4632391.125371.705280.7177930.6458880.783968
27-1.46602-1.249220.307978-1.550970.6189080.6625080.4759570.4847190.4015650.55988-0.376938-0.9339830.390013
28-1.83318-1.532692.55674-1.518280.789410.9087480.6499720.6683741.200580.2779631.25053.31372.22036
29-0.7845470.2765833.01105-1.119790.9208240.7502181.26154-0.4033640.400667-0.217598-0.72467-0.391945-0.650024
300.4644561.33264-1.2306-0.3579761.18251.54316-0.603393.383090.823741-0.129951-0.65798-0.499535-0.414477
⋮⋮⋮⋮⋮⋮⋮⋮⋮⋮⋮⋮⋮⋮
" - ], - "text/latex": [ - "\\begin{tabular}{r|ccccccccccccc}\n", - "\t& Trait1 & Trait2 & Trait3 & Trait4 & Trait5 & Trait6 & Trait7 & Trait8 & Trait9 & Trait10 & Trait11 & Trait12 & Trait13\\\\\n", - "\t\\hline\n", - "\t& Float64⍰ & Float64⍰ & Float64⍰ & Float64⍰ & Float64⍰ & Float64⍰ & Float64⍰ & Float64⍰ & Float64⍰ & Float64⍰ & Float64⍰ & Float64⍰ & Float64⍰\\\\\n", - "\t\\hline\n", - "\t1 & -1.81573 & -0.94615 & 1.11363 & -2.09867 & 0.744417 & 0.00139172 & 0.934732 & -1.22677 & 1.11608 & -0.443628 & 0.824466 & -1.02853 & -0.394049 \\\\\n", - "\t2 & -1.2444 & 0.10966 & 0.467119 & -1.62131 & 1.05668 & 0.978947 & 1.00015 & 0.324874 & 1.16232 & 2.69227 & 3.08264 & 1.09065 & 0.0256616 \\\\\n", - "\t3 & 1.45567 & 1.53867 & 1.09403 & 0.586655 & -0.327965 & -0.303377 & -0.0334355 & -0.464463 & -0.33194 & -0.486839 & -1.10649 & -1.42016 & -0.687463 \\\\\n", - "\t4 & -0.768809 & 0.513491 & 0.244263 & -1.3174 & 1.19394 & 1.17344 & 1.08737 & 0.536023 & 0.802759 & 0.234159 & 0.394175 & -0.767366 & 0.0635386 \\\\\n", - "\t5 & -0.264415 & -0.34824 & -0.0239065 & 0.00473916 & 1.25619 & 1.20389 & 1.29801 & 0.310114 & 0.62616 & 0.899289 & 0.549968 & 0.540688 & 0.179675 \\\\\n", - "\t6 & -1.37617 & -1.47192 & 0.29118 & -0.803111 & -0.26424 & -0.260573 & -0.165372 & -0.219257 & 1.04702 & -0.0985816 & 0.947393 & 0.594015 & 0.245407 \\\\\n", - "\t7 & 0.100942 & -0.191616 & -0.567421 & 0.378571 & -0.246656 & -0.608811 & 0.189081 & -1.27078 & -0.452476 & 0.702563 & 0.332636 & 0.00269165 & 0.317117 \\\\\n", - "\t8 & -0.319818 & 1.35774 & 0.81869 & -1.15566 & 0.634484 & 0.291462 & 0.933324 & -0.741083 & 0.647478 & -0.970878 & 0.220861 & 0.852512 & -0.225905 \\\\\n", - "\t9 & -0.288334 & 0.566083 & 0.254958 & -0.652578 & 0.668922 & 0.978309 & 0.122863 & 1.47909 & 0.0672132 & 0.0795904 & 0.167532 & 0.246916 & 0.539933 \\\\\n", - "\t10 & -1.1576 & -0.781199 & -0.595808 & -1.00555 & 0.789829 & 0.571058 & 0.951304 & -0.295963 & 0.99042 & 0.561309 & 0.7331 & -1.73468 & -1.35278 \\\\\n", - "\t11 & 0.740569 & 1.40874 & 0.73469 & 0.0208323 & -0.337441 & -0.458304 & -0.142583 & -0.580392 & -0.684685 & -0.00785381 & -0.712244 & -0.313346 & -0.345419 \\\\\n", - "\t12 & -0.675892 & 0.279893 & 0.267916 & -1.04104 & 0.910742 & 0.866028 & 1.07414 & 0.0381751 & 0.766355 & -0.340118 & -0.809014 & 0.548522 & -0.0201829 \\\\\n", - "\t13 & -0.79541 & -0.69999 & 0.39913 & -0.510476 & 1.51552 & 1.28743 & 1.53772 & 0.133989 & 1.02026 & 0.499019 & -0.369483 & -1.10153 & -0.598132 \\\\\n", - "\t14 & -0.193483 & -0.286021 & -0.691494 & 0.0131582 & 1.52337 & 1.40106 & 1.53115 & 0.333066 & 1.04372 & 0.163207 & -0.422884 & -0.383528 & -0.489222 \\\\\n", - "\t15 & 0.151246 & 2.09185 & 2.038 & -1.12475 & 1.66557 & 1.62536 & 1.58751 & 0.635852 & 0.842578 & 0.450762 & -1.39479 & -0.560984 & 0.28935 \\\\\n", - "\t16 & -0.464609 & 0.361277 & 1.23277 & -0.826034 & 1.43475 & 1.74452 & 0.211097 & 2.64816 & 1.02511 & 0.119757 & 0.0596832 & -0.631232 & -0.207879 \\\\\n", - "\t17 & -0.732977 & -0.526223 & 0.616579 & -0.55448 & 0.947485 & 0.936833 & 0.972517 & 0.290251 & 1.01285 & 0.516207 & -0.0300689 & 0.878732 & 0.450255 \\\\\n", - "\t18 & -0.167326 & 0.175327 & 0.287468 & -0.402653 & 0.551182 & 0.522205 & 0.436838 & 0.299565 & 0.58311 & -0.704416 & -0.73081 & -1.95141 & -0.933505 \\\\\n", - "\t19 & 1.41159 & 1.78722 & 0.843976 & 0.481278 & -0.0887674 & -0.499578 & 0.304196 & -1.23884 & -0.153476 & -0.870486 & 0.0955473 & -0.983708 & -0.356345 \\\\\n", - "\t20 & -1.42997 & -0.490147 & 0.27273 & -1.6103 & 0.990788 & 0.711688 & 1.18858 & -0.371229 & 1.24703 & -0.0389162 & 0.883496 & 2.58988 & 3.3354 \\\\\n", - "\t21 & -0.147247 & 0.123284 & 0.617549 & -0.187131 & 0.256438 & 0.17795 & 0.412612 & -0.244809 & 0.0947625 & 0.723017 & -0.683948 & 0.0873751 & -0.26221 \\\\\n", - "\t22 & -0.187113 & -0.270777 & -1.01557 & 0.0602851 & 0.27242 & 0.869133 & -0.657519 & 2.32389 & -0.999936 & 1.44672 & 0.971158 & -0.358748 & -0.439658 \\\\\n", - "\t23 & -1.82434 & -0.93348 & 1.29474 & -1.94545 & 0.335847 & 0.359202 & 0.513653 & -0.0731977 & 1.57139 & 1.53329 & 1.82077 & 2.2274 & 1.50063 \\\\\n", - "\t24 & -2.29344 & -2.49162 & 0.40384 & -2.36488 & 1.41053 & 1.42244 & 1.17024 & 0.844767 & 1.79027 & 0.648182 & -0.0857231 & -1.0279 & 0.491288 \\\\\n", - "\t25 & -0.434136 & 0.740882 & 0.699576 & -1.02406 & 0.759529 & 0.956656 & 0.6333 & 0.770734 & 0.824989 & 1.84287 & 1.91046 & -0.502317 & 0.13267 \\\\\n", - "\t26 & -2.1921 & -2.49466 & 0.354855 & -1.93156 & 0.941979 & 0.978917 & 0.89486 & 0.463239 & 1.12537 & 1.70528 & 0.717793 & 0.645888 & 0.783968 \\\\\n", - "\t27 & -1.46602 & -1.24922 & 0.307978 & -1.55097 & 0.618908 & 0.662508 & 0.475957 & 0.484719 & 0.401565 & 0.55988 & -0.376938 & -0.933983 & 0.390013 \\\\\n", - "\t28 & -1.83318 & -1.53269 & 2.55674 & -1.51828 & 0.78941 & 0.908748 & 0.649972 & 0.668374 & 1.20058 & 0.277963 & 1.2505 & 3.3137 & 2.22036 \\\\\n", - "\t29 & -0.784547 & 0.276583 & 3.01105 & -1.11979 & 0.920824 & 0.750218 & 1.26154 & -0.403364 & 0.400667 & -0.217598 & -0.72467 & -0.391945 & -0.650024 \\\\\n", - "\t30 & 0.464456 & 1.33264 & -1.2306 & -0.357976 & 1.1825 & 1.54316 & -0.60339 & 3.38309 & 0.823741 & -0.129951 & -0.65798 & -0.499535 & -0.414477 \\\\\n", - "\t$\\dots$ & $\\dots$ & $\\dots$ & $\\dots$ & $\\dots$ & $\\dots$ & $\\dots$ & $\\dots$ & $\\dots$ & $\\dots$ & $\\dots$ & $\\dots$ & $\\dots$ & $\\dots$ \\\\\n", - "\\end{tabular}\n" - ], - "text/plain": [ - "6670×13 DataFrame. Omitted printing of 8 columns\n", - "│ Row │ Trait1 │ Trait2 │ Trait3 │ Trait4 │ Trait5 │\n", - "│ │ \u001b[90mFloat64⍰\u001b[39m │ \u001b[90mFloat64⍰\u001b[39m │ \u001b[90mFloat64⍰\u001b[39m │ \u001b[90mFloat64⍰\u001b[39m │ \u001b[90mFloat64⍰\u001b[39m │\n", - "├──────┼───────────┼───────────┼────────────┼────────────┼────────────┤\n", - "│ 1 │ -1.81573 │ -0.94615 │ 1.11363 │ -2.09867 │ 0.744417 │\n", - "│ 2 │ -1.2444 │ 0.10966 │ 0.467119 │ -1.62131 │ 1.05668 │\n", - "│ 3 │ 1.45567 │ 1.53867 │ 1.09403 │ 0.586655 │ -0.327965 │\n", - "│ 4 │ -0.768809 │ 0.513491 │ 0.244263 │ -1.3174 │ 1.19394 │\n", - "│ 5 │ -0.264415 │ -0.34824 │ -0.0239065 │ 0.00473916 │ 1.25619 │\n", - "│ 6 │ -1.37617 │ -1.47192 │ 0.29118 │ -0.803111 │ -0.26424 │\n", - "│ 7 │ 0.100942 │ -0.191616 │ -0.567421 │ 0.378571 │ -0.246656 │\n", - "│ 8 │ -0.319818 │ 1.35774 │ 0.81869 │ -1.15566 │ 0.634484 │\n", - "│ 9 │ -0.288334 │ 0.566083 │ 0.254958 │ -0.652578 │ 0.668922 │\n", - "│ 10 │ -1.1576 │ -0.781199 │ -0.595808 │ -1.00555 │ 0.789829 │\n", - "⋮\n", - "│ 6660 │ -0.52427 │ 1.04173 │ 1.13749 │ -1.27262 │ -0.607382 │\n", - "│ 6661 │ 1.32516 │ 0.905899 │ 0.84261 │ 0.869143 │ -0.691461 │\n", - "│ 6662 │ -1.44368 │ -2.55708 │ -0.868193 │ 0.390457 │ -0.0331364 │\n", - "│ 6663 │ -1.8518 │ -1.25726 │ 1.81724 │ -2.17285 │ 0.948696 │\n", - "│ 6664 │ -0.810034 │ 0.0896703 │ 0.530939 │ -1.14212 │ 1.26049 │\n", - "│ 6665 │ -1.22395 │ -1.48953 │ -2.95847 │ -0.645307 │ -2.67022 │\n", - "│ 6666 │ -0.282847 │ -1.54129 │ -1.38819 │ 1.48327 │ -1.23406 │\n", - "│ 6667 │ 0.475008 │ 1.46697 │ 0.497403 │ -0.46861 │ 0.496553 │\n", - "│ 6668 │ -0.408154 │ -0.325323 │ 0.0850869 │ -0.182984 │ -0.46577 │\n", - "│ 6669 │ 0.886626 │ 0.487408 │ -0.0977307 │ 0.830857 │ -0.168597 │\n", - "│ 6670 │ -1.24394 │ 0.213697 │ 2.74965 │ -1.80285 │ 1.11401 │" - ] - }, - "execution_count": 63, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "#using Pkg\n", - "#pkg\"add CSV DataFrames\"\n", - "#using CSV, DataFrames\n", - "\n", - "cg10k_trait = CSV.File(\"cg10k_traits.txt\"; \n", - " delim = ' ') |> DataFrame\n", - "names!(cg10k_trait, [:FID; :IID; :Trait1; :Trait2; :Trait3; :Trait4; :Trait5; :Trait6; \n", - " :Trait7; :Trait8; :Trait9; :Trait10; :Trait11; :Trait12; :Trait13])\n", - "# do not display FID and IID for privacy\n", - "cg10k_trait[:, 3:end]" - ] - }, - { - "cell_type": "code", - "execution_count": 64, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "

13 rows × 8 columns

variablemeanminmedianmaxnuniquenmissingeltype
SymbolFloat64Float64Float64Float64NothingInt64DataType
1Trait10.00221138-3.204130.125013.47940Float64
2Trait20.00135253-3.511660.03351734.913420Float64
3Trait3-0.00129591-3.93844-0.0007821627.91630Float64
4Trait40.00230893-3.60840.2281653.127690Float64
5Trait5-0.00179039-4.148750.03103432.717180Float64
6Trait6-0.00119598-3.824790.0362422.589730Float64
7Trait7-0.00198906-4.272460.0698012.653780Float64
8Trait80.000614075-5.62549-0.03863015.80570Float64
9Trait9-0.00180965-5.381970.1065712.571940Float64
10Trait10-0.000437029-3.54851-0.09665076.537820Float64
11Trait11-0.000615918-3.26491-0.06804374.262410Float64
12Trait12-0.000588783-8.85191-0.14109913.21140Float64
13Trait13-0.000151238-5.5921-0.14102224.17440Float64
" - ], - "text/latex": [ - "\\begin{tabular}{r|cccccccc}\n", - "\t& variable & mean & min & median & max & nunique & nmissing & eltype\\\\\n", - "\t\\hline\n", - "\t& Symbol & Float64 & Float64 & Float64 & Float64 & Nothing & Int64 & DataType\\\\\n", - "\t\\hline\n", - "\t1 & Trait1 & 0.00221138 & -3.20413 & 0.12501 & 3.4794 & & 0 & Float64 \\\\\n", - "\t2 & Trait2 & 0.00135253 & -3.51166 & 0.0335173 & 4.91342 & & 0 & Float64 \\\\\n", - "\t3 & Trait3 & -0.00129591 & -3.93844 & -0.000782162 & 7.9163 & & 0 & Float64 \\\\\n", - "\t4 & Trait4 & 0.00230893 & -3.6084 & 0.228165 & 3.12769 & & 0 & Float64 \\\\\n", - "\t5 & Trait5 & -0.00179039 & -4.14875 & 0.0310343 & 2.71718 & & 0 & Float64 \\\\\n", - "\t6 & Trait6 & -0.00119598 & -3.82479 & 0.036242 & 2.58973 & & 0 & Float64 \\\\\n", - "\t7 & Trait7 & -0.00198906 & -4.27246 & 0.069801 & 2.65378 & & 0 & Float64 \\\\\n", - "\t8 & Trait8 & 0.000614075 & -5.62549 & -0.0386301 & 5.8057 & & 0 & Float64 \\\\\n", - "\t9 & Trait9 & -0.00180965 & -5.38197 & 0.106571 & 2.57194 & & 0 & Float64 \\\\\n", - "\t10 & Trait10 & -0.000437029 & -3.54851 & -0.0966507 & 6.53782 & & 0 & Float64 \\\\\n", - "\t11 & Trait11 & -0.000615918 & -3.26491 & -0.0680437 & 4.26241 & & 0 & Float64 \\\\\n", - "\t12 & Trait12 & -0.000588783 & -8.85191 & -0.141099 & 13.2114 & & 0 & Float64 \\\\\n", - "\t13 & Trait13 & -0.000151238 & -5.5921 & -0.141022 & 24.1744 & & 0 & Float64 \\\\\n", - "\\end{tabular}\n" - ], - "text/plain": [ - "13×8 DataFrame. Omitted printing of 2 columns\n", - "│ Row │ variable │ mean │ min │ median │ max │ nunique │\n", - "│ │ \u001b[90mSymbol\u001b[39m │ \u001b[90mFloat64\u001b[39m │ \u001b[90mFloat64\u001b[39m │ \u001b[90mFloat64\u001b[39m │ \u001b[90mFloat64\u001b[39m │ \u001b[90mNothing\u001b[39m │\n", - "├─────┼──────────┼──────────────┼──────────┼──────────────┼─────────┼─────────┤\n", - "│ 1 │ Trait1 │ 0.00221138 │ -3.20413 │ 0.12501 │ 3.4794 │ │\n", - "│ 2 │ Trait2 │ 0.00135253 │ -3.51166 │ 0.0335173 │ 4.91342 │ │\n", - "│ 3 │ Trait3 │ -0.00129591 │ -3.93844 │ -0.000782162 │ 7.9163 │ │\n", - "│ 4 │ Trait4 │ 0.00230893 │ -3.6084 │ 0.228165 │ 3.12769 │ │\n", - "│ 5 │ Trait5 │ -0.00179039 │ -4.14875 │ 0.0310343 │ 2.71718 │ │\n", - "│ 6 │ Trait6 │ -0.00119598 │ -3.82479 │ 0.036242 │ 2.58973 │ │\n", - "│ 7 │ Trait7 │ -0.00198906 │ -4.27246 │ 0.069801 │ 2.65378 │ │\n", - "│ 8 │ Trait8 │ 0.000614075 │ -5.62549 │ -0.0386301 │ 5.8057 │ │\n", - "│ 9 │ Trait9 │ -0.00180965 │ -5.38197 │ 0.106571 │ 2.57194 │ │\n", - "│ 10 │ Trait10 │ -0.000437029 │ -3.54851 │ -0.0966507 │ 6.53782 │ │\n", - "│ 11 │ Trait11 │ -0.000615918 │ -3.26491 │ -0.0680437 │ 4.26241 │ │\n", - "│ 12 │ Trait12 │ -0.000588783 │ -8.85191 │ -0.141099 │ 13.2114 │ │\n", - "│ 13 │ Trait13 │ -0.000151238 │ -5.5921 │ -0.141022 │ 24.1744 │ │" - ] - }, - "execution_count": 64, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "describe(cg10k_trait[:, 3:end])" - ] - }, - { - "cell_type": "code", - "execution_count": 65, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlgAAAH0CAIAAABuMsSDAAAABmJLR0QA/wD/AP+gvaeTAAAgAElEQVR4nOydeWAURfb4X/fck8mdyQUBBULCIQIiqyEgh6IiEF1RUVjQVVYEdgFRyQIGD+SUa/2xX5dL1FXDIRhcBbkRiHiAHAmQcEguyDn3Pd1dvz8axjgzmcz0TKZnJvX5AyZvuqte95uq19VV9R6BEAIMBoPBYNorJN8KYDAYDAbDJ9gRYjAYDKZdgx0hBoPBYNo12BFiMBgMpl2DHSEGg8Fg2jXYEWIwGAymXYMdIQaDwWDaNdgRYjAYDKZdgx0hBhOuUBQ1bdo0pVI5aNCgmpoaVqhWq8eMGZOQkDB27Fi1Wu1BiMFgWLg7QtwIMRh+WbNmjU6nq6ioyMnJWbhwIStctmxZ586db9682alTp+XLl3sQYjAYFoJziLX333//zJkz69evX7hwoVqt3rhxIwDk5+cbDIaVK1fOmTMnOjp6yZIlLQkdlJeXZ2RkyGQyDjrQNE0QBElyd+eRUQK/lJeXp6eny+VyXy+BoiiBQEAQRMieZbfbRSKRT6cEk/79+3/00Ud33323Xq8vLy+/5557ACArK6uoqCg7O/vSpUt5eXllZWUtCR0cO3bs+vXr48ePb7VGhmEQQgKBwEsNfT0eIUTTtFAoDP7xAoEgrNtgRkaGVCr16W644uv9DM0SOMDdEQaqEfbv33/jxo39+/fnoINGo5FIJNycKItWqxWJRHK5nHMJOp1OIBBERUX5U4JYLJZKpZxL4Jf+/fuvXr164MCBvhpCrVbLZDJfL7ypqSk6OlosFvt0VkNDQ3x8vK+tq66uTqlUhmz/mJiYOGXKlPXr13fp0uWjjz666667AEChUDQ0NMhkMrPZnJKSotPpWhI6yMrKSk9P37lzZ6s1ms1mu90eExPjpYYWi8VqtcbGxnp5vNVqNZlM8fHxXh5vs9kMBkNCQoKXx1MUpdFokpKSXL9SKBSh/NDjGbYX7d27t16vT0xM5FyO3W7XarVu70/QSqAoSq1WK5VKziVwgLvXraio+OKLL4YNG8Y2QlZYU1PTuXNnAGDfw3gQOtDpdCNGjLh06RIHHbRarUQi8ceF6HQ6kUgkk8nOnTtntVqjoqJ69uzpUwl6vV4gEPjjSvV6vVKpDF9HGGR0Ol1JSYlIJLr//vt9HeFFGDqdDiFUWlq6bt26KVOmnDx5EgAQQuxtYZ+s2SPdCh1ERUWtXLnSG/cjkUjsdruXjs1qtZ45cyY6OvqOO+7w8oqsVqtYLPbJEQqFQu+PpygKALw/HuMnGo3mwoULANClS5fU1FS+1WkR7o4wUI0wOjr6ww8/5Ob/RSKRnyNCsVgsEom+//77p154hYyKTZOTl86d9qkEiUTi54hQIpFIJBLOp7c3Xn71nz/8WmKsvGCxWHwdF0YYSqVy1qxZaWlpM2bMWLNmDStMT0+vqqrKzMysqanp0KGDB2Fb8+aiZatXr35wxPA9RV8Gp0ZMqPHMi9OKT5+ndI3vv7Ng+vTpfKvTItzf+TRvhCUlJayQbW8A4NoInYQOCIJISEggeaWmpgZ6PWiYuIlCBC8KcLZCO6Sy4rph/DqCFGzdunXv3r18q8MnDz/88JYtW6xW6/r16wcMGMAKx4wZs3nzZoTQ5s2b8/LyPAjbmqtXr1J3/onGed7aMRXXrhqe+xD6P863Iq3AvQsO8UaIiWwYgXjq+59Of3Uu34rwyZIlSw4dOpSSknLw4EF2tRoAFBQUnDt3LiMjo7S0dMGCBR6EGAyGhfur0SVLlvzlL39ZtmzZ3XffvWnTJlZYUFAwYcKEjIyM/v37f/rppx6EGIxf2C2mMW/TX07lWw8+SU1N3b9/v5MwLi7um2++8UaIwWBYuDtC3AgxQeZ86YXJL8+oqKzkWxEMBhNR4NmpZlj0Nxua0u7s3rlrd4Zh+NYG48y/P1z/qyjLRjmvt8JgMBh/wI6wGZobNoaofemrqutXOW+vDCMGDRpE3Gbq1FvvGEM9NlBKNyC83Z2NwWAw3oAd4R8RiiElk28lggFC6NKlS9XV1Xq9Xq/XOxbf4wBdGAwm4NA0bbPZXLfPhQjYEbZT6urqbDZbXl5eWlraxIkTHaFGdu3aNWPGDIlEMmPGDEeoEbfCYFJZWdl74OBPv9ga/KoxflJ3o2b58uV/f2N+5+w+//73v/lWB8MDdP31WbNnS2XyyS9O4VsX9wQ1nlsIYjab//Of/5w6dYqmuYeGCUdqa2vvvffeVatWderUafbs2TNnzvziiy+AU2wgs9n873//u0ePHmzYDu8xGAwURdnt9laP3L9//290jEXmHLcJIUav13tZl1Ao9DXEmsFgkEql3mz0lMlkQY6OCACDBg0qLi5mP7/88ssffvghAKjV6kmTJp04cSI3N/fjjz9mo6i4FbYper3eaDKCvr6kpi7/rfeAFKK07BsuPx5Me4DR1qLxq0Eks9M/8K2Le7iPCMNyhsmFuW++PX/Lnh3HztE+duLhTt++fQ8dOtS3b9+EhISlS5d+9913rJxDbCCCIBQKBdGWAAApiyFEf4y/Y9LcqK1PSkmb+eprbVq7lxoGmRB/uT1s9JP79h8AhiaUXRBlI0QSkEYHoV4MhgMcn2EdjZCNOuh4Fmbb244dO+bMmbN8+XI20YRbIe/877sD6zd9dPr0r9T9M+BqcXt7R3z69GmLxZKTkwMAYrHYEeONQ4AuqVQ6adIkhULha6w7u93uZdBtqVTqxtnoGxmxnHpsvtl+TqFQeC7BbDYrFApfB21Go1GhUIRm6B/Hy+2ysrIRI0asX7+evZO7du0qKipi32Pn5eWxzc2t0IHNZjt48KA3UXYtFovdbvcmImDZ+V9JRRINAPD7ujOKoiwWi4ezrFar1Wr1fExzbDabT8dTFNXS8SKRyPssGZgIg2MLD+AME8MwNTU1dNCZ9dobX9uzbuqcmsStRnv16tUrV66YTKbgKBP8RapGo/GJJ564ePGizWZ79913H3/8VgykcIsNRIKwnYZpZV9ub9y4saKiIjY2dubMmaycw8ttu92+e/dusxdYLBaLxeLNkQiguQtkP9vtlDfn8gLeMdWe4TgiDOAMk8FgePnll8+cOcNBDX+yT9A0A70fJk7dCgf8hyYrie4/5CGrqu7r3V+1mh+KzT7h/TOp2xKSkpL8CR3Ogdzc3IULF44ZM0ar1Y4aNeqDDz5g5Tg2ULjAvtxmPy9durRXr17sZ27ZJ9auXRvY7BMEQQCwg/jf/5XJpJ5raW/ZJyiK+sc//rF9+/bu3btv27aNfd0SIrO87QqOjjCAjTAmJmbbtm3Jyckc1GDf6XFzIbdeg9x+3/aH/y0645ra2FVD4uPjW1VMKpX6mX1CKpUGP4sCQRDTpk2bNm2akzwcYwPdrKn+8ssv+/fvf+edd/KtS/AI4MvtgHP48GHKbm/3S/FaZ82aNTqdrqKiYuHChQsXLmQDxobRBFPEwPHV6OnTpx3L1VwbIfiSfSJkYWyW/6xfX1BQ8NNPP/GtC6ZlKn89fubixOmv7du3j29VgkrIvtwuKSkZM36yxWr943sWjBs+//zz119/XS6XFxQUvPLKK6wwNLcwRTYcn9mMRuO4ceOOHDnStWtX10a4ePFi10boJAx9jKr6j66QwqN7U1JSBg4cyLc6mBawGqH7YFIa1BfLoUDIvty2Wq3CmCTQNLVpLZFBoNKb6/X6p59++vvvvzcYDP4sOLDb7V7uR/KyBPYVoEMhq83a2NjouQSKorRaLYeV2DExMZxfrXF0hCHbCANM7vPCn/7LtxIYjBsi6eV2uyVQ6c3lcnl+fn5sbCxJkt7M4LYEu6k3ICVcvXq1vLzcbDbD7xNPIBKJWi2coiiGYTjo4M+iX4KXoJqPPPJIXV0d+7mmpiYpKYlbinaGYTjv4qqqqaHkSWBoBGk02K0AAGIZmDQQ3wEar0PSHaCqgphkwmpIkItjYmI86AAAjhX2WVlZhYWFPmmi0+nEYjG3JT984WTBxMREiUTiqyG8NB9N00ajUWW0IMoOEgWYtRCdDNqbEK0EkxqEUgAgSCIxShId3eJONZqmSZL0RsPmFqyrq1MqlaG5fcJPuLVBhBBCyPMNsVqttQ2NDGUHUgAECQQJdiuQJAjFcXKJ51Ue3pTf6vEe2iBFUWq1WqlUell+W5Oenn7q1Km0tLTa2tpu3boZDAYAyMzM/PbbbzMzMy9fvvzYY4+Vl5e7FbpaUCwW+3T3XEEIMQzjj0dxlFBRWQUCEWO3QmwKGJpAGg1ARIElubWbjxDq0qVLkF//8jOdXVJSsmnTJm4LZEKZ8vLyxYsX861FMMAWDHewBUMBNr35q6++6prevNUJpgi24LvvvhvkSnlb19WrV6+OHTvyVXsbEZFDh5bAFgx3sAV5x8/05tiCgQIvcG7vlJSU/OlPfzIajeyfeA8TBhM0cHrzECGcnp4wAUer1T7//PMmk8khCZFIlRgMBhM0sCNsvyCEnn/++fz8/OZCvIcJABjf4VHbkpKS5vEcwi7wPQbDO/46QtwIw5dly5Z17dp13LhxzYUc9jBFGDRNN9ymqampwTusVisv2uIxPQbjP345QtwIw5fDhw/v3bvXNVAThz1MEYZAIEi5jVKpTPEObvt//ASP6d2CELK2DJuwwhUcdLs949eOE9wIXUEIWXwn+N7l4MGDR48eFYvFrIcjCOL48eMQWUHyIh48pncLQsjoO76mlcZEEtxXjQa2ET7+1Piq2gbOyviEmISvv9yalOSc7jwgIITYYAreY7FYgpx6AgAWLVq0aNEi9jNB/B5XgXOQvIixYLjAjuld1xxyHtNHjAVJkkxISHD7FUVRBEG09G24EzEWDD4cHWHAG+HRY8c1ectBkchNH5+QbproT9Ykz5Ak6evuAoFAIBKJ2kgfX+EcJC9iLBguOMb07J8EQRw7diw3N5dz9glswXAHW5AzHB1hwBshAEC3HIgPxms3UtxOU7m2RPMwe37tYcIWDCIBH9MDYAuGP9iCnOA4R7ho0SJ0GwBACOXm5kIIpIAJODRNZ2dn860FhiNFRUW9e/eOi4sbMmQIG7Mx4ikoKDh37lxGRkZpaemCBQs8CMOCvXv39uzZMy4urmfPnu0t01Yk4bS/INQIcGSZCMs+sXbt2s8//7ysrIxvRTBcqKysnDhx4r59+/r27btu3boXXnjhxIkTfCvVJgRsTB9iMAwzYcKEbdu2DRs2bOfOnS+88EJNTQ3fSmF8xnV/QagRgA31ro2wurp69+7djjwaboWhxpQpU1avXs1+fvHFF1etWgUAffr0efPNN3nVC+Mtrha8du3a+PHj77//fplMNnnyZPxAE+K4WpCiqE8//XT48OFGo1EikcTFxfGrIcYzbntRt/sLQg0cWeYWTz755K5duwDAarUWFRWNHz8eAIYNGzZ69Gi+VcN4hasFhw4dumHDBgCgabqgoOCZZ57hW0eMJ1wtKBaLR40aZTQaY2Ji8vLyWGtiQha3vajb/QWhBnaEtxg+fPiFCxfq6ur27dvXr1+/9PR0vjXC+EZLFjxw4MC9994bGxu7du1afjXEeKYlCyoUCoPBsGjRopkzZ/KrIcYzrhZsKXBHqIEd4S3EYvFjjz22e/fuwsLCiRMn8q1OMHC7DCF8g+S5WhAh9M9//vOdd94pLCxcunSpUIhzrYQ0rha8fv3666+/DgBRUVEvvvjixYsX+daxTYiYQJWuFmwpcEeogR3h74wbN+6zzz7bv3//E088wbcubQ67DOGDDz5QqVTvvPPOCy+8wMrDOkiekwWLi4t37dq1e/fu9PR0g8HApv/GhDJOFkxPT9+0adPRo0cRQlu3bu3Xrx/fCgaeCAtU6WTBlvYXhBrcHWGEjScA4KGHHjp16tTw4cNjYmL41qXNaWkZQlgHyXOy4JEjR8rKyuLj46Nvw7eCASbi26BYLN61a9err76amJhYWFgYeXOEkReoMkx7UY6OMCLHE1KptGfPnq7vRZsvi40YWlqGENaRKp0sOH/+fPRHvCmEYRjtbXQ6ndY77HZ7W16Zez3bQxt84IEHTp06pVKpiouLI29Hb+RFi22pFwVfOlKGYep8x58MMBxnTRzjCYPB4DSeKCoqYh9b8vLy2DlSt0InYuIS7DtmE2I55yvxHqvezROx3W4/f/58VVXVyJEjg6BDiMAuQ1i7du3MmTN//vln8CNIXsRYkCAIR8Q7kUgkEolIsvXnRW+OCSy4DbYEm33C7VcURbHZJ1y/8tLQASTggSojxoIkSSqVSg5nca6RoyNkxxMGgyEmJqb5/CfnZ5lD3+z64YcfuCnjMy8/6hp19+uvv37llVc+/PBDR9w4bjAMo9frfTqFvY3+VMqB69evr1u3bsWKFewyhMWLF7NyzkHyIsaCBEHI5be6EplMJpfLg+/kvAG3wZZgs0+4/YqiKKPR6DbGvUKh8LNeXwl4oMqIsSAE/8kS+YfBYHjvvfcGDBjA/imXy81mM/tDlMvlLQk7dOhQVVXlZ9UhyJkzZ+666y69j9y4cUOv1wdZVavVGh8ff+TIEYZh1q5dm5uby8pnz56dn5/PMEx+fv5rr73WkjCCLdinTx/Hn7W1tTRN86iPN+A22BwnCzpht9vr6+uDqY+XNO+KcRvs1atXkCvl6HVbWtaMs9kRBKHwkaioqOCv7G9pGUKERaqMYHAbjFRwGww+HB1hS8uaIy/odgTjdhlC+AbJa2/gNhhJoIgIVBm+cHSEeDyBwfALboMYTKDg/kaOHU84CSMp8j0GE+LgNojBBIRQXA6HwWAwGEzQwI4Qg8FgMO0afsIQy2SygoIChUIBADabTSQSsXtFA4jdTjEMDQBACgExAAAkCQwNAjFQVhBKgLKCQAQMBYSghQNsIBDePgABMAAEAcjDFpnGxsbAXkXI4mRBoVAYqH0/FEXRiACGBoEQaDsIhEDTQJAACAABIXD5iv3lsF9RIBABbZdIJNxqb7cW9L8NWq1WAAIcZSD0+5/sZ4Df/yRIoYAUCAT+1OgWbMHgY6coBhFe9KK3D6BpEIqBsrptp7xYkEB8xA+7cOHCgQMH2M8rVqx49tlnO3bsyKEcs9ksFAodcUCa8/bi5ap+48mTnzOZg6H+CikUMspuxIUD6NG5sHUOjF9FfFWAHvgb+VMh03UQNP5GEsCk9iBK9qDH5sHWOfD0CmL32yj3r+SpHcydfwJVNYloJnt4wunPFs6f27wii8VCkqTDOyqVymeffdanq9DpdGKxWCqVcrgDfOFkwSeeeKJz585uDeEBk8nEhm5pLtz99dcH68Xk1WJm4Hg4uhFyJhKndqEOvUmrDkwaptM9xKVDaNBkOPj/4L4JxPk9KOlOkgDQ3GC6DiJK9qAhU+DIhx07dsxISx3/9O+RqwwGg5db45tbsK6uTqlUhuaGej/h1gZtNhtN064b0i0Wy7w3CxiBBAnFIFYQdiMyakl5NCOQkGIp01RFiGVIKAaJgrCZkFEtSu36zLAB9957r1M5bOQXR0CDVqFp2mKxNE/dAB7bIEVRarWaQ9SSEMTVgmlpaa53wyfc3k8PfL7jqx+PHQaBGPrleduLdriLOPs1Gj2f2Pb6yvdXuD4MMQwjk8lefvllzlfBhSDvW3SlX79+p06d4nauWq02mUxOwsOHD7/zzjtxaZ2h4GdBRm+Y9CEMHC8YPBn+uplM7Q7/sQAAbLCRMUpYcFLQqQ9MXAf3TxDmTIQpn5Ip3WCDDQgC/mMmY1Ng3nFB577w3FrImSy871lYeKpz1l1O1Wm1WoPBwE1/RwnsZucg89VXX/Xq1Ss2Nnbw4MFlZWWsUKVSjR49Oj4+fsyYMSqVyoPQQb9+/Y4cOeJqiFZRqVSuF/763Hx44l1BYga8ugeEEnilkIhLh6FThfc8LrjzHnjmfTIuDeYeAYKElz4hlV3g/omCnAnCTn1g4joyRgnzjgNBwHNrs/r9qXmx9fX1drvdVw3DYkO9/3jfBo1Go0ajcZVrtVqxPJqMS4OUbtDlPjI1E8RyQXIXyOgj6J4LIikZlwbJ3aDrfWRqdxDLZHeP/OSTT1zLsVgsTU1N3mtutVobGxu9Pz5kN9T7CWtBX++GKzabraGhwfvju/buDwtOCp170a6eetGphaTyTthgI4Uiq9XqWiYvNuL+qFtUVNS7d++4uLghQ4aUl5ezQt4j39M0PXzEgwt/smh1ujatKNyprKycOHHihg0bbt68OXbs2MgI2fw7nfvzrUEwCM02iMGEHRwdYch2o4gNVpT3FkEGfvohkrh27dr48ePvv/9+mUw2efLksrIyVs4hBQzDMDdv3qQDBMME6l09CpRK3oCCPsUQsm3QO5Ber8cZIjEhAsfFMo5uFAAmT568dOlSVs458j0myAwdOnTo0KEAQNN0QUHBM888w8o5hGw2GAyzZs3KycnxdZpTo9HIZDKnCXOLxQyggFtuBTX71y2I/cf1CJqmm5qaHH+q1WqGYXwNZadWq0mS9GaOMDY2lvMKHW4EsA3SNP3LL7/06tWr1UqtVqvdbnfN3uAxA44b81Gaxhn/mLn/8PeF//3YqRyWVjVhsdlsLWWTcEtIZZ/AhA4cHWFgu9EXXnjh8OHDHNTQarUSicRsNrN/njhxoqamxn2/2VJfilr4382BCABohlGpVM3ler1eIBD4kwpLr9fHx8fzsljmwIEDb7zxxsiRIxctWsRKkO8pYGJiYjZt2qRUKt0G9feASCSSyWSOCy8vL79/2EOahjoYveD24kOi2b9uubUW0fUIgUCYnJz8+3EEER8f76sjRAiF7GKZALZBs9m8bt06p6x4bjGbzXa73ek23rx585kXptopO+Ep5cAfWhZj1qIeI0x2yilNhNVqNZlM3j9S2Gw2NrO0l8eHVPYJzC+//CISiXr06MGufeURv7ZPBKQblclk06dP57bSyW63SyQS9md97ty5Z1+aTnS9z/07qpb6UqKV750OJQnCSVWapgUCgZ8rtYIfdBshNG/evBMnThQWFnbv3t0h55wCxn9KSkqojn2RrL6Nyo9IAtIGFQrFxo0bXbPquGIymex2e/NYl7/8enbSCy9e0QFCqIVG1OLTjFgkdqrUarVKJBJvNGFhtw14fzxFUQRBeH88pu1gRPJHJ//dcqP88P7vcnJy+FWGY/8bwG5UIBAMGDCA22slyW2AdSfxadpJm+Dkl9wuCgA8OMRbrZkgnFSVSCQCgcCf12ISiST4Y47i4uJdu3adPHlSKBSyUzXsQxkbnXnx4sWuIZudhG0CQQJPe6HCjhB5lFn6/uqLNU2CRC7bnzDhS0lJyfnz5w0G33Kv/gGLXjfn+9h/PRg4pbjDsf9lu9Hdu3enp6cbDAbHpDeOfB8uHDlypKysLD4+Pvo2rByHbA4XQqgNynAyBI6E77rf+wYPfXldUX1DhIQv4OgI22k3ajM1aPW9B+YOuD+Xoii+tfGL+fPnO+2kYeU4BUy40E7bYAQRvut+EUJGTZP++U8JAT+xyQIOx8uYP3/+/PnzXeURHvleXWO2WEuHLyLff4hhGL61iSiqq6srKioQvqte007bYAQRwHW/CCGtVsvchrNKrZZgNBqnvvrPK5fLOVfhDAKnGjlfhT8TTOHtz2vr6lb/6/8JBMIRQwd3SE8PQo2ESIq6DcLzWIGlqamp/6BhOp2WzugXmBJNmuuV1fGpHadN+et7774TmDIxmIASwHW/Op0uLy/vwoULBoPBH0dot9v1er2HTbFnz5796tu9pqfXwNH9DiFy+uBydrMDnDdEIYQ0Gk1DQ4NDQlGUVqvloHxcXBzntRrh7Qjnvvn27goGKKvNbn9p8kS+1cFwxGQymcwW658mCW+cD0yJ6hq7SG7903MNf9zrggkpbtZUffbZZxMmTOBbET4JyLrf2NjY7du3Jycny2SyxMREzsrY7XapVJqUlNRcaDQaN2/eDADV1dUMw4BYDj0ean4A4fTBZZjQ7ADnJcQESSQkJKSkpDgkFEWJxeIgx4MNb0cIANDrIVBVF+39/PDRY2YzDwHEMSGKQASyWADsCEMVXf2vNXV/mzaj3TrCEFn364EPNmz5cP2Gmzdv6mI6gbYWgCQT0mk/9kyHLKG4U9hnqs+r4rOu3fEw5fKghAl9Rj4xvlOnTiaziW9FMD6z7+ChM2fPcjyZoYnU7nwkvwkVQmjdbwusWr3mwl0v6mgBfe+zdMd+0HWgfeAET4GewpaIcIQAREwKKLvyrUX4QdN0dnZ2c0nwl27/eOwI/H1XRLauiGfc089eq6nlW4twJZTX/TY1NV2+fJmi7JDRB0ThlCGOG345wlDoRjGcWbt2bU5OjiPcNkvoL93GNIf3NkiKucdUAoTspPi+B0ZcvHgxcBqFDaG5hen8+fP79u3rec999wwbVV1T03YVhRTcHSG/3ahKpTp58qRarQlIae2TPn36vPnmm05CDtknQpn6utrjx483X5MWSYT9owxDU0ZtyW83uK0SxAScCxcu5D37/NOvL62vvKZ/6wIp8C3VdvjCfbFMnz59unbtOmbMmObCoGWfeHzii2cuXTU21EDGk/6X5isIiH/OXyAUCKZPeyUuLi74CgSEYcOGuQo5LN02Go0FBQWFhYUWi8UnBTQaDZuZ0yFpYd01l+wTAADV578pObb3wOE1S98dMWIEAPga01Wr1QqFQm/2JykUCpEo2L1GoNogTdPnzp276667Wq3RfhvnLzy92fb41pskhfIYiqLYMlss31d9WoCtyO3xAoEgNKOrBxO9Xk/EpmhfKYJp4dqtcYO7IwxUN6rRaO67776Kigqfar9WXqaf/JlgfWvrzRydbIvZJ5y2vjgf53YHDKKpVVeiJCc25Q7KycrKEggEcrncB+3/iF6vT05O5iX7hCsclm6LRKLBgwdLpVJfs09oNJrS0lLa07Ynb7NPuMekQf3yRHa9WCyWyWQymcxXR8helDf9Iy99aKDaoNlsXr58+WsWOJUAACAASURBVOjRo1utkc0+4fgTIdSKn/udlvO6IGQwGNhBIZtNQiDwNpmozWYzGAzem5WiKL1e7zbLRExMDM4+EXwYu23Hjh2nTp164IEH+vTpw5caAd4+waEbjYuLKywsTEtL86kitqkQrW5sdxzQYvYJp60vzse1uAPm4dekZXsTExNTU1P9zD4RFRUVOi2Qw9JtsVj80EMPSaVSX335gsXvf/XtPpPp90Q8nOMUeDqRIEQikUQikUqlvjpC9qzwGihwzj7htHvMLc2zT5SUlDAM08qzyO/ftRzOniDi4uLY2q1Wq1Qq9Sn7hEQi8X7nHEVRAoHAmyvFBAeTqv7/HbtOfHlgBUI8OsIAt3C2xwQA127USdic4CchwrREMJdu/3btmunPK3CYnsDCrQ36ypUrV3JHjjaZTV6PCDEYNyCGtj84i8gawq8aAXaEobMDBsONUFi6jfGH4LRBvV4P8jg/xvC/Y9Zrc3IGvf322/4XheFMY2PjY+Oe+8fr/7RYfZvpjwwCPBQrKCiYMGFCRkZG//79P/30Uw9CTIjgFFcwaCGbV61adePmDegZ2FJbpLa29vjx4xKJZNSoUUGqkifCrg1SJi3qhaPY88ae/YemzXxVp27UJvWilXeR1FG+NeIBfx1hkLtRhNDs+W8dO3a8tp7/POaIoa9cuQIAHTt29GeOsB2ybNmy+Rt2oca6YFTGUGfPnl2wZGUTKOiKs2ajH6lEQxJeHmWuXLnicZUTJmzYtOWT65l5cKVYkNgJMu6G8vboCMNpFQAA2Gy2D1YsOT3wDbud/3SAZq1q6ty3ho8au23bNr51CRsoilq0dMWOnV/RPR4kxL6tMuWG9cblD7btrbz+m/HPK/B0VkD48ssvn5v8V5PJHMAyjx8//tFHHwWwQIwPJHZq59mVw8wRArtStMfw1teLtj20xWR+fgsMfIZvRcKJ/fv3L163+ZfrjRCsKJPIZrY99BpBersiH9MqZWVldIfWNx36QMPVQ2evTn1tfkp6R40GR8kIEqd/PfPa3Pzz5wOU8sU/dDpdbW0tX6EVws8RYsKXL7Zt/8+GTRCXBvEd+dYFE0oggLg0Sqyw2Gk8WRg0nhj/l5WnzVdq+J9momuvvPXe0s7dsv827e+8KBA2jrC+vr5f7oj0O7vTdMi1E6vVqtfrfY2r0t5obGycNPn5onKdzeZtHJAAgxhKIBk5dtyChXiBInfWf/LFijUftEnRpADvpQkCCKFF768dOXZcQ2Mj5D5PymL41giQvoF+/F3bM6vtPPXuYeMIL1++fK1Oo3q5KNT2LTH1vxW89XZCUvLkF//Gty6hjlAig245vFVvt9Am7X7o+e3B9rgcwH8oinr2penzCxZqGHFbtUKEqqqqHAmJMG2BzWZbmP/a/g5PWSIxsyA3guEIAxX5nhBJIDEjgIoFBEZbx4xbRk1YV3Lh4mtz81etWs23RoHHHws2NDQsXLhw6KN5A4aMsNlsbaShtxAC6J4LAGq1+s/PTX7gkTGDHxr152eea2xs5FmxNiYgbbC8vPzbnVtpIEEoCax6DmwgGPnIY2P+/NTHH3/SRlWEKYHqRauqqsrKyoAg4Z4/E0TIDYRomjIYDNevX7906dKlS5dQsFYSBONG+Bn5vr6+/qG8p6bNfiOk3z1e/eFSo2Xlz7p5S1Z36Jp9Z2Z2SGvrI/5Y8C9/m/7Ot6VHjxyu6PtiSEz/6BtLL1xMSe/4VWnj8V/OHEfd9ny332g0tn5iOOOPBWmaLr98ZVTen/vnPEDRdJv2nshiYOLSjpRc//Kb7w4fPrxy5craWpzvECBA+UPKy8vvvm9I7qhxDBOSOcwbr3/zv/8pU9PvH/7IwJGP9+zZ02wO5MpkDwQjtlmrke8vXrzIRr5nGObIkSMajcZoNHbt2rWhSbVz9zdlF0vPNCFGoSTNl6CmFACg6iwCBLXliLJBUyWYtYhAoKoGuwWqzjc/AOw2UFeDSQN2MaiqgLJCzTn2AEAI6i4juxXUNWBUI5KEpkqgrFB1FgCg+hwgBHVXkN0KmhtgUiHGDo0ViLLdPuA8IAbqroDdApobYGwi4tIgJdP2s+nGn5eQm55ftmxZbW1tbGwsQig2NjY1NbWxsTE9PX3voaMMg4Qk/PtfayQSCQDY7fYQDzLXqgXLysr69esnFAoZhtmxY8elS5dMJlNWVlZpaenVq1ch6ymidD8SCAAh0NaC1QiEAPT1wNDQ+BsggMbrwNCgb0BmPVhNoK0Fhob6awAIGiuAtoOhEZm1yGYBzQ1ADNRfBQBQVSDaBiY1ALplR4Sg7jIA3PoxmLVIIADKBqpKQAjqrwIguP4zJY1l4jJIRSJBCiG+A0MKps6cI2CoJ5/IO3fuXEJCQmpqalpamsFgqKmpGTBgQG1tLRuiMzs7GyF0uPinLZ9+JhXA4X17HHHGQzx3ga9t8Ny5c3FxcXK5/NT5i198uau2qgJEUkFiZzCcR4gByopsRoKyAgCiKbCbwUICQoAYsNvAbgLKBoAQQ4HdgkjBra8oG9hMQFkBsWdZwCYAhIBhgLKC1QR2KwAAQ4Esbv/BQ//btYMUisxm8x133KFSqUpKSrp169bU1JSRkTFo0CBWVZ1Od/XqVYTQ3XffbTAYGhoaDAZD//79GYZBCJEkSRAEQohhGIIgSJKkKEqr1Uokkps3byKExo4d62h6EWDBrKwsu91utVqPHz+u1+uvXbuWnp5+s7Z+/9HjNGX79VwpY7fqQUZN2gDLh7ddL4rsNo69aMNV6Hof1ftRYs9y/Qv/JVeMePb5KQatuusdnUQiUUbHDtnZ2Xa7vUOHDrW1tayxHnnkEUeIdn8sSARh7KlQKBoaGmQymdlsTklJ0el0zb/NzMwEgB9++AEArFbrgyMfphFhNhrkUrGNRoxQYtRpaCClErFer09ITKq7UZ3aIaOxvi4uIdGg04qlMqDtFAMyqUSn0ycqk2pr2APqY+MTTAadSMIegGQyqU6rTVQm11ZXpnbs1NRQHxMXb9TrRBIZwVB2BsllUp1Gm5icXFtTldIhQ9VQHxMXbzbohRIpwVA2CkXJZRqNOik5pbamKiU9Q93UoIiJMxt0ArFUAIyNoqOi5Bq1Oik5pa6mKiW9o6qxITo2zmLUS8QikiAsFmtMbGx9Y1NSSqqYhEPffct2o3q9PjExMSaG/ynrlmjVggzDHD16VCqVWq3WoQ89YjSZjXqtTCq1A0nZ7VJ5lF6jjktMVDc2xMbFm0xmkYAAgdhmtcgVCp1aFZ+YpFE1RcfEWKxWAQGEUGK3mOXRMZqmhoQkpUatUiiirDaKBCQQS80mQ0xsnKqhLjEpWavVyOUyO42AoUUSqdloiImLb6q7mahM1un0MomYAgJRdrFMbtBp4xISVA318QmJBqNRIhQwpJC22aRRt3TT1NcmJibpjOb4xES7XiWTK0iRWK9VK+Qys42SRyloqzkuLtZqtZrsNE0Im1sQQj53ga9t8GZtbWxcvM1kICRRao1aLJIQBIEArGajUCwRkgQiBbSdomlKLJEQBDCIsFlMIrFEQBCIJGmKoilKLJEAAIPAbjWLxBKSIAhSQFF2mqLEYjFBEAwQNotZKBILSQBSSFF2irKLRWKSFDAAdotZKCATExPMeq1IHtPUUCeTR4lkUXIB6t+vr81mq6ioMBqNNpphKKprlzsrKiqEEpndZr37rt6VlZU0Td9xxx0SiUStVtfX18fHxycnJ9tstt9++81isdgZkMuk+/buiSQLHj161GAwREVFPTTyYQagpqYmOTlFo9UJpHLGbm2pFzXqtGKZHFE2igGpRKzT65OUSp96UZNBL5JIgabsDJJLJVqNNiklxade1GLUC8RSEtE2ipbL5Rq1SpmSWnejOjmtg6qpMTom1mrUS8WimJhou8UsFArNVps8KooE+C5QFkRtj1wuN5vNCCGj0SiXy52+7dev36lTp7iV/NFHH504ccIf3f773/8ePXrUnxIKCwsPHDjgTwk7duzYv3+/PyW0Na1a8K233iouLva12HXr1p05c8bXs95///3Lly/7etbbb79948YNX8+aO3euTqfz9awQJFBt8MiRI5999pn39f70008bN270/vjz589/8MEH3h9/5cqVFStWeH/8jRs33n77be+PDx28sWBZWdmqVav8qeXatWvLli3zp4SKioolS5b4U0JNTc0777zjTwkcCMYbOc9JfNLT0//xj39wy+d39erVmJgYpVLJWbdr165FRUWlpKRwLuH69etSqTQ1NZVzCRUVFU8//fSDDz7IuYS2plULbty48auvvvLVEGVlZUql0vucOyylpaU7duyIjo726ayzZ8/u37/f14yJp0+fPnnypDePmW+99VZODn8LYlsjUG2wrq7OZDJt2bLFy3obGxvVavXWrVu9PF6tVtfV1e3evdvL4/V6fWVl5b59+7w83mw2l5eXHz9+3PWrCLAgRVHV1dV79uzhXIvBYKioqDhw4ADnEoxG47Vr1w4dOsS5BJPJdOXKlWPHjvl6oj8WDMar0VdffVUikSxevHjevHkURa1YsaL5tw0NDb/++mtb6xDi9O3bNzk5mW8tWgRbsFWwBcMdbMFwxx8LBsMRajSaCRMmnD17lo18z2b1xIQR2ILhDrZguIMt2KYEwxFiMBgMBhOyhO5yYQwGg8FgggB2hBgMBoNp12BHiMFgMJh2Tdg7wqKiot69e8fFxQ0ZMqS8vNync/0P3+dP7Q5KSkoiI8E9hwvx9QZyMxlnM0WMafzB890bNGgQcZsXXnjBg3Vcbed9yVOnTnUr8Vw+4YLn8iMGf363EdCjciHI+xYDS0VFhUKhKC4uNplMK1asyMnJ8en0uXPnTp8+3WKxTJ8+PT8/P8i1s2g0mnvuuSfcDYE4XQiHG8jBZJzNFDGm8QfPd49hmISEhOrqar1er9fr58yZ48E6TrbzqWSz2ewq8Vw+QkjfjDfffHPu3Lkeyg/kXeMVf363EdCjciO8G/nhw4dfeukl9nN9fX1iYqJPp3fv3v3ixYsIoYsXL3bv3j3ItSOEGIZ5/PHHt2/fHu69LbcL4XADOZiMm5kixjR+4vnu3bx5U6FQ3HPPPQqFIi8vr2vXrh6s42Q7n0quq6tzlXguv/lX586dGzFihN1u91A+1zsUWvj5uw33HpUzEdLIKYqaOnXqtGnTfDorKirKZDIhhEwmU3R0dJBrRwgtWbJkzpw5CKFw7239vBDvb6A/JvPJTBFjmkDh9u79+uuvw4YN+/XXX5uamiZNmiQUCj1YpyXbeVPy+PHjXSVelm+1WgcOHFhaWuq5fM53JqQI1O82THtUzoRfI8/KynJ6qbt///5+/frNnTu3+ROfN3gO3+clnGs/dOjQAw88YLPZUBj2ts2t4NOF+Gk+zibzqZawNo3/cLPRjRs3CILwYB23tvOy5Pj4eM+SlspHCC1evHjGjBkertdtaeEC55botgSWMO1R/SG8GznDMPn5+YMHDy4rK+Nwerdu3crLyxFC5eXlmZmZQa59/vz5TvO1x44d41AO73C+EA43kIPJONQSMabxH89379SpU46o942NjQKBwIN1nGznU8mpqamuEs/ls0KKojp16uQapb3V0sIR/3+3Yd2j+kN4O8Ljx49nZWWp1WrHlLhPp8+ePTs/P5+9+6+99lqQa29OxAw7fLoQDjeQg8n8NFPEmIYbLd29w4cPI4S+//775OTkCxcuWK3WmTNn9unTx9U67JHIxXY+lTx16lRXiefyWeG+ffsGDRrU/IpaKr8t7h6PcPvdRkyP6ivh3cgXLVrkzyJYtVo9atSoDh06jBkzRqPRBLn25kRMb+vThXC4gRxM5qeZIsY03Gjp7rEfGIZZt25d165dk5KSJk2aVFlZ6WodxylOtvOpZK1W6yrxXD4rfO6555zyLrVUfpvdQn7g9ruNmB7VV3CsUQwGg8G0a8J+Qz0Gg8FgMP6AHSEGg8Fg2jXYEWIwGAymXYMdIQaDwWDaNdgRYjAYDKZdgx0hBoPBYNo12BFiMBgMpl2DHSEGg8Fg2jXYEWIwGAymXYMdIQaDwWDaNdgRYjAYDKZdgx0hBoPBYNo12BFiMBgMpl2DHSEGg8Fg2jXYEWIwGAymXYMdIQaDwWDaNdgRYjAYDKZdgx0hBoPBYNo12BFiMBgMpl2DHSEGg8Fg2jXYEWIwGAymXYMdIQaDwWDaNdwd4aBBg4jbTJ06lRWq1eoxY8YkJCSMHTtWrVZ7EDooLy83m81OQoqiEEKcdYuYEsICJwva7Xb/y6RpmmEYPwthGIamaf+VCcgVhTJu2yCLn79h/03g580PyA8p9PFgQfDdiL5aDSFEUZT3xwdBJV/h6AgRQpcuXaqurtbr9Xq9fs2aNax82bJlnTt3vnnzZqdOnZYvX+5B6GD8+PEXL150EqrVaj8vW6PR+GqbgJeg1Wojvg+FP1oQIdTU1OR/mTqdzmaz+VmIzWbT6XT+K9PU1BTZDzRu2yCLn79hq9Wq1+s5nw4AKpXKH09mMplMJpM/CoQFHiwIvndlvt40mqZdRzieaWuVfIWjI6yrq7PZbHl5eWlpaRMnTnR0N7t27ZoxY4ZEIpkxY8bOnTs9CB0ghHQ6HRNKVFZWHj9+/McffwyaYpzt5w+BGtOHIGazubi4+OTJk1euXOFbF4xf1NfXFxcXFxcXGwwGvnXBtMi1a9eKi4s9jEpDHCG302pra++9995Vq1Z16tRp9uzZM2fO/OKLLwCgpqamc+fOAMAOAdmD3Qod6HS6MWPGOHVYTU1NFEUJhRzVAwCVSmWz2UQiEYdz7x083CJLMleWfLn1i4EDB3LWQa1WW61Wb3SIi4uTSCScK+KAY0wfGxsLAI5bzQ7fd+zYMWfOnOXLly9ZsqQlYSjzxsJFmz/fDgRxX+/uB/d8zbc6GO7kjZ/URAkt1ZeOHT7oT2PEtB01NTWPPPGMUVV/7szp7OxsvtXhAkdP07dv30OHDrGfly5d2qtXL/YzQoggCPaD492mW6GD2NjY7du3p6SkNBeSJBkfH++PIxQIBLGxsb46wnPnzjU0NDRU/YY+LIl5f3BCQoJSqayrqwMAuVzO+gzvEQqF0dHRYrHYp7OCg2NMX1ZWNmLEiPXr10ulUgDYtWtXUVERO3zPy8tjfZ5bYShz9epV02MLQSCmrn3Cty4Yv6itum5+91Lsv0fzrQimRUwmE0WKpMmd+FaEOxw9zenTpy0WS05ODgCIxWLHaCY9Pb2qqiozM7OmpqZDhw4ehCHI5cuXhzwyllB2QfD7hND6zR/PnDULgBj96COrVy4Xi8Wpqak8KhkoAjim12g09957b3V1NQAghBoaGvyfUdNqtVKplPMo2WqxAABoak4cO0qQJAFQWVkpEAi4lVZfX88wDPsw1xLx8fHskwQG037QarWlpaURMAvL0REajcZx48YdOXKka9eu77777uOPP87Kx4wZs3nz5sWLF2/evDkvL8+DMAQxGAwgj9P8/Tv42+/970+//GLLexfUNUV71u85dLRDivLyhXM8KhkoAjimj4uL2759e1paGnsASZL+PytIpVKZTMbZtUjYE1XV0PV++PtumBqVmprK+QUDQRApKSmeHSEG0w75y9SZe4p2IsqGFIlRimi+1eEOx8Uyubm5CxcuHDNmTIcOHdRq9bJly1h5QUHBuXPnMjIySktLFyxY4EEYZmjriLseNf/9G3ukrMQ+ffp0cXEx+9l1TA8ArmN6J2FzQsdJzJq3sM99Q04Un/xdRODNsmGP2aAbNGSoRBa1bds2vnUJJBRFTZs2TalUDho0qKamhhWG0YK1365dpeI6Ml3uC/dl1Rz7CIIgpk2bduXKlYaGho8//jgmJoaVx8XFffPNN9XV1bt373bMqLkVhj4Wg3bw4CEffbSFb0XaBKPR+MQTT1y8eNFms7mO6RFCrmN6J2Fo8q+l754fstBIu3HMBoNh586dO3fuPHnypOu34UgEr/t1gjJqqX/8T9Tn4ePHj2/btu3ChQt8axQY1qxZo9PpKioqcnJyFi5cyAo5bELjmZB5DuYMflhuEcqkQ3OPCNK6861ImxDJY/rMXELoZoHSzPw3J85d8pd/Ll/w7tLgKxVwAriXNzwQSW11V//zTfGkOW9t376db20Cw+eff/7666/L5fKCgoJXXnmFFXLYhEbT9JkzZ+xB548DQURRVPB1cODPPjTuyzLbBWTEPiiwY/pp06Y5ydnhuzfCcEIaff+IR69evWZ+9C2wmYq/fTchLUNEQtVvV0NzTa83RPa6XwBQqVQ6na55V4tsVvtjr0FdOY9aBZaKioovvvhi2LBhXbp0+eijj1ghhwVrZrN51apVY8eOdVsLG9PA+yX0RqMRIeSNX2GarRhACOn1eq1W600VbaFSTEwM5+aMHaFvIIZhLR1G73gxyKz9pd9MwcW/AwA0XLMl3Gl+8RPBguywntgI4LpfvV4/bty4n376ybUWlUpltVo59y8Wi8VqtXKI0GQyme4b/rBOb7JYLQ4havZtY2OjN+UYDAaCICwWS6tH+tONcob19KWlpevWrZsyZQr70p7DgjWFQrFx48akpKSWKvJpL5lUKkUIRUe3vvhF0GwBGkGQ8fHxHnQIjkrcwI7QFwyNNTdvJnfoZDPq/OkdMDzQcwQhvLUgiBBJIaEjv+r4TwDX/crl8nnz5rl9tqMoyp+9sGKx2Gq1cnhqpGlaVV9nXVUH02IcQsdMlEQi8bJMkiQJglAoFK0eyXl3jT8olcpZs2alpaXNmDHD8XI7rDehhSkR++rPV06cOPHzzz+3EuDUqEayWNuaRlLIJWANpk0pKirasmVLGA/xfCSA634FAkFmZqYoxPBw7QKBIODVkXzMgzz88MNbtmyxWq3r168fMGAAKwz3BWvhCHaEAAD/+9//Rj45cfZ7a03m1l+hYEKQPXv2PDdtzt8/Ogzh/LbTJyJ13W+7YsmSJYcOHUpJSTl48ODGjRtZYZguWLNYLL3v6rNy5Uq+FeECfjUKAHDz5k3oOdzUbRi5+22+dcFwwWg0CjvdrfvLRvi+vcRUc6z71Wq1o0aN+uCDD1h5QUHBhAkTMjIy+vfv/+mnn3oQYngnNTV1//79TsIwXbBGmXREr4f8zNjDF/6OCEtKSqKiohx/RuQeJgwmBIngvbwXLlw4efIkQi0sEbQaP9n21UNjnly99l/B1QvTGmEbvMIvvbVa7fPPP9880FzE7mHCYDBBoaysLOfBx8bPfNNmayEPYsWpa1GZB8zpR3/4ObiqYSIW7o4QIfT888/n5+c3F3LYChqOIGlMl55339G955kzZ/jWxV/wmB4TUmi1WohO0v/1c08HpWVDXIfv9u2PSUwZ8chjwVINE7Fwd4TLli3r2rXruHHjmgs57GEyGo2zZs1S/xGNRqP2Dy9LuH79+szZrxZu20ZT7HrRFpdaNE9JgYzqmgn/baJEdXV1/uugVqvtPCWyx2N6TLjScI3qPEA/ccONBhXfqnCn/QTJC3E4OsLDhw/v3bvXNT4Fhz1MIpFo9OjRMp6Y99aifx29eqi8sZWNEwDNdjEBAEBatkAil0gkAVGDl6XbARzT22y2YGgcYNCZM2d+/fVX3K2EKyIpyOP5VoI74R4k78CBA40NDXxrERgIbsE1FixY8N577zWXHDt2LDc3NzMz89tvv83MzLx8+fJjjz1WXl4OAK7CRx55hM12CwA1NTVJSUlOmedommZ3wnK9Lm9LaGhsNCAx2K0AAGIZmDQQ3wEar0PSHaCqgphkMDSBNBrsVgAE4igwqSC+IzReh8TOpK42JSnBKVVQVlZWYWEh+7mpqSlkE/MCwNKlSxsbG99//32C+P1noFAoGhoaZDKZ2WxOSUnR6XRuhc0tWFVVRdN0RkYG+ydN0/7vTWbz/3n/AzCZTCq9iYlOvmU7dQ3I44CyAW2DmBT3wqZKUiBENJWQEN/85XCXLl3+7//+r3nh9fX1SqUywvIRttoGWXw1hBMIITYzl/enWK3W2oYmRqEETQ0kdobGCkjMAHUNxCSDvhHkcWDRg0gKDA2IAWmMyKzq2CG9eQnN26Ber/dyQ33wqa2tzczMzMrKcgTJS05OBoCsrKyioqLs7OxLly6xIfTcCr20IPjenbKRzDxbjWGYispKIEggSCAFQNsBIRBJExRSb1ZjeVapuQVZDAZDKEaWWbRo0aJFi9jPzbtRL/MRlpSUbNq0ibV6JFFeXr548WK+tfAKdkzvunTbyzF9ZFuQza3oICLzEUa2BfnWwiv8DJIXwRZctGiRSvWHN95srFHPU0gKhSJUYo16v4epV69eHTuGfZgrJ3h5w8mNgwcPHj161PG7IQiCHdN7H94JWzBEKCkp+dOf/mQ0Gtk/1Wr1pEmTTpw4kZub+/HHH8fHx7ckxBbkF/+D5EWqBQmCaP6SBm6/WnASOsE58zYEJLJM85erEbCHqZ2waNEidBsAQAjl5uYCjksSbuDlTuFLYJNjRxIEQUh8x59noLB5esIEhzAN79Q+ac9bmDxD+07wU5HgIHmhAw6xhnEzpnc6IPTDO7VPArWFKcKgabqpqYn9zKZhslqtrZ4VGxvb0mKTNgIHyQsdsCPEYMISP5c7RTACgcCxhEQmk4XsqtH2lRw7tMGvRjGYsMSx3In1cARBHD9+HPAMEwbjO9gRYjBhCV7uhMEEilB5Nfr4U+OraoMUpEBMwtdfbk1KSgpOde0EbMEQgfMME7ZguIMtyJlQcYRHjx3X5C0HRWIQ6pJummix4AS8AQZbkEcCstwJWzDcwRbkTKg4QgCAbjkQH4zZC1Ic1LVh7QhswXAHW5APOIdEcAO2ICe4zxHu3bu3Z8+ecXFxPXv23LdvHyuMsLjpbmPDY8IIiqKmTZumVCoHDRpUU1PDtzoYnyFc4FujABPxIRGOHj3at2/f6OjoDyNpVgAAIABJREFUvn37fv/993yr4x6OjpBhmAkTJnzwwQcqleqdd9554YUXWHkkmbCl2PARQ3t4lFmzZo1Op6uoqMjJyVm4cCHf6mB8Rt+MN998c+7cuXxrFEjaQ0iEiRMnzp8/X6VSzZs3b+LEid6cwoYV9RU2Vjg3ODpCiqI+/fTT4cOHG41GiUQSFxfHysPXhFOmTFm9ejX7+cUXX1y1alVdXZ3NZsvLy0tLS5s4cSKbh6FVEEKW21itVot3BH9rV+Q9yrhaEAA+//zz119/XS6XFxQUvPLKK7wqiGkFtxZU3Oa3334rLi52xPqPDCIsJIJbC8bExGi1WoPBoNfrvdzQyTCM9o/odDqdTqf1CEVRnDXn6AjFYvGoUaOMRmNMTExeXt6GDRtYefia8Mknn9y1axcAWK3WoqKi8ePHs7HhN27cWFFRERsbO3PmTG/KQQiZfcefZxluRN6jjKsFAaCiouKLL75ISEgYNmyY95HpGd9pwwtrN7i1IIvNZnvppZf+9a9/+RNYOdQIYFbXEMGtBT/++OMpU6YkJCS89NJLn3zyiTflCASCpD+SmJiYmJiY5BF/Et759atSKBQGg2Ht2rUzZ878+eefIZxNOHz48IkTJ9bV1f3000/9+vVLT09PT093GxveMyRJOuaxGYYJ2XyE7KOMwWCIiYlx7MWGcH6UcbUgAOh0OoRQaWnpunXrpkyZcvLkyVbLoSiq4Y/pRpuamlpN5xYXFxfkAF2Rh1sLsqxcuXLgwIE9e/bkUb2A438GmFDDrQXnzp37xhtvzJo1a/Xq1fn5+QcOHOBbTTdwHBFev3799ddfB4CoqKgXX3zx4sWLrDx8o1qIxeLHHnts9+7dhYWF7IvslmLDRxLso8yiRYsc493wfZRxtSAAKJXKWbNmpaWlzZgxo6SkxJtyBAJByh9RKpUprcHLzyPCZnndWhAAaJr+8MMPvXwlE0ZEXkgEtxb88ccfZ8+enZaWNnfu3B9//JFfDVuCoyNMT0/ftGnT0aNHEUJbt27t168fKw9fEwLAuHHjPvvss/379z/xxBPQcmz4yCDyHmXAxYIA8PDDD2/ZssVqta5fv37AgAH8qhdYIm+WF9xZEAAOHTqUkZHRrVs3HhULJmGdAcbVgn369Nm0aZPBYPjkk0/uvvtuftVrCe5zhLt27Xr11VcTExMLCwsdc4RhbcKHHnro1KlTw4cPj4mJgWax4Tt06KBWq5ctW8a3goEkIh9lnCwIAEuWLDl06FBKSsrBgwc3btzIr3qBJfJmecGdBQFgy5YtI0eO9L4QhJDpNmaz2eQd/L7qiJisrq4W3Lx587fffpuWlrZjx46QbYPc5wgfeOCBU6dOOQnDOm66VCrt2bOnY0TfUmz4yMDxKPPbb79lZ2dv3ryZlYd1ChgnCwJAamqqa36GyCDyZnnBnQUB4LPPPvOpEHbxPfvZbrcTBOH40wMikUggEPhUEcYVVwtmZ2efOHGCR5W8IVSWYMXEJdh3zCbE8iDUZdW7mSOx2+3nz5+vqqry6dkzrAnsowy2IC8EcMFaxFiQJEnHmIld5RSaaZgCTsRYMPiEiiM89M2uH374IUiVvfxoQkKCk+zrr79+5ZVXPvzww9Bc5Bn6YAsGmevXr69bt27FihXsLO/ixYtZOec1h9iC4Q62IHcQH3To0KGqqoqXqtuUM2fO9OnTx/FnY2Oj1WrlUZ+2I4IteNdddzkJb968yTAML/p4wGq1xsfHHzlyhGGYtWvX5ubmsvLZs2fn5+czDJOfn//aa6+1JIxgCzZvgzqdTq/X86iPZ/bs2dOjR4/Y2NgePXp89913rFClUo0ePTo+Pn7MmDEqlaolYTuxIIter2e3QrUROB8hBhOWROSCtXZFRK77DVNC5dUoBoPxlchbsNaucKz7NRgMTut+i4qK2CW+eXl5bOgZt0JMoMCOMMDU1NzgWwUMBhMGROS634DAMIxKpWouMRqNqNliYLcoFArOc5PYEQYY1PohGAwGc4tIClQZKAiCiIqKai5hZ/KchE74E4eW+xxhUVFR79694+LihgwZUl5ezgrDN7wTBoPBBJOIjO4UEAiCkPgOSXJ3ZxzPrKysnDhx4oYNG27evDl27Fg8zRuO4EcZDIZHIjK6U5hCIMTlZd6RI0c+++wzdqFaQ0NDjx49GhsbASArK6uoqCg7O/vSpUt5eXllZWVuhZmZmYMHD2Z3udpsNpFIFITE0wghm90OAjFQVhBKgLKCQAQMBYQAEAMAQJLA0M0OsIFA6HwATYPQ7QEIADWpNYcO7Fc1NbI1NjU1hWz2icrKyl69eu3bt69v377r1q3btWsXG/0hPz/fYDCsXLlyzpw50dHR7Jy8q9CDBR3vcAKO3W5nCBIQAoRAILplBdoGpCAQQgYE4qa6G+Xl5efOnWteb21tbUpKSoTlRuelDTbHZrMhUggMBaQAGBoIEhAAICBIQDQIREDZbrVToRgo+6221trBer2+tLT07NmzbC16vT6UN9QfPXq0eXSn7OxsANBoNBMmTDh79iwbyIkNDuAq5MWCVqsVgAC2HoSafQYABA4FEOIchr6xsbG5BVkMBgNCKDo6mqvircDRETqgaXrGjBkkSa5btw4AFApFQ0ODTCYzm80pKSlsMltX4YULFxzJON555x2DwfDmm282L9ZoNMpkMn+GuiaTSSKRNI+ZVFNT89H2ry0jZsPWOTB+FfFVAXrgb+RPhUzXQdD4G0kAk9qDKNmDHpsHW+fA0yuI3W+j3L8Sp3agO/8EqmoSUUyHPsTZ3Wj0Atg6B55aTnz9Lhr0PHl6J3PHvaCtJe1mptfDcb98cun8r2yNarVaoVCIRKJWtQ1+Eh8/H2WaW3D58uWjR49m2zAA6PV6/3+vZrNZJBI5vfT/cNOWsth+0PgbaWhkBr8Iha/C+FXkt0uYux6FpkpSV8cMmQKFr8L4leS3y5jeD4OqmtTeZB74G2tQcs9yptdI0N4kVdXM0Jdh62vw1FLyu5VMjxGgbyAbf2OGTSO3z9244T+TJk1qXq83jrDVPE1tQVFR0fz586urq/v06bNx48bu3bsDgFqtnjRp0okTJ3Jzcz/++GM2KZirsLkFV6xY8eyzz3bs2NG1CqPRKJVKOcces9vtFEXJZDLXrxavWF2XPZb8eSvTYwRc+wmSu4LFQFp1THov4tJhNPgl2LMCnloK2+dC3kJizzKU+yL54+dMr5Fw9QdI7gZmHWk3MSlZxOXvUc5k2L8Gnlwi2DnvvXfe6tSp07PPPsvWEuKO0B+8tCD4bkSr1QoAbnukWbNmEYoERiAlJXKmsYKUyBihlJBFg64eaAqkCiSOIiirwKx++sknMjMzXTfde6OSUql0WJClrR2hXxvq9+/f369fv7lz59rtdlYil8vNZjNCyGg0yuVyD0IHffv2PXz4MP1HamtrrVYr7Qd1dXVs5ncHP/74Y2xmf/iPBQBgg42MUcKCk8JOfWDiOrh/gjBnIkz5lEzpChtsQBDwHzMZmwLzjgs694Xn1kLOZOF9z8LUQlJ5J2ywAUHC/xnIuDTIPyq4oz+MXwVDXhLeOw7ePtuhaw9HjfX19Waz2Rtt/bGCn1AUNXXq1GnTprF/RkVFmUwmhJDJZIqOjvYgdNC1a1elUllXV1dXV1dbW3v+/Pk6vykrK6usrHQSPvDIGJhaCA/+Q3D3KNhgY+0oSO0GL38OI2cL+zxyS7jeIkjrDn/7Lzw8R9B75G2DmgRpWfDSJ/DoG4JeI2CDDUgB/J9ekJ4Nf/0IHpsn6DEUNtgEYmlFRYVTvefPn6+trfWssMViCaLREEKooqJCoVAUFxebTKYVK1bk5OSw8rlz506fPt1isUyfPj0/P9+D0EG/fv1OnTrlthY/g0KYTCa1Wu32q+z+90H+98IOPeCvH0HWAzBmAQydKrzncXh+A5meDYsvgVgOG2wgksKaWjI2BeadELIWzBoCY9+EIVOE9z4Jf/k/smMveOccSBWwwSaOitFoNM1rCfEN9YHCgwURQg0NDTabzfvSPOxeJ0hSkNwFMu4W9BgGYrkgPh3SehDZQ4m4NCIqnkjtDt1yyIy7CHmcOD5106ZNQVApIHBcZoMQmjdv3okTJwoLC9nnUBYO4Z0IgoiJiXEa/JG34aae2xK8K83vh3ri94r8v4q25sCBA2+88cbIkSMXLVrESpDvK9ZiYmK2bduWnJwMt5+r2M/+IBKJZDKZVCptLhSLfk9h6hC6HYcRbj79/pEAN6cTt/9VKpVOIxiGYZKTk0Pt1ei1a9fGjx9///33A8DkyZOXLl3KykN/F9rFixd/+OEHrUbDow6YNgVRVjJ7EN9a+ABHR1hcXLxr166TJ08KhUKDwQAA7MsHdkZ38eLFrtO8TkIMvwTwUQbDC0OHDh06dCgA0DRdUFDwzDPPsHIOu9BMJlN+fv7WrVtda9FoNBRFcZ7ntlgs7ICyuXDIyFGmTveZ6+sB2Hkmb3E+lD33D1Kk0WgYhnH8bTAYvMw+4eUsBsZLaF3Ty9Nm7N6z76vthXzr0jocHeGRI0fKysrY6QcW9rcemkl8VCrVC9Nm3aypslgswa89NMGPMm5hxFEDH3hIICCXvTX/4Ycf5lud1gnImF4oFObm5rqdyZPJZDKZzJ8FXyRJOpVs1utMT60iL/T3dXmCy5DcdYxOSKXS5tVRFEUQhNtLc9XTR3UwnmAMjXTWMI3BzLciXsHREc6fP3/+/Pmu8tAM77Rt27a9F2ptyb1IOw77covwepQJGoxJWzJ4gezg6traWr51aYUAjunFYvHo0aOdXkSzSCQSqVTK2RGyDti55OYvmf154Uw0+xcAAGih5NEnn5WKBB+v/3dmZibczkfo9tJCAX+WO/GtuxcQYfNsETaK+olAeQd0vodvLUKI+fPnO00Xs/Kwzo4dGLrlCGJT+Faiddgx/e7du9PT0w0GAzush/a9C4026n7tO+Pc5etarZZvXVoH78YOHdqLI8RgIgzHmD76Nqy8vWefyBoilMfwrYRXOJY7yWSyyZMns1uVAGDXrl0zZsxgVzbt3LnTg9ABTdOlpaX2tqS8vDztzu7I+zlddgIXMW2qVXOazw37Co41igkDho564ucTR62IhAlPBaM+hj537ty3336bnJw8YMCAYNToO+E1PYFxJbDLnd577/+zd+bxURTZA3/dcx+5EwIJoBASWFQOxQsQAY9V5FRUdmURXV1ZZEUOFQUBEcP1U0FlZZVl3XW9UEBAVuRGLkUDBIKBADnIncncV/f0Ub8/GsY4SSbTPZM5MvX9I5/Jm66q1/2m+nXX8d4b999/f4sN2e12AAh8KZAQ4drHr/z888/O5B5QcbGpsE2vyLJci2/nIVHJh8TERBx0G9Nh4Xn+4M6t8HatfFmYFmTTNSXvbTz1j20Hb+qReXAX9h+YdiQky50SEhLWr1+fnp7eWitJSUmBex21Wo2a7V5PTEwkZb7+wt8ELwEAoFAoWtMqeJVCCHaEIYbn2NLSUgDo0aNHpHXpWOhSg1pYIQZEu5h753j0aVzxB+FpEROH4C1M0QOeIwwpdkO9wdR/6N29evVyu2Nj3TAGg4kI8bDcyWxsbD6jGYVgRxhSKDtSah1LzsuUUbpcG4OJIBUVFT/88APf0dPpBUjHX+7kMJ69UPbQQw9FWo+2CcoRchznDbUsEG1JfIrOFt8z5qE1f/+A49hwtosJFe+9995rr70WaS0wIcDpdA66Y+R9U2e62zmuBeOhX1y45OHHHv/xxx/btaEg6fhbmDiGyPpdpJUICOmOcM2aNYMHD/Yu+RWIth0wb6x6c48j4xyl5YJYWduBifJHmc8//3zuyveXnOBEBeLCRCcMw9jNRuvcw+0910uZDfsVA7756VxZWVm7NoTpMEh3hP369fPJnQSSdsC0O90HQkq3cDcaC8TEo4yy2/UwpoVNAhiBKH+U8bJjx44PP/yw+XLH9uKGUYqMa8PUFib2ke4IR4wYMXr0aB+hhB0wbrd7xYoV9t/icDjswSHUwLQVbLfZi0arbx6oha/8vaYIs98BasuyERi5DdWjDM/zFRUVweTMao1gdsheIbg3SWGpeiCI2GgcOmLiUQYACgsLH5n6zKvbzrB4dhATlYR4+4SEHTAkSXbu3Nknxw1xFcmaBFs+FAR/Fu3KiBEjmgslPMo4nc6ZM2cOGTIEABBCZrM5+Cj+FotFo9EI2TibiK98bips0QO16JTQrx9aKI5aOpZlWaPRaDab5XK5fzsmJSWFObUyAPTr1y8nJ2fMmDFNhVGYholhGEVKpmv0Qvjxq3C2GxNwHHfdddedO3fOK4nCWKMURR09ejR8L/RhJ8SOUMIOGJVK9fjjj/ukkHa73Xq93idBuSgoitLr9XKFwv87QQDx7L1ftBDq3k/Ner1eJpPp9fpgIveHH2mbeb/44ovQ5iOsrql96533ykovcWxT73I1d2C75SP0OVahUHTq1Ck68xFC6B5lbDbbXXfd1fRe7MVsNrvdbslPNkIaJpPJhPiW+6GoN+lA0jA1+ZK3Wq0NDQ0EQTidzjYrT05ODv+jzJo1az799NMW3+m/+uqrOXPmrFy5UnhqaVEYNua/9saad9cR3ftJK37ixImcnJxoXuMTYkcYPUl8Xnpt2Sf/+pBFJEy8OTwtNgXJFH96ahrw/OsLX+nbt2/4FZBMlGzmfX7egu/tqWCwyfWxEGU/mpD2KLNu3bqMjIzmtZEkmZCQIPlhzu120zRtsVgIsuXHCFGPFy0/thItH0IQZGJiYnp6OkEQPs/ZLRKRNEyx8k5fX1/Pdeol0VuoEwbfOXLjJ/8ZO3ZsaLUKISF2hNGTxGfP7l30jO2yT54NT3M+cLRrs+xWzcHVz9bVxZYjjJ5HGeg/GkoOgulymJprCavZuHnzZovFMmXKlGDGJ8KJhEcZgiBSU1Nb9ATkVaQpc+7cuTvv/r3TYpR1yWv7aLG05UMJgiBJUvgb+tZDQQinJ5577rnt27e32IqQrFhsYM+mCxc8Hg+IS6J8Neg2AFB21aAHHA5H01VawavUnGBSKwfbt31GNqIr4K9KF7agXC1w+2RFwacRa10q0fMoE3lqfim+VPHEyo8dP21/5JFHAnmriAai6FEG4OzZs+TvRsLv7kf/Wx62RmMdCe/0SqVy/PjxreUfFrIrB+4khMVf3trKy8sF1yv5ZopYz65du5KSku66666QqNQiwTzuxMZDLqb9iLZHmS++2nzg4PcXLl2CnuFpsHU8LuicZ3tqo/xU9M5tNAc/ysQ6Et7pFQrFyJEjW8s/LGRXDtzrsCyLEPLW9vvxj1TV1EB6T3GesMm4tbuu7D/VFz0gf+CBB0KiUsiJ0hEDTHzCsuwfHn14naFbvTEGEqtGCS0+ykRDXJKzZ89WV1fzOBiCSKIt1qjZUE/oUoOqgudQVt9o/h10QEfo8XgcDgffyio1TNRDwN3PEUp/YyCY6GfTpk03DR25eNn/0TQdEQV42v2PDf9+9vk5u3fvjogCkomeWKNms3nEmIlOlzMEN1PKvnnz5m49coKvqT3oIEOjVquV53m5XK7T6frffuel4jNMFJwaQ7uefWF+UmLiS89NmzBhQqTVwWDCgd1u/+STT7799lvPkCcRZZdHaLkTXXP+gPp24vjla7NPRnnvi7bpCS8lJSU/F53jeRSCFyaXhet+k/HyT2+//fakSZOC32ocWjrCG+GhQ4cyu16Tmd29U48+KZnZ504eZ94xk/LIX2ja3HAuZ0KBXVNeXh5pXTDS4RWaO+8be+vwe3bt2hVpXWKAGXPmzXz3ix2HCiI/JnrdPXDNjS6Xq7Gx0ZvkCCMKmUobusqQm/K8snRVFN4PO4IjLCwsJG77I/Pw/3mUetvcQ5FW57f0uJlMuybSSkQ1BoNh1apV+fn5ox969LEnnw42Klo7wLvtJwY+d9pMVFdXR1qXGMDhcHhu/RPXuXekFQEAgPqLb7+7Nqt7j6f/OiPSqsQYCKEDBw6EMvojQiCTqTK62e12qp0zkIgl8uOHoYSUQ1JmpJXAiGPy08/uraRR42UgCH7YU4D+E2mNWqLPcNmZLZFWInopLy8/deoUAAwbNizSuvwGZK1Bo+bxKV0ZOsoekaObS5cuvbbk9W27DxCJIb6jum3m+0c9kJXT5/cjhq1ftza0lUumYzlCTExRWVlZVVXV2GjgbpsBxXvlthp+8J/gv3+LtF4YEZw/f76srOzxac+5U3tSpSdze+XU1dfDPXdGWi+MRHieX//Rx4veWG53OFCXvqTTGNr6Wbed79Sr2qPauWdfaGsOhnAMjbZHCpiampotW7YsXPJG9943LH5jRUSyNwQKQg0NDefPn28eDyJWCJUF7Xb73r179+7d++mnn+7atavfLUPuf3LWqcIzIVQV0yLt0QfNZvOlS5duGTZy0itvN9RW2Sev5wjZLzkPWj1RN7jdAQhbIq2ysrJ58xfYbTakT2u/VkCf1mhzXpvbZ9GiRSdOnNi0adPq1asjtcAYwuMIQ5ICxu12z3jhlfsffHTMhIlFRUX3jH9kyqJ3899cU9nnQWNq74gkwQkQvq7k/977YOCQEVOfeqaoqOjChQuR1kg0wVjwH//4x4oVK9566y0A+Oucl8c9NeuBPzz5+Iw5j8x9w9JQY51ziNTFxnZ11lTzl+kz1PrE0RMmRloX0YSkDyKEFi5a9Mr8BY9PnTrjb891733DDTcPsZmM1unf/Bp1pMfNhCL6tr44TXsOHup53cBbh9zxyvwFy5bHXpib9k6kRVFUY2Pj/v37+w+6lSGUBCkLeRO/wUN5PMzlOsPStf8acd/oR5/+29wXXyorK9u0adPx48fbt+mWCMfQqP9wsRzHFRcX33DDDQDA8/zevXuNRmN9ff3AgQNTUlJ4nr98+fKcBUvsFpPZSfO3PQaH/vnN15tAkwiPfyBrEPbTEMhpAlMVMBRUngEAqCxEgKCuBBgPmKvAZQFGCaZKYGmoPi0cAAhB/QXE0GCuBqcZkSQYLwNLQ2UhAEDVaUAI6i8ihgZLDbhMiGegsQKxnqsHnAHEQ/1FYCiw1IDTiGgHNJYBx/zmgIZLvLmKHzKVddv27Pnkhhu2J3TuPujGgUoSvvz0Y2+gBJlMFrXhEEGMBTmO27t3L0Jo87d7vv7sYwAEiZlEdl9Uefrtdf80GM30vS/AxSMyl9n6wGIoGg6VhYhjobEM7I3IbfOaDzgWjOVXhVdMhjgWGsvBYUBu65WLXFmIWBaMFWBrQJT9qvA04hgwVoDdAO5mQlv9r0dWnUGsB0yXwVaPaOevQk4QNgDluCKsLmINFdyIGaxMceCHT7vlXacioeDHo1rtlTV1HcaCPM/v37/fbDaXlJRce+21J04XlZSWO13uisuX3TaLgdcCzxEeB0rpRvIk/8RH8M64K5eo5hfhB3/l6lEOsNWDx4Xc1itdr7YYEA+VhYAAqs8C4qHh4pXrTzvAVg9OM3JbwVwFDA21564cDE0OFixIO8BaD04jooSDKag7f/VgBNVFV7r2lYOdYK0HS41dlWZN6l1+avtxe6K67LCHpgFg7ty53njiHcmC33//vcFgyMjIEELIms1mnU4nnGBFRYWQpZUkSZvNZnV7Vr/zXmZaaklZhdVmA5eF6N4fTFWAELAMeJzAUAAAHAMeNwIeAIBngaEQ5QDEI8QDSwPlAI4FhID1AO0Cxg0IEMsC40YUCQCI58BzpQgIRTxOAACeQ4TMRqpItQyxzO8G3Ezw7LU5uT26ZWsTk+oNjXcPufXhhydWVlbabDaE0PXXXy+coNPpNBgMADBq1ChvEOBgLEiE4V1Kr9cbDAaNRuN2uzMzM202W9Nvc3NzAeDYsWMAQNP03ffexyJw2a06jbpLly5Go9FqtRKaJIah7XZ7alp6fU1V5+xujQ31yalpTptVqdEi1sPyoFGrbDZ7WkZ6XXVV5+xuhvr65NRUl8OuUGmAY1geaTRqm9WaltGprupy567djYaGxOQUl8OuUKmBYxkeaTVqm8Wa1qlTXXVlZnY3Y0N9YnIK5XTIVWqCZz0s0mk1Fos5vVNmXXVlZlY3s9GgT0ymnHaZUk0izsNyOp3WYjand8qsr6nq1CXbZDDok5I9LgepVJPA0R5Wr9dbTMb0zM5KEvZ99z9v6LzExMRoztYUuAUpihpx1z0gkxsMjWp9As94PCyXmJTUUFOVmd3NbGzU6BMQ6/F4uKTkpLqaqs7Z3cwmo0arQxxDe9jklGTBfGZjo1qrJ3iW8jDJKSmCySxGo1qnA+63QpNRrdUBz1K0rxBxLEXTKalpgr1sZqNSoyMQ56bolNTUuurKzKyuNrNJqdGSwLvcdEpqan11Vaes7KtC5HK5U9LSfnvkFWFHtaDQBzkE1VWVnbtkGRqN2sRkymlXqLU8QyFSriAJysPodFqbxZqant5QV5OZ1bWhtiY9s7PFZNQnJrsdNqVGy3koRMrlBFAeRq/X2azWlLR0Q11tZla2cLDZaExIunow7Qa5UkYg2sPqtFqbzZqSlt5YV9spK7u+pjqjcxezsTExOcVltyk1Oo52CQd7GE6rUdts9pTU1MaGuk5dhIOzzEZDYnKK025Ta3WM2wVypZxAHpbTaNR2uz0l5crBHdqCv6comgQ+PT09Ozvb7XZXVVUhhHr27Hm66KxcqbSbTdqEJNrtVGn1ToqmPSztdiqUahkJiJAxNEWQpEIuA0LGsizHMkqViiAIHiHa7VKptSQJCAiP2y1TKuUkAYTMQ1NAEEq5HGQylmU5hlEqVQRJ8Ahol0Ol0ZIEgQjSQ7lkcqWcJBBJsh4PAhCKcCzHeGiVWk0QBAKgXE6VWqPVJ6iBuenGG0+dPkPKFS6bZdCgQSRJVlVVmUwmHki1WrVr57ehsSBqf7RardvtRgg5nU6tVuvz7cCBAwsKCnyEr732Wk1NTTCN5ufitb6BAAAgAElEQVTnV1RUBFPDihUrSktLg6nhzTffLCkpCaaGKCFwC9I0PWvWrOBbXLduXfNfhVgKCgrWrVsXvDKzZs2iaTr4eiKIhD4oEORv+Pjx4+vXr5dcHCH0wgsvCPmZpfH111/v2LEjGAWiBMkWRAi98cYbly9fDrytHTt2fP3114EfX1tbu3jx4sCPD4NKYgnHG2Fubu7//ve/3NzcCxcuPPDAAyUlJU2/HT16tMVi8Q4xCRQWFubl5fmPNe6fM2fO9OzZU6fTSa6hqKjo2muvDSbnwC+//NK1a9fExMQ2j1y8ePHgwYMlN9TeBG5BnudPnDgxaNCgIFssKSlJS0tLSwtqut5oNBqNxry8YBMA/fzzzzfeeKP/UZcOY0EfAv8Nt0hjY6PZbBbeV6RRUFDQv39/yTmwKisrCYLo2rVrm0d2VAuC+Juh8PrYrVu3AI93u90lJSX9+/cP8Ph2UikYC4bDEc6ePVulUuXn57/yyissy65atarptwaD4eTJk+2tQ5QzYMCA4LO6tx/Ygm2CLRjrYAvGOsFYMByO0GKxPPbYY4WFhUIKmLAFv8eECmzBWAdbMNbBFmxXwuEIMRgMBoOJWqJ3uTAGg8FgMGEAO0IMBoPBxDXYEWIwGAwmrolqR1hUVBTM/oetW7def/31ycnJw4YN81lt3CbBR/YLpvUOQJC2g6AvYKhiM8a5HQWCsWYEu2E8227IkCHEVaZNm+bzbfML6/9aNa3tiSee8GOUFk0WeOWCqmKVDwHtt0UxSCwWy0033SRZw4qKCr1ef/ToUZfLtWrVqsGDB4sq/tJLLz377LMURT377LPz5s0Lc+uxTpC2Q6G4gEFaMFRqdACCsWYEu2E8247n+dTU1KqqKrvdbrfbhZ34TfG5sP6vlU9tc+bM8WOU5iYTVbnb7RarfEiuWJQ6Qp7nx48f/+WXX0q+me7fv/+pp54SPjc0NKSlpYkqnpeXV1xcjBAqLi7Oy8sLc+sxTfC2Q6G4gEFaMFRqxDpBWjOC3TCebVdbW6vX62+66Sa9Xj9u3Lj6+nqfA3wurP9r5VNbTk6OH6M0N5moyuvr68UqL/0yNSFKHeGyZcvmzJmDEAr+nZVl2WnTpk2fPl1UKZ1O53K5EEIulyshISHMrcc0IbQdCuIChsqCQaoR64TKmhHshnFou5MnT44YMeLkyZNGo3HKlCmTJk3yOaC1C9vitfKpTS6X+zGKH5MFUvmkSZMkKx8M0eIIe/fu7R2q3bdv35133unxeJDI7te0EoHdu3cPHDjwpZdeYhhGlD7+I/sFiOTWY4uQ2A6F1HwoRBYMXo2YI3hrRlU3jB/bNb/sAjU1NSkpKT5CrVbrDT3ovbCBXKuamhqCIPwYpTWTBVi5j6qtKR+qru0lWhxhU+bPn+8zkXno0CGxlfA8P2/evDvuuOP8+fMSdOjVq5cQa7ikpCQ3NzfMrccuIbEdCsUFDNKCoVIjpgnemhHshvFsu4KCgiNHjgifGxsbO3fu7HOAz4X1f618apPJZH6M0txkoirv3LmzWOUDuyRtEI2OsCmSB2QOHz7cu3dvs9lsv4qo4rNmzZo3b55gwrlz54a59Y5BMINpwV/AIC0YKjU6DNKsGcFuGM+2+/777zt16vTLL7/QND1z5sxp06YJ8v379wsffC5sa9dKON6ntn79+jU3Sms1o9YN0WLl06ZNE6t8SK5Yh3WES5cuDWZ9rNlsHjVqVHZ29pgxYywWS5hb7xgEc9bBX8AgLRgqNToM0s49gt0wnm3H8/zatWtzcnLS09OnTJlitVoFufci+FzY1q6V8MGntsuXLzc3Sms1o9YN0WLlVqtVrPIhuWI41igGg8Fg4pqo3lCPwWAwGEx7gx0hBoPBYOIa7AgxGAwGE9dgR4jBYDCYuAY7QgwGg8HENdgRYjAYDCauwY4Qg8FgMHENdoQYDAaDiWuwI8RgMBhMXIMdIQaDwWDiGuwIMRgMBhPXYEeIwWAwmLgGO0IMBoPBxDXYEWIwGAwmrsGOEIPBYDBxDXaEGAwGg4lrsCPEYDAYTFyDHSEGg8Fg4hrsCDEYDAYT12BHiMFgMJi4BjtCDAaDwcQ12BFiMBgMJq6JvCMsKSlxu91iS3Ecx/O8hOYQQizLSigIAAzDSCvIsixCSFrZ6CdAC0q+CJILSraX5IIxirQ+GExX4nme4zhpZSX3fei4li0pKbHb7WK7CcdxYq0gzegSunAwvxAJSHeELMtOnz49IyNjyJAh1dXVgtBsNo8ZMyY1NXXs2LFms9mP0MukSZOKi4vFtu50OiV0XQDweDw2m01CQQBobGyUVtBms3k8Hmllo58ALSj5IpjNZmldwmQySbhjIoSMRqOE5sJPZPsgy7IWi0Wa5m632+l0SisbTG+S9pOIfiZNmvTjjz+KPTWXy+VyuUQV4XneZDKJKgIAVqtV7CMIRVEOh0NsQ5KR7ghXr15ts9kqKioGDx68aNEiQbhixYprrrmmtra2e/fuK1eu9COMRWpqao4ePfrTTz9FWpG4oLGx8ejRoyUlJZFWJHqJoT544sSJo0ePXrp0KfxNY0JIYWEhRVGR1qIdQFIZOHDgqVOnEEI2m+3nn38WhHl5ecXFxQih4uLivLw8P0Iv/fv337x5MysSk8lktVrFlmJZ1ul0NjQ0SCjIsmxa9jWJeYMSUtLEFmxoaHA6nX4O4HlesiEizsCBAwsKCto8zGg0UhQVeLU3D/+9OqPbmIcebWhoYBhGgmJ1dXUcx4ktxfN8bW2thObCT6j6YIAW9MHj8RgMhkCO3L17ty6zu6ZLz8em/lmQOBwOq9UqtkUBk8nkdrullZX2k4h+Bg4cuHv3bpZlRZWy2Ww2my3w48vKyjQ6/cGDB0VqhxobG2maFlXE6XRaLBaxDUlGLtmDVlRUfPbZZyNGjOjZs+e//vUvQVhdXX3NNdcAgPD46Ufoxel0zpw5c8iQIaJat9vtMplMwrOJx+NxuVxI/JwTQshYXQFvH9Muvl7s0JnFYqFpWqlUtnZAUlKSSqUSq1LH5nLZJarf2J3ffZrVtetzM2YMGzZs3LhxkVYqughVH7TZbKNGjTp9+rSo1hmGsdvtgQzHnTt3DnKHunsOc7oPNTQ0AIDL5eI4Ttq7hcVi0Wg00vpLY2MjQogkWxgJw32wTex2u9sZvuHKcCLdEQqPEmfPnl27du3TTz/9ww8/AABCiCAI4YN3XqdFoZeEhIQvvviiU6dOolpXq9UymUyn04lVm6Zpp9OZmpoqtqDXdxIEIVZbuVyu0+lwNxONtQ7l3M6XHl+97VhhSRl2hD6Eqg/q9fo1a9akpaWJap1hGLlcHkgpvV5PECQAqFRK4Xi1Ws1xXGJioqgWBUiS1Gg0arVaQlmWZdPS0lp0hC0KMU0J5+qVMCPd9hkZGc8//3yXLl1mzJhRVFQkCLOysiorKwGguro6OzvbjzCmkbxUJ6pgQ7TUon0hZQjx/HX3dtxVt9IJVR8kSbJLly6ydsPrYwiCbL9WgkR4UMD44cnpz0VahfZCuiP8/e9//9FHH9E0/cEHHwwaNEgQjhkzZsOGDQihDRs2eJ/fWxTGKh63m0XJaRl9ru8faVWCJYaWWmBaJE77YIejqKio6eBW1D2MXqXs4sWItBsGpDvCZcuW7du3LzMzc+/evevXrxeECxcuPH36dLdu3c6ePbtgwQI/wliFcfMMzb76syf212B/+umnL7zwglarXbhw4V//+ldBuGXLlhkzZqhUqhkzZmzevNmPsCkURfGhplkjKORNhJB2tFPrxGkf7FhYrdapU6c23cYQtQ+jNOUGTaJ3NrojIX2OsHPnzrt37/YRJicn79ixIxAhJuKEaqmF1WodOXJkRUWF/+YsFovb7fazYqgpW7ZscTmdkPHr1KzHwxgMhoBPDgBAWNMkdu4HIWQ0GmUymahSycnJ4Z8Dxn0w1kEITZ06dd68eQ8//LBXuGXLlq1btwrPnePGjVu2bFlrQi8sy/7www+DBw8W9bulaRoAAuySAMAjHgBoDyMUDByPxyOsGhWlG8OIa0ihUEie6JXuCDGxTqiWWiQlJX355ZeZmZn+m1MoFAGuGPrqq69mv7GGcjoBwDtzo1Qq22yiORkZGRIcIQBIaAuDEcuKFStycnImTpzYVCjhYZSiqC1btjzzzDOiHKHwGiq2gwib0EQVcTqdBEGI2lPvdruF1ViBF9Hr9YE7dR+wI5QE7TTZXZndc24ZdOP2zV9GWhuJNF1qsXr1akEorKrIzc1tvtTCR9h+uFwuWY9BYPqmvRvChBkPTRsMBo1Gg1emAMD+/ft37tzZ/J1e2rrfFStWpKaminKECoUCABISEgI8ngACEL9j53elpaXeOelAQAglJCSI8lIul4thmKSkpMCLBANeMSwJu4FhmIYH8i9WVEVaFengpRaYsGK4tHXr1uxres6a+2KkVYkK9u7de/DgQaVSKXg4giAOHz4M0b72HrlslkhNircf2BFKhZRDclaklQgKvNQCE1YstTBwLPNgPtvR7qISWbp0qTeyCQAghIYOHQpR/jDa4VyggHRHOGTIEOIq06ZNE4RRu/AX0xxhqYXFYjl48GCvXr0EobCqoqqqatu2bd5xiRaFGIwUEDp16uSCVxd99913kVYlSonqh1FeYr6RKEeiI0QInTt3rqqqym632+127wxT1C78xWA6HjH5MHrh8GkDs3pv8c69+yOpRpTRdEUlfhgNPxIdYX19vcfjGTduXJcuXSZPnuxNbCRtFxoGgxFL7D6MEt37w+9GRlYHDKYpEleN1tXV3XzzzW+99Vb37t1nzZo1c+bMzz77DKQG3X7xxRe//FLc2ksh6LaEtGQ0TbtcLrGL1hwOR7PnaAQAPMcF8nxtsVg8Ho+fnQN6vV5YwYXBBIj3YfT8+fN33XXXBx98IITflLALjabp//znP3l5eaIUYBjG6XQGEvOToigECOA3ne6XM2dWrlx544033nbbbaLadTqdHMexknICO51OjUbT4oYBtVotarE+piMh0fADBgzYt2+f8Hn58uXXXXed8FnCwl+FQjF8+HCNRiNKAYZhZDKZ2FIAQJIkQkhsweEPTLh44cJvZQQAECQRSFUURfmPl48D/mLEEsKHUf5q5ilRCjRd6CGaS8eO11l/Wr912n01t956a9jaDUpnTMdFoiM8ceIERVGDBw8GAKVS6b3FS9iFplQqR40aJTaWvMfjkclkEiLQEwTBcZzYgheKCt1LimFWl2a1kYFUpVKp1Gp1tGWfGDJkyNGjR4XPzzzzzLp16wDAbDZPmTLlyJEjQ4cO/fe//52SktKaMMzwPO92u9VqNd6CJhDCh1GNRvPSSy8Fvp9MgGEYnucDKaVWq4nfvg4CxxJ5Q7mUbgqlWWy7LMtKzj7hcrkSEhLwc6cEGhoaaE9HTMkLAJLnCJ1O54QJE4qLiz0ez+uvvz5+/HhBHtULf0MO4s1G499mzVm//p+RVkU0MTbDZDccOnhAp9OdOnUqAq1HJULOd+Fz84dRiNJdaJhYpbS0FHXMrRMAAITkEYb333//rbfeslqto0aNevfdd4XUYhaL5bHHHissLLzxxhs//vhjYXVTc+F9991XX18vVFVdXZ2eni72bUnY0SnhyU4YGBFbsLyiAqV0BeNlSMkGSw0kdAJHI2iTwWUBpUZN8F06X4nI1bt3788//9ynuMlkirZ8hHV1dbm5ub179/bOMAlJFnv37r1169Y+ffqcO3dOmH9avnz5m2++mZmZqVAoGIapr6/v2rVr06qqqqq0Wm2br4kcx5EkGcj7nMPhMDooRLtAqQGEgKFAqQHEkzzXuVN64PEpAm+xKbm5uStXrhQbYk2lUokNTxokhw4dmjhx4oEDB3Jycl588UWapt9//30AmD17tkqlys/Pf+WVV1iWXbVq1fLly4WvUlNTTSaT8KFpVdL6IEKI5/lAztrhcDTa3VfvNAgIEngOZDIg5YlyPk1kclCe54WFsq0d0GIfFKivr5cQdS/iLF++/IsvvvBzQHV1dWpqqtgZH1F3UZqma2qEQXWUlZUl6tfSpsl86N2794YNG8IZWUaiIwySrl27/vOf/xSb3jb6KSkpyc/PLyws9JFHoSM8derU7NmzvTNMHo9HmGHS6/VCECy3252ZmWmz2Z599tnExMRHHnkk0iqHg5KSkqVLl27atEmsI9RqtWFe7hT4w2i8WbDFPigQo44wDi147NixcDrCiK2Suu6663xeLDoAMdTBRM0wde3adeDAgRHRM8yQJEmSZGJiYvRv1SIIYvr06dOnT/eRt5hoIq4sGGkV2gVswfZtNPxNYqIBUTNMGAwG04HBjjBOEbXcCYPBtAc7d+7s27dvcnJy3759d+3aJQijKAxQ3IAdYZwydOjQRYsWjRkzJjs722w2r1ixQpBHS0hDDKajw/P8Y4899u6775pMpiVLljzxxBOCPPKLtOMPHEkhThE1w4TBYEIOy7Iff/zxyJEjHQ6HSqVKTk4W5P7DAGHag2DfCIuKinQ6nfffaA/4i8F0OALpg5goRIgl4nQ6ExMTx40b9+GHHwpy/2GA4gGe5x0Oh8PhsItBWtQ9gaAcodVqnTp1qsvl8kqidDs2BtNBCbAPYqIWvV7vcDiWLl06c+ZMQeI/DFA84M1ULIpgWpQ+NIoQmjp16rx58x5++GGvUELAX4HxD0+qrDNIVkYUShK2b/oiPT29nepv/tvlrtJaEQn7viPI22ve+e9XW8PTFgnw6gszx44dG57mYovA+6AP2ILRQHl5+dq1a1etWqXT6f785z/n5+cLcv8xKQU6tgUJgtDr9QzD6PX68LQo3RGuWLEiJydn4sSJTYUSAv4KHDx02DJuJejTJOsTOOp/Tqao9gqax3Gc0Wj0EVosFpqm/YRESUpKitR2+6KioltvvdXpdAr/BhJW9Ezx+ROqvtB/dBjUUx54T9jOgWlO4H3Qh45tQY7jWjtxg8HAsmyLO9VSUlKkxS+VTFZW1j//+c/Ro0cPGzbsiy++8G4TFFZu5+fn+1m53bEtGH4kOsL9+/fv3Llz9+7dPnIJAX9/pddgSAnHxjVS2Y4uRyaTNY+YI5fLoy2yjEBrA2tfffXVnDlzVq5c2epEfWZueFLKkae/DkMrsYioPtgCHdeCMpmsSxff+PhXlCHJ6Ikso1Qqt2zZMnv27LKysj59+mzYsEGQL1y48LHHHuvWrZsQG6jV8h3XguFH4g9i7969Bw8eVCqV3sHcw4cPQ8cK+NvQ0DB58uQuXbp07dr1L3/5i91uj7RGIcY7sNZUGNNZlDmO69OnT1NJB145IqoPxgrNLdiasGNw5513FhQUmEymo0ePes8xppPRNzfW1q1br7/++uTk5GHDhpWUlERKMf9IdIRLly5tmtkLITR06FDoWNknnnzyyR49elRUVFy8eDElJWXx4sWBlLJYLO2sV8iQPLAWnaxZs2bw4MHnz59vKpSwcoTneYvFYhKJhBzRQSKqD8YELVqwRSEmOmlurMuXL0+ePPnDDz+sra0dO3asd69ktBHiIYIWt2PHxB7tp59++u233xY+//nPf37rrbcOHjw4d+5cpVKpVqvnzZu3adOmQOoRArpHP8LAWvORz5hYsdbcWADQr1+/V1991edICS+4BEFotVqdSKInuXmMdjdoxYItCjERJ0ALlpaWTpo06fbbb9doNI8//njUPtCEwBE2zV/R4kt9TLzpP/TQQ1u2bAEAmqa3bt06adKkQYMGLV++3GKxNDQ0LF68OLZej9okpgfWmhsLAEaMGDF6tO/aAQkvuARBCMFXRRHZaac2+2C0EbgFWxRiIk6AFhw+fLiwP5LjuIULFz766KMR0bZNomLSOBoYOXLkL7/8Ul9fv2vXroEDB2ZlZX300UfCk/Xtt9/es2fPVJFZ06KcmB5Ya26s1o6MiRfcOCRwC2KiE1EW3LNnz80335yUlLRmzZqwaSiKaBnPiThKpfKBBx7Ytm3bgQMHJk+eDAA6nW7Lli3CnoeDBw/27t070jqGg0BXrEWU5sZqjUC2ZGHCT+AWxEQnAVoQIfTKK68cOXLk888/z8vLC6eGosBvhL8yceLETz75ZPfu3RMmTACAF1988ZlnnrHZbLW1tfPmzXvuuecirWB7EXMDa9DMWK0REy+48UmAFsRELYFY8OjRo1u2bNm2bVtWVpYQNS2cGgYOdoS/cs899xQUFIwcOVLI9L1q1SqTydStW7e77rrrySeffPDBByOtIOZXfIzVGjGxciQ+CdCCmKglEAseOHDg/PnzKSkpCVcJp4aBI90RBp5JK1b2cqnV6r59+3pf89PS0rZu3Wq1Wn/55Zenn346srphfPAxlpemb7cQOy+40ojpbHYBWtCPEBNZArHg/Pnz0W8Jr46BInGOUMiktXHjxhEjRmzevPmJJ56orq6GVuKSBBKsJDE5lflqFqHUBnMyAULbW7g7MAxz5syZysrKe++9Nww6xDSpqamqPR8rLv8YhrbY0p9g/PU+QmwsENkHfcAWjHWwBUOLREcoKpNWIFGA9+3YcuzYMcmnIY5n7m++BHT79u1//etf161b5yciKEbgjUULBvQN29Kh+2+55RYfETYWBJfNDlsw1sEWDC0SHaGQScvhcCQmJnq3oEEQQbdzcnJycnKkKRMSHnzwwZDMAiIEzcejLBaLx+PxE2tUr9crFIrgWxfFzp07Z8+eXVNTk5WVtXr1auHJLpCg2yqVKrIr/UJlrJhGVB/0oWNbkOf51gIiOhwOtVrd4qZPjUYT/qgIW7dunT9/flVVVb9+/davXy+sq8R9MPwEtVgmwExa8bWXiwCNeMK/HVsYWHv33XdNJtOSJUu8oY9wNrvYAmeza47YPHbBZ7OTRmvhx3AfDD8S77/l5eUvvPACAAiZtIqLiwV5Rwq6LQ0CQN0MlUrVXNgUmUwWZj29A2tOp9NnYC12g26HBMF/iCX8qwBE9cG4Qshm1yI6na61r8L/Otha+DHcB8OPRNuLyqQVSHotTJgJZmCtY8PzvNlsFjtSHf6MksFks8NEA8OHDx8+fDg0Cz+G+6DQBxmGETWkodfrJU9YSp8jDDyTVkwEK4lPhIG1NWvWzJw586effgI8sAYgk8nS09ObJ5WMNoLNZoeJDvbs2fPiiy/ee++9S5cuFSS4DwqB7xmG0el0gZcK5p1eekkhk5aPUNi2FYiwo8Jy3O7du3v16tWjR49I6+KP8vLytWvXrlq1ShhYy8/PF+Q4JlkMEXgfxEQhrYUfw32QIAghkH3YRllwZJkQ46bosX/6y3//+99IK9IGwsDawYMHEULNB9ZwTDIMpr1pLfwY7oPhJzJBtzUazcKFC/V6PQB4PB6FQhGRVVvNYRiGJ0jgOJDJgWNBJgeOAbkSWM9v/soUwLFAygBxQJCAeAASABmNRqTQ0Df/MdLn0TaBD6xpNJqvvvqqqKhIGLFpPnkWEgvStAeEGhAPBOn3AwIAhVze5lJb7xBT4DQ2NorXPdoRLOgnFVxo+yDP8wzLAUEAQgAEgPevlyZCgrhi1l//Ct1KBoDkJCF2HVkMWdAbfswrEZZcSeuD8gB6hGQQQs1TT7fT61pELEhEJObNL7/8smfPHuHzqlWr/vCHP3Tt2lVUDRRFkSQpYWqUZVmPx6PV+oawcTqd2//37fkLl0w9RpBnvuUHPUQc/xwNfZI4+AG6dxZ8uxLGLoItr8IDL8N3b8LQJ4gfP0fX/5648D3q2o80lPL6DJJx8xxLpnRB19782i3qpgkqTSaTTqcL82KKUOF2u4WMYq0RoAVdLpdSqWxtHP+lV16lkruTSjWqOoMy80hShurOo855JM8hQynq3Jtk3MhUhTJ7k5RVhTx/nTxRWFDgB7vdrtfrxd7f09PTR44c2blzZ1GloplQWdAHjuMoimo+i3P27NmPvz3s1mWStlpen0G6zLwqgWTcPCknCeA5llSoedpJapN4h5FIzETmGkjrDoYyyMwl6s6h7OvIqjN89wEKl2ncDZ3vuOMOn/pdLpdCofCzmikjI+MPf/hDi1/V19dnZGRENnmkBAKx4NixY3Nzc0WdGk3TEJg/KykpWbvuA0DoyrMLkARwq996K8CGnE6nqLXxGRkZ48aNYxgmbGERI/NG2Ldv3759+wqfP/roo0mTJt14442iarDZbDKZTNRUqgBN006ns3lkmWkzZh5zpZNMGVx3D3HxMAwYQ5zahm5+GI58BLf/Cb57E4Y9BVtehdsfgz3vwICxcGo79BlB1J5FPW4mOZrPyCXdFp6hibRuURpNTyoajcZ/5o0ALej/aeDV15dRGT0IbSKqOw+dckiZnGssh069SM7DWWogM5d0W3iHEXXOI201St71yCOP3Hrrrf5blHbXQwjV19eLKhLlhMqCPjAMY7Va09PTfeTffPPNFwVV7vTfkTVn+E65pOkyn5BJui28XE0QCBgPoUkAu5FIyQJDGZHVBxEkXDsIGBp63gKUBXJuJ+wNkDtUZiq7886+M2bM8KnfbDZrNBq1Wi1K25gmEAtOmDBhxIgRol6ghbADgQTC3rt379oP/vmrIyRkBILAE/IYjcaEhARR7y0ulyvwg4MH5yNsQt5Qsq6Yj7QWGEyMYrVa//y3uUWFJyg+OdK6YDAikD5EsHXr1uuvvz45OXnYsGElJSWCMKazT2AiAkLojfxlNE1FWpHYI9r64I8//rjzaMF5TV5oFv0jZLFYqqurw/xygIlDJDpCUcGBcMQgjB/ef//9xRu20LTvVLwfWA81b/HS5Sv/r/20in6isw/KE9KgU2iCBnMNlxbnr+jRKy/612BjYh2JjlBUcCD/EYM4jjt//jwTaXhe0phoK/OBHMeFo/XgiJ73CcW1N4pa0uI2NRxokG/+366QaxJDhLAPIoSMRiMfNKFdecfbGrnxSxW3/wEhFFQwvx4AACAASURBVLxubRJCzTExh8Q5QlHBgfxHDHK5XAsWLLjnnntEKWC322UyGcuyYjX3eDxOp7PplHJhYeGERx+jaBpGvyqmH/96bNPuL3ykadpqtXqFNpuN4zg/c8WJiYlhzmYivE/s2rVrwIABa9eufeKJJ44cOQJBZJQMN936g+VopJWIJCHsg3a7/dFHH/VGKw0QhmHsdntT52exWIR/BdnVvwi8vaVZT/GlSZGrAmS32w0Gg1ditVqF+L2itBUwGo0A0OL6qeTk5Bhd1x0hUGFhYf/+/SOtRmgIarFMgMGB/EcMSkhIWL9+ffO1Z/5RKpWSV42q1eqmq0YPHz5sv+mPsnP7AEDMQnvi1z/Eb6QIQKvVNj0jkiSjbfuE930CAB5//PHly5cLcgkZJSmK+vDDD1etWuW/RafTiRBiGMZHTtM0avXV+le8d9MrN0mGvlBWcfvwu9eszPeuQG7eooTMHgghp9Pp3d0cIGq1OvxRmyFEfTAxMXHjxo2ZmZmimmYYRq1WN/2dp6SkCK0QTfrFFYlwxG97Sgs0KXJVQCQmJjbVTalUBrNqNBa3T0Sc5uNASKFbnL9yyxefRESfkCOx64oKDhQDEYNUOiBDeRf7/vvv8/Pz77vvPrFL0sNGCN8nBM/R5rAYukpzuZQTMFfZWNmZ8jqz2dxaDa21KE3PaKOj9UFMFPPvTz73FRHARXsXEYHEu78QHOiHH36Qy+XCs7MQJgZnnwAAVHJoL6CD679KTk6OWkcoEJL3CY1G8/zzz7e5G4lhmBZfi9VqNdEs9EhzCAB09T1DOJLQJitUpFarba1pl8uVkJAg4Y1QKCiqVPjBfRATNpwdfeGuREcYeHCg1oQdGYRQ37vkjrpI6+EP/D4R68RLH0TIbDZXVVWlpqY2DwiFwYQEiWPl8+fPR79FkAuR76uqqrZt2+aNjtOiEBNZRAX8xVGAo5Co6oMGg+HWkff/cepT7lC/OjD1pQuWvNEzt/fGjRtDW3OUwHFcnz59mkqicDe2xWLxkSAETqe4efRoBk8axyne94mEqwjyhQsXnj59ulu3bmfPnl2wYIEfIQbj5ezZs8VVjcbrH+T4EE8cIYeRfWiF6paJoa02SlizZs3gwYN94qFH4W7sM2eKfEU8t+/b7WFWo/3AjjBOiar3iWBgGEbCLhpMyJGptKAXt/Yb069fv6bR+QUk7ARtbzjeZ2UAAl9JbBPXsUbLyspGPzK5tqIUbnsq0rpgpOC2Gu++595bBg/94dCBSOuCwYhmxIgRzYUSVm5brdbRo0eXlpaKCrotTIgEE8EuwPD0ZrPZ7Xb7SRjSHLfbzTAMRYmIvBjMTtC4doQ7duy4KMv26OhIKxK/FBcXHzt2jOc10oqzThsas4BqOBBSpTCYSCJtJ+iaNWsyMjJEOUJhL2YwC6QzMjICOYwkSQnZJxiRaZiC2R4a1NBoTEzz+odMSAeVvj1q5h2m11at6Xvjrd7Mi5jm3Pn70Z9/e4ANSYzmuCQa+uChQ4cOHjzIcniAOjQIi7QBoPnKbR+hF4Ig1Go12T7s2Lmrtb1Nf//739upUQkEc82lF46Vad7W8Hg8zaOchBC29kJD7zGXuQQhqlN0EvHbqNNmYbP7hbDCuCIa+uCpU6dGPfKnZf/a7Ha3Y/4Qzm6at2T57wbe8t1337VfK1FCtK3c/uBf/2l5l68mcd2/PwunJu2HdEcYK9O8LVJYWJiamT1n7tx29YWQfg2p980AHD1Ew200JJgbG5YtW1ZRURFpRcJNNPRBl8slS+lC3zYlhHU2h2korb/+4csoJZofK0NFzKzcjvroS4EjfY4wVNO8Tqfz+eef37p1q6jWhaDbHo+I3D0CNE27XK7jx48T192FqKaDOWKNin7903ooYSH8mMVi8Xg8fiZy9Xq9qJnkkNCvX7+cnJwxY8Y0FUqINRphjBVVBvOS1f+45ZZbhJ9Z/BCqPuhyuV5++eXPP28WRssvDMPYbDafuNsC6NfO4fNFix+bHfWboNtXSb+WNF0QehNN09KWRVitVrlc3uIwWkT6oIDPBRQWafsc06IwbLRmLA9NeScvY5oQL5aRMM2rUChGjx6t0YhbLsEwjEwmE1sKAEiSRAgplcqgTUe08KlZKGHiaoxgjUbjp+sGOcAtjVDdRu12+8SJE48fP+6/ObPZTFGU74T51VtA8+wEVwXo1wNbfOxAiEjOUiYkWq3WxsbGpmVNJhNBEGKvLULIZDKJjaAd/vwhrSGhD8rlciGdk6iG5HI5wzAqlYpoFkC71c7VZtBt4Sui5c7F2RrnLX5Du+LNd1Ytu//++0VpK6BWq1uLwx6RPhgTWC3mlj0hz10oOklRlIT7cLQRYkcoIUCXUqm8++67xcaS93g8MplMQgR6giA4jlMoFC35LPEQbRUlCIVCoVKpJCeOCTMSbqNarfbll19uc30Xx3FNY40aDIap02fSPu/0za7krzdZ4tdcH6jZ4Sztem3l2yOO/LBq+RveshRFJSUlSXCEQkFRpUSt1mtXpPXBsWPHiu1NMpmMpmmlUtnC7//X1CyhhGmscg+ZRpfsoShKWvYJoSdinycKyt3K/gqEQK7++OOP//KXv4RXo9AT4h9EtE3zNsdqtT4xY/ai/JUShlXjAQkr1mQyWV5enkIkGzduPHDZ7XWrbd40/R9AG2tOQ9ddBw+LVSNURM+9Nfr7YFAkZ5OaxEgrgbkK20H2noW490bzNK/H45m3aOm4hyd9e7jgorxrs1gJGIDw3kbJjJ6hqgoAICOnztA4asLDp06dCmW1sUbY+qDRaLxn/KOjxk90OsObmoDnysrKCgoKmua+xrQfjYZGP9/OnvuCyWQKmzLtRLBDo9E/zevl9OnT7/xjvbvrIJmagtTuYKsJT7scx7EsG/357QRiOHeBtdbqpA4WXqitrR0wYECktQkfkeqD5eXlF6vqHUOnkz99Edqa/UMZLi9ZfWrJspUbP/nP2LFjw9l0HLJt27YyP+uxZQqnw7Fs5f+tWp4fRqVCT7xEltm+ffvOnTtBnQDd+0NRmFwgAHgaqyZP/hMA+uabb0aNGhW2dgMnhh5l2kaTpEjpDACFhYUGgyE7OzvAyBcYaZAKJYR9gxCiHPRDK5MK/hvmduMTl8sFiG/1a5IEQvXZV18vf+P16Jkml0C0TGy0K2VlZQ9Pnrru6GWaDve8IGdvRE9/nHBDC+sz45kjR458++23PN96B5MK4tg9e/YMuXf0mIl/aHMhKwaD8U9RUbO8Ez4QUF1d9fe//z0s6rQXceEIEUIKXRJ/44ORVgRzhbvuvW/XqdL2cITuxup3//WpW6ZV9hx0+PDhb775JuRNYKxW6/DRD419+A8ulztSOiDWs3379jVr1sRhIIWwQVHU8jYDaCAEPHfu3LmysrKwKNUudPyh0bKysp9//pnH0SyjhtraWoQQdMppj8p52s31G0OeP+iuL/vou+rd3x+76aab0tLSpK22xzTnwoULP/zww08nC123PU0e/Xek1HDVlv7rJ7n8650///xzbm7ulClTrr322kgp01Gpra3lCEUAwUbQp1//r6Ki4sMPP+zSpUs4NAs1HfmN0Gg0njx58qYhw//80utuMek82gOWZT0eT6wsmWk/vvvuux55fcOwdwWxDNvnrqqa2mt79sLvhcHj8Xhmz1805K77brj1jr++/DrF8JAc0Vsez3F3PMUSsv9eYJet3XDp0qVIKtNB2blzZ4BHWs3GHTt21NSEb/lFaAmHIwxP9gmbzXb8+PEjR45UV1e/tPiN/rffmd2r7x0PTDQ3Njgmrm6nRgOEsjSMf/BBtVp98uTJyGoijVBZ8IkZsx/545+YIU9AeGIyMW6kSVb1GTp/6coHxk7YuHHj5cuXw9Fu9BG8BSsrK99//x9HNQM9hMI55vWQaygRBHDbH1Wdrjl//vxPP/0kJNjrkIQ/h8/OnTuffX5uQIciHnEcAKxbt06nT4jF3RThcITtF7KZZdnS0tLS0lKXyzX5mefuevjxO+4f37P3dSvz3zh900zaanDODPSJpl3hnFb03PaEnjdUV1eXlpbGXEb14C3odrtnvfjy5//6wJaaKz6sa1DQDeUlZNa3+w4+PmPOyy+/vGDBgq+++urUqVMWiyWcakSWYCzIsuzLS/KnPjODI+TQ87Z20jAYXKb6OUvfuv2eB9Iys7QJifPnz3/vvfdOnDhRVlbWvlH1w0jYAt+bTCabzbZq1aqnp01HgXdVjgWldv1/PnFxkNGtlzIxZcGCBXa7vf30DC3hmCP0H7KZ47ji4uIbbrgBAHie37JlS3l5Ocdxw4cPP3nypMPhUKvVAwYMaGhosNvtxcXFarVaqVQqlKpPNm2z2h1V50+DSp+UlOSkPeyDy4mNcz2Pr4f1U0CdAABQcw4QgoZLwLFgqgSWBmsdol1gb0RuG4AMEQhoB9jqgaHAXIM4BkyXgefAUAYIQf1FQAC15wAAas4CANSeA4SgsQw4BsxVwFBgrUeUA5xGRDuA9SB7I3icYK1HLA3masSxYKxAPA+GUpfVMnbsWNAmpyUnqRVk0akTOp1OuAgymSx6QpM0R5QFv//++/Lycoqizp4vcTicRpMZsR7gue9+qQOEgOeRzQAIgLIjRyMAgNsOCiUCALcNEAcIwGVDchkAApcV8QwgBE4rou2AeHBZEOUAhMBpBtqJEA9OM/K4gefBYQKGAp4DhwlYGgEChxFxHgRyUKhBrmRI9WffHQLnNkKm5J0meVInGUd375nbJS3p4YcmHD52vLSqlkRcnx7dSJK8/fbbKYoym819+/YVwvIJBsrJyTEYDGfPnuU4bvr06YqrYZo7kgUPHDhgNptLSkpyc3MLCgoUCsWq1e9y/ccRHgoay4FjoLECsR4wVSKWBlPVlb7jcYOtDmjnlR5hNyC3/UpHI+WIdoDMDYCAZYFnwG1FCiVQdmQzAOUAWz14nGCrB9oN1jrkcYOlFhgKLDVX+hHrAeNlxDFgrEAcC43liOegsUzoqpzbzoxeQG5f4hm3hPx25YqPNvMOI7IZQKYYPW6CwWylKYqjXX+b9nRRUVF6enppaWlOTo5cLn/uuec6jAVLSkqGDh3K8zzP85s2bUpLS+vcuTP8dn+UcIJCxESEUEFBQVVVVV5e3u9+9zsAKCoqeunl+WabAyHkpmgATlyYPJ4DkgCe43mO58hlb65Z897fc3JyLC5P7+6drTa71enO7pw5buzoCyUl6enpW3d8+/qiV7Va7alTp0aOHOnVk+d5giAoimJZ1ps0+PTp0zqdrkePHl4buVyuM2fO/OlPf/KGBQ7GgkQYZq30er3BYNBoNG63OzMz02azNf02NzcXAI4dOwYANE2PuOc+hgeedirlMheLlCoNwVI8y5BKNeVyUxwvA0QSBMexSKHmPLRSo6VcTo0uweWw6ROTbRZTcmqaydCQntm5oba6U5dsQ11NeqdMU2NjcmqqzWrV63VOp1ujUdEeViknWR5IxIFcyTMemVLN0m6lRke5HFp9gsNmTUxOsZiMKWkZRkNdRmZWfXVlZnZ2Y31danons8mYnJxss9n1eq3LTamVSg/LyUnggCR5VqhQrlJ73G61Tuey23WJiVcqNDamZnTSKGT7vvufN1ht9IRsbhFRFrz73t+bTCZCJrdYbTqdDuRKh9mYkJjIETKX3arX6xEhdzmsep2eI0jK6dDptBwiKLdTq9VxCDxul1arZXnkoSmtRstwHOOhtRotw7Ecw6o1ag/D8Byn0ahpmuERr1apaNoDBKiUKoqmSIJQqlSU203KZHK5gqbccplcoVK6nU6FQiFXqlwOu1KpvPJBpZIrlC6rSafTIYXKYbVqtWqZXGk3NyYlJhIyudVs6n5ND4fVxAORmJTitJlvu+22H374QaFPVpDE3o5rQYZDNdWVXbK61tbVpWdkNjYaUtPTTY2NKWlpFpMpOSXVYjEnJiXZrNbEpCS73Z6g1zscTp1O63K5tVq1y01p1GqKptVKJeVhVAq5h+UUcpLhkIIElgcSEAeEHBBPyAjEIlJOcAzIlYj1EHIVYmhSqeI8tFylZmhKoVIzlFup0XrcLqF7qrV6yuXQ6BLcTrtWn+B02HX6RIfdmpCYZLdaEpNTbGZzYkqq1WxMTk0zGxtT0jLMDbVdunSuratPz+xsqKvr1LmLSk52JAtyHHfs2DEhAOyddw7PyemZmZkJAAghb6xgb8RgwREeO/6z3W6/pmt2nz69EULHfy64ePEiAhKAB4K4Gt73qoNAcPXz1QDAV/8DAECo2XwHAoKUkzK5Si0H3uFyA0GqFWR2dle7w84zjMPlvvfukWazubS0fMSIO+FqIGjhA8/zHMcplUpB1cPHfuickd6zZ0/viTQ0NFy4VHpg397QWBC1P1qt1u12I4ScTqdWq/X5duDAgQUFBWLr/PLLL3ft2iVBmaKionfeeUdCQYTQjBkzpBVcs2bN2bNnpZWNBkJiwXfeeaeoqEhC60uWLKmurpZQcO7cuU6nU2wpmqZnzZoloblopj36YHl5+bJly6Tps2fPni+++EJa2X/84x8///yztLIvv/yyxWKRVjaytGnBp59+uqGhQVSdW7du3bp1q6giDQ0NCxcuFFUEIbRq1aqLFy+KKnLgwIFPPvlEbEOSCcfQqP/I91lZWc8995xWqxVVZ0VFhVKplLBU12Kx1NXVbd++XWxBADh+/LhPGtsAKS4u/uSTT/wkNFi8ePHgwYMl1BweQmLBc+fO/fe//xWb1QEATp8+/d1334n9hQBAQUHBiRMnxAa84Hn+xIkTbe8j/i3xYEEfHA5HeXn5vn37JOhTW1tL0/T69esllC0pKUlLS0tLS5NQ9sSJE0ePHm3xvSHWLbhp06Zz586J2iYkRNJ/7733Ai9CUdS5c+eEwYPAOXv27KZNm7zjnIFQX1/vcrk++uijwIsEY8FwDI3Onj1bpVLl5+e/8sorLMuuWrWq6bcGgyFG11KGkAEDBnTq1CnSWrQKtmCbYAvGOtiCsU4wFgyHI7RYLI899lhhYaEQslnCOwEmsmALxjrYgrEOtmC7Eg5HiMFgMBhM1BK9y4UxGAwGgwkD2BFiMBgMJq7BjhCDwWAwcU1sO8KioiJvcJbA2bp16/XXX5+cnDxs2LCSkpJAigQT6E9Ccx0G/+c+ZMgQ4irTpk0ThP4vtZ9vJbTlRx6IMhhpv+0I9iZpd4woh+O4Pn36NJUEfoX9//59kGY4UU0Ecy5BEbYdiyHHYrHcdNNNYk+hoqJCr9cfPXrU5XKtWrVq8ODBgZR66aWXnn32WYqinn322Xnz5rV3cx0D/+fO83xqampVVZXdbrfb7cJmYdTWpW7tW2lttSZvszkMCuK3HaneJO2OEeWsXr36lltu8TmpAK9wm79/HyQYTlQTwZxLkMTqb4Ln+fHjx3/55Zdif9b79+9/6qmnhM8NDQ1paWmBlMrLyysuLkYIFRcX5+XltXdzHQP/515bW6vX62+66Sa9Xj9u3Lj6+npB7v9St/attLZak7fZHAYF8duOSG+SfMeIcvbt2yeEB2kqDPAKt/n790GC4UQ1Ecy5BEms/iaWLVs2Z84chJDknzXLstOmTZs+fXogB+t0OpfLhRByuVwJCQnt3VwHo8VzP3ny5IgRI06ePGk0GqdMmTJp0iRB7v9St2kIUW21Jg+8OQwS/9uOSG8K/o4RzficVIBXuM3fvw8SDCe2CcnnEiQx85vo3bu3dyx33759d955pzfPraiyArt37x44cOBLL73EMEwgrfsP9NcmYpuLXaRd6pqampSUFOGz/0vt/1uxbbUpD9LuHY/guxKKRG8Se8eIZpqbADU7KT9XuMXiqPV+EWC1gRBIE0jMuYSQmPxNzJ8/32em89ChQwGW5Xl+3rx5d9xxx/nz5wNvsVevXiUlJQghITdN4AWlNdcx8H/uBQUFR44cET43NjZ27txZ+Oz/Urf2rbS2WpO32RwGBfHbDn9vCuaOERP4OI8Ar3Cbv38fJBhObBNI6rkESUyuGl26dKnPVRs6dGiAZY8ePbply5Zt27ZlZWU5HI4AU1qPGTNmw4YNCKENGzaMGzcucFWlNdcxaO3cDxw4AABOp3PChAnFxcUej+f1118fP3688G2Ll1oo0tq3kttqTd5mcxgI4rcd/t4UzB0jFgnwCrf2+w+y2mCaCEmjUmgnBxs2xJ7C0qVLJVwBs9k8atSo7OzsMWPGiErjIq25jkFr5y584Hl+7dq1OTk56enpU6ZMsVqtwrctXmpv2dYMIa2t1uRtNodBQfy2I9ubOmQf9DmpAK9wa7//1pBgOLFNSD6XIMGxRjEYDAYT18Tk0CgGg8FgMKECO0IMBoPBxDXYEWIwGAwmrsGOEIPBYDBxDXaEGAwGg4lrsCPEYDAYTFyDHSEGg8Fg4hrsCDEYDAYT12BHiMFgMJi4BjtCDAaDwcQ12BFiMBgMJq7BjhCDwWAwcQ12hBgMBoOJa7AjxGAwGExcgx0hBoPBYOIa7AgxGAwGE9dgR4jBYDCYuAY7QgwGg8HENdgRYjAYDCauwY4Qg8FgMHENdoQYDAaDiWuwI8RgMBhMXBN5R1hSUuJ2u5tKeJ7nOK7NghzH8Twf5sNYlkUIhfkwDAaDwbQf0h3hkCFDiKtMmzZNEJrN5jFjxqSmpo4dO9ZsNvsRepk0aVJxcXFTicvlcrlcbSrgdDp9PGiLOBwOiqLaPMxms3k8njYPs1qtDMO0eZjFYmFZts3DTCZTIN4Xg8FgMO2HREeIEDp37lxVVZXdbrfb7atXrxbkK1asuOaaa2pra7t3775y5Uo/wiBhWfbEiRMhqQqDwWAw8YxER1hfX+/xeMaNG9elS5fJkyfbbDZBvmXLlhkzZqhUqhkzZmzevNmPMEi+//77seMnhKQqDAaDwcQzcmnF6urqbr755rfeeqt79+6zZs2aOXPmZ599BgDV1dXXXHMNAAivgMLBLQq9uFyuV1999dNPP/VKnE4nQsj/mGFlZSWPwGaztTkCabPZFApFm+OZwtAoTdNtHsbzvFKpbPMwAFAoFG0eJpfLZTKZVqtt82AMBoPBtAcSHeGAAQP27dsnfF6+fPl1110nfEYIEQQhfPAueGlR+KsGcvnAgQObugG5XA5teRGZTCYc2ab/UCgUAR4mEP7DZDIZSUZ+1RIGg8HEJxId4YkTJyiKGjx4MAAolUqVSiXIs7KyKisrc3Nzq6urs7Oz/Qi9KJXKBx98UKvVeiU8zyOEmkqao1KpCAK0Wq3/wwCAYRiFQtHmYTRNazQatVrt/zC3263Vatt8I3S5XIG85Gk0Gq1WKzh1DAaDwUQEiS8iTqdzwoQJxcXFHo/n9ddfHz9+vCAfM2bMhg0bEEIbNmwYN26cHyEGg8FgMNGAREc4dOjQRYsWjRkzJjs722w2r1ixQpAvXLjw9OnT3bp1O3v27IIFC/wIMRgMBoOJBiQOjRIEMX369OnTp/vIk5OTd+zYEYgwSMxmM+Px2Gw2nU4X2poxGAwGE1fE6hqNl199jaao8vLySCuCwWAwmNgmVh2h0+kEhGOyYDAYDCZYYtURYjAYDAYTErAjxGAwGExcgx0hBoPBYOKa2HaEgWRrwmAwGAzGD7HsCJXajRs3RloJDAaDwcQ2wTrCoqKipjv5JOQjlA4Ry14cg8FgMNFBUL7EarVOnTq1aRLdsOUjxGAwGAwmJEh3hAihqVOnzps3r6kwbPkIMRgMBoMJCRJDrAHAihUrcnJyJk6c2FQoIR+h2+1evXr12rVrvRIhH2Fb7SMAYBjGbrf7P87hcCgUijaX1TgcDpZl20xb6HA4AKDN7BMOh4MkyTazTzgcDqVSKZPJNBqNkHwKg8FgMGFG4s13//79O3fu3L17t4+8eerB5cuXKxSK22+/nSAIhJBMJhs4cGDTIo2Njfv37x86dGjTSgBAqKc1NAqZWkF+8803R48e9a9qILUBAM/zBEEEcpj/3IG5ubkfffQRcRX/tQV4GAaDwWDaD4mOcO/evQcPHvS+GBEEcejQoaFDhzZPPVhZWTlt2rRHHnkkZCpHMSUlJfn5+Xq9nqIovV7f5huhy+XS6/U4HyEGg8FEEImOcOnSpUuXLhU+C696wmch9WB+fn7T1INdu3b1eQvsqOBE8xgMBhNzhPjGjVMPYjAYDCa2CMECjaYLW9oj9SAGg8FgMO0HHsrDYDAYTFyDHSEGg8Fg4hrsCDEYDAYT12BHiMFgMJi4JgLRTN5e885/v9oanrZIgFdfmDl27NjwNIfBYDCYmCMCjvBM8fkTqr7Qf3QY2lIeeK+ysjIMDWEwGAwmRolQfMvMXPjdyDC0Q57+OgytYDAYDCZ2wXOEv8JxXJ8+fZpKdu7c2bdv3+Tk5L59+/5/e/caE1e57gH8XTPMDAMzAwMou9MibikwrWxaMPqBHmugSnKodKQaooE0Ia1JNc2u1lh3sO32WA7Emlg5hqSmlOyIzUFrwhma6j6tW7ECzdZdiaUtt9gqFJBrBzr3y3rPh2XnjHNZMw5zdf6/T/adh2e9wwf/mVmL9zl//ny0NgYAAOETfBB6DYlwzeANv9bW1rKystHRUecKy7J1dXXvvffe0tLSm2++2dDQEMXtAQBAmAQZhL5CIi5m8D7//PPHjx/n/nv37t3vvPMOIaS4uPjw4cOuZXa7vbOzs6KiwmAwSCSS9PT0KOwVAADCLMh7hM6Q0Ov1riHR3d2t1Wq5GbwajaalpSV0Ww2Zp59+urm5+eWXX7ZYLFqt9ujRo4SQ8vJytzKxWFxVVaXX6xUKBcMwfX19gTSnlOr1eoPBIBQK/U6f4CJWKBQmJydjHiEAQFQE+T9fXyHBP4M3RlRUVNTX18/Ozn7zzTclJSUqlYqnWCaT6fX61tbW/fv3f/vtt36b018LsPi3vQEAAAidVT0sw4VEU1PTFy2LqgAADIdJREFU/v37uRXPwbwxSCwWb9++vaenp6urq76+3lfZjz/++OqrrxJCUlNTd+/ePTw8HEhzgUAgl8tlMpk8AM4yfBwEAIiWIIPQV0hwg3kJIc7BvLHpmWeeOX369IULF2pqanzVqFSqU6dOffXVV5TSjz76KEFGKgIAJJogg9BXSHCDeSmlroN5Y9ATTzxx+fLliooKhULhq0YsFnd3dx84cCAzM7Orq+vkyZOR3CEAAERG8PcIuZC4efOmWq3u6Ojg1o8cOVJXV5eTk1NaWtrZ2Rm6fYZYcnLyxo0bPb8Xdbtd99hjj12+fDmC+wIAgEgL/taU15AIZDBvRkaG5PNO0cQ/g7504Ow3viVPFbkt2my2oaGhycnJysrKCOwBAABiWRSe0fjPvx7avLEwUlf790ceecRt6ezZsy+88MKJEyfEYnGktgEAADEqCkEokUh4ntWMgJ07d+7cuTOKGwAAgNiBs0YBACChIQgBACChIQgBACChIQgBACChIQgBACChBR+EWq22qKgoPT1969atY2Nj3GL8ziMEAIDEFOSfT0xMTNTX158/f37z5s1tbW0NDQ39/f3k7jzCTz755JVXXjl27FhLS4tUKv3444+vXbuWlJRkt9sJIW4HTFutVpFIxB3VHSCr1eo8AoZhmNj5c8CFhYVobwEAAH4bJrgZQL29vadPn+aO35yfn9+wYQOXAYWFhVqtVq1Wj4yMaDSa0dFRk8l0//3379mzJzs7e3Z2tr29/fXXX3dt1dLSUlJSsm3bNucKl3MSicTX1d9597+m7854SpWn/cehv/Bs1Ww2BzIa0GQyiUQiv1MgjEYjN0HQV8E999xTW1u7uLioUCj8XnR+fj4jI4PbHk9PAAAInyCD0MnhcOzbt08gELS1tRFCZDLZ/Py8VCo1mUzZ2dkrKyu+Fp3UanVubm5XV5dzxWAwUEplMpmvi26tePzqteuEEMIwmfeuGb/yL54drqysiEQiqVTK/0Z0Op1UKuVJX87S0pJMJvP7GfS3BqFMJvNbDAAA4bCqk2U+//zzgwcPVlZWNjU1cSte5xHyDylMSUlpaWlRKpXOFZFIRCmVy+W+risQ/v+2GYHA9We9FAsEIpEoJSXF79uRSqXJycn8NSzLyuVyv0HocDjS0tL8ZpvNZlMqlfgsCAAQRUEGIaW0sbGxv7+/q6uroKDAuc7NI8zPz3edR+h1MVSsJkNoGwIAQEIJ8qnRgYGB7u7unp4elUql1+v1ej237nUeYRiHFFJ2ZX4mlA0BACDBBBmEvb29o6OjSqVSfhe3fuTIkStXruTk5Fy7du3QoUM8i6GxqvubAAAAq35YZvVKS0vb29tLS0udK3q9nv8eYVrW3SduKCV2C/9bWF5eDuQe4e3btwO5R7i4uBjIPcKFhYVA7hHOzc1lZmbiHiEAQBTF5ckyRqMx2lsAAIDfibgMQgAAgFCJvyC0Wq2sg3VdcT6qAwAA8FvFXxBOTEyw1CUIBUl9fX3R2w4AAMS3+AtCd2LpqVMd0d4EAADEq/gPQtbxj6/7o70JAACIV/EfhJRlmVUdFAcAAIlsVUHocDjUarXritd5hBhSCAAAMSv4IGxtbS0rKxsdHXVd5OYRzszM3HfffceOHeNZBAAAiAXBf6lYXFycl5dXXV3tutjd3a3VaiUSyb59+zQaTUtLi69FJ7vd3tfX9+CDDzpXLBYLpdTX6S1Wq9VthVJqsVh87dNisbAs6/f0FovFIhAI/M4Htlqt3PYCKWNZNpAybh6hQBD/X1MDAMSh4IOwvLzcc3Fqaio3N5cQwn0E5Fl0MpvNH374YX19vXPFaDRSSn0Fg8lk8lw0GHzOoDAajdxcJ/63w13Uc0qU54UYhrHZbH7LApkG7BzzG8iMQwAACIcQP2YSxDxCmUx24sSJjIwM54pYLOY5a/TmzZtuKwzDuP64Gy6Q/J41yjBMIGeNchvzG1osywZy1qjdbucG8/KXAQBA+IT46zhu9CAhxHMeodti0D777LNVdgAAAHAKcRBGYB7h3Nyc24r5zrLdbl9lWwAASEwhDsIIzCO80HvxV/+mrMWw7PbwKgAAQIBWe4/Q7SGU9PT0c+fOudV4XQza8vKK+5JU8ZfGQ2e13aG6BAAAJI44e2R/bm5uZtr9uVPCCPz8mQIAAIAPcRaEs7OzXtctFgvPX1AAAAD4EmdB6J3d9o/er1/Y9+do7wMAAOLP7yIIWTsRMH//3/MLCwvR3goAAMSZ30UQEkIomZ+55Xn6GgAAAL84C8J3333X+wsOG0kSX7p0KbLbAQCAuBdnQfi30//t/QVKCCHT09OR3AwAAPwORCIIQzWP0GazUZ55DozgpYONtbW1er0+6EsAAECiiUQQhmQe4eTk5P1/fIB/hARrt575n7P3/XG9NCV1ZGQkuAsBAEBCYfzOJ1q9wsJCrVarVqtHRkY0Go3bcWibNm06ePBgbW0tIYRl2TNnzmRlZSkUCoPBMDQ0JJVKBwYGvu4fmJi8RQkhDEMoQ5xDAyklhBLCEMItUsIICOsgQpFQKExLTsr+wxq90cjabfnr11dWVl69erWmpmb9+vUXL17U6/W5ubnj4+NlZWVarXbbtm3r1q2TSCQTExMrKytFRUXEZW6Gc3qGTqeTyWRCodBt3flrZBiGYZilpaXl5eWVlZXi4mLXJlyZxWK5evXqc889t7S0lJmZKRQKhUIh5hECAERFJIJQJpPNz89LpVKTyZSdnb2y8qsz0vLz8wkh3HMuFovl3x59dOOGDUqlcmpqanR8XCIWT/88a7dYKCO4GzWEkF/yh/xyc/CXdYYwhFBCnYtUyDAOlkokIrudvSdTuXh7+aGSTWq1+tPP/s4wTFpa+s8/z5SWbB64dGnro4+qVCqBQHB9eNhmtW/a9CdngLkym80ikSgpKYl7lfvtMQzDzeB1hpnZbL5x44bJbC4tKSF3I9D5Izqd7vsrQxe/6jUYDEqlUigUKhQKzCMEAIiKEM8j9Ip/HqFcLm9vb8/KyuL+efPGjXPnzjkcjh07dvC3PXPmjFKpfPzxx/nLPvjgg7y8vC1btvCXvf/++w8//HBpaSl/2fHjx6uqqgoLC/nLmpub33rrrZycHP6yw4cPv/TSS5mZmfxlAAAQPpH4RJifn//pp5/m5+ePj49v3759bGzM9dUnn3xSp9O5Ds69desWpdRvivz0009isXjNmjX8ZT/88INcLr/33nv5y8bGxrKysngG/HKuX7++bt06hULBXzY0NPTAAw+kpqbylw0ODm7cuFEikbzxxhtlZWX8xQAAEA6RCMIDBw5IJJLm5ubGxka73f7222+7vjo/Pz84OBjuPcS4zZs3+41qAAAIh0gEoU6nq6ur+/7770tLSzs7O9PS0sJ9RQAAgABFIggBAABiFh7ZBwCAhIYgBACAhIYgBACAhBajQbhlyxbmrr1793oWBHh+KX8fh8OhVqv9tvUs82yr1WqLiorS09O3bt3q/PsQz25ey/y+WQAACJ9YDEJK6cjIyK1bt+7cuXPnzh2vo5cCOb+Uv09ra2tZWZnbeW+ebT3LPNtOTEzU19efPHlyZmZmx44dDQ0NXrt5LQvkzQIAQBjR2DMzMyOTyR566CGZTKbRaGZnZz1rCgoKhoeHKaXDw8MFBQVB9Pniiy/Onj3r9hvwbOtZ5tn2yy+/3LNnD/fq3NxcZmam125eywJ5swAAED6xGISDg4Pl5eWDg4OLi4u7du169tlnPWtSU1ONRiOl1Gg0yuXyoPu4BaGvtq5lPG3tdvvevXtffPFF/m6uZYFsEgAAwidWgtB5eqfb+vT0tFKp9KxPSUkxmUyUUoPBkJKS4re/rz5uV/TV1tdHZ9e2Fy5cKCkpee2112w2G083zzK/mwQAgPCJxKHbgXAdH/jdd9+ZzWbu7E2xWCyRSDzrVSrV5ORkfn7+1NTU2rVrvfYMpE9I2lJKGxsb+/v7u7q6CgoKfHXzWhbEJgEAIIRi8WEZg8FQU1MzPDxstVqPHj361FNPedZUV1d3dHRQSjs6OjQaTdB9QtJ2YGCgu7u7p6dHpVLp9Xq9Xu+1m9eyIDYJAAChFNXPo96xLNvW1paXl5eVlbVr167l5WXPmtu3b1dVVa1du7a6ulqn0wXdx+034Kuta5ln26amJq+/VbduXssC2SQAAIQPzhoFAICEFotfjQIAAEQMghAAABIaghAAABIaghAAABIaghAAABIaghAAABIaghAAABIaghAAABIaghAAABIaghAAABIaghAAABLa/wG2jEm26H1ojAAAAABJRU5ErkJggg==" - }, - "execution_count": 65, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "Y = convert(Matrix{Float64}, cg10k_trait[:, 3:15])\n", - "histogram(Y, layout = 13)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Pre-processing data for heritability analysis" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "To prepare variance component model fitting, we form an instance of `VarianceComponentVariate`. The two variance components are $(2\\Phi, I)$." - ] - }, - { - "cell_type": "code", - "execution_count": 66, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(:Y, :X, :V)" - ] - }, - "execution_count": 66, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "using VarianceComponentModels, LinearAlgebra\n", - "\n", - "# form data as VarianceComponentVariate\n", - "cg10kdata = VarianceComponentVariate(Y, (2Φgrm, Matrix(1.0I, size(Y, 1), size(Y, 1))))\n", - "fieldnames(typeof(cg10kdata))" - ] - }, - { - "cell_type": "code", - "execution_count": 67, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "VarianceComponentVariate{Float64,2,Array{Float64,2},Array{Float64,2},Array{Float64,2}}([-1.81573 -0.94615 … -1.02853 -0.394049; -1.2444 0.10966 … 1.09065 0.0256616; … ; 0.886626 0.487408 … -0.636874 -0.439825; -1.24394 0.213697 … 0.299931 0.392809], Array{Float64}(6670,0), ([1.00547 0.00656224 … -0.000121918 -0.00555374; 0.00656224 0.99614 … 0.00176999 0.00683183; … ; -0.000121918 0.00176999 … 0.998578 0.000962983; -0.00555374 0.00683183 … 0.000962983 0.999599], [1.0 0.0 … 0.0 0.0; 0.0 1.0 … 0.0 0.0; … ; 0.0 0.0 … 1.0 0.0; 0.0 0.0 … 0.0 1.0]))" - ] - }, - "execution_count": 67, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "cg10kdata" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Before fitting the variance component model, we pre-compute the eigen-decomposition of $2\\Phi_{\\text{GRM}}$, the rotated responses, and the constant part in log-likelihood, and store them as a `TwoVarCompVariateRotate` instance, which is re-used in various variane component estimation procedures." - ] - }, - { - "cell_type": "code", - "execution_count": 68, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " 51.732018 seconds (29 allocations: 1021.427 MiB, 0.98% gc time)\n" - ] - }, - { - "data": { - "text/plain": [ - "(:Yrot, :Xrot, :eigval, :eigvec, :logdetV2)" - ] - }, - "execution_count": 68, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# pre-compute eigen-decomposition (~50 secs on my laptop)\n", - "@time cg10kdata_rotated = TwoVarCompVariateRotate(cg10kdata)\n", - "fieldnames(typeof(cg10kdata_rotated))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Save intermediate results\n", - "\n", - "We don't want to re-compute SnpArray and empirical kinship matrices again and again for heritibility analysis." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "To load workspace" - ] - }, - { - "cell_type": "code", - "execution_count": 83, - "metadata": {}, - "outputs": [ - { - "data": { - "text/latex": [ - "\\begin{tabular}\n", - "{l | r | l}\n", - "name & size & summary \\\\\n", - "\\hline\n", - "Base & & Module \\\\\n", - "Core & & Module \\\\\n", - "Main & & Module \\\\\n", - "Plots & 22.448 MiB & Module \\\\\n", - "Y & 677.461 KiB & 6670×13 Array\\{Float64,2\\} \\\\\n", - "cg10k & 1022.983 MiB & 6670×630860 SnpArray \\\\\n", - "cg10k\\_trait & 1.386 MiB & 6670×15 DataFrame \\\\\n", - "cg10kdata & 679.508 MiB & VarianceComponentVariate\\{Float64,2,Array\\{Float64,2\\},Array\\{Float64,2\\},Array\\{Float64,2\\}\\} \\\\\n", - "cg10kdata\\_rotated & 340.136 MiB & TwoVarCompVariateRotate\\{Float64,Array\\{Float64,2\\},Array\\{Float64,2\\}\\} \\\\\n", - "maf\\_cg10k & 4.813 MiB & 630860-element Array\\{Float64,1\\} \\\\\n", - "missings\\_by\\_snp & 4.813 MiB & 1×630860 Array\\{Int64,2\\} \\\\\n", - "mp & 28.748 MiB & 6670×630860 SparseMatrixCSC\\{Bool,Int32\\} \\\\\n", - "people & 8 bytes & Int64 \\\\\n", - "snps & 8 bytes & Int64 \\\\\n", - "startupfile & 57 bytes & String \\\\\n", - "Φgrm & 339.423 MiB & 6670×6670 Array\\{Float64,2\\} \\\\\n", - "\\end{tabular}\n" - ], - "text/markdown": [ - "| name | size | summary |\n", - "|:----------------- | ------------:|:-------------------------------------------------------------------------------------- |\n", - "| Base | | Module |\n", - "| Core | | Module |\n", - "| Main | | Module |\n", - "| Plots | 22.448 MiB | Module |\n", - "| Y | 677.461 KiB | 6670×13 Array{Float64,2} |\n", - "| cg10k | 1022.983 MiB | 6670×630860 SnpArray |\n", - "| cg10k_trait | 1.386 MiB | 6670×15 DataFrame |\n", - "| cg10kdata | 679.508 MiB | VarianceComponentVariate{Float64,2,Array{Float64,2},Array{Float64,2},Array{Float64,2}} |\n", - "| cg10kdata_rotated | 340.136 MiB | TwoVarCompVariateRotate{Float64,Array{Float64,2},Array{Float64,2}} |\n", - "| maf_cg10k | 4.813 MiB | 630860-element Array{Float64,1} |\n", - "| missings_by_snp | 4.813 MiB | 1×630860 Array{Int64,2} |\n", - "| mp | 28.748 MiB | 6670×630860 SparseMatrixCSC{Bool,Int32} |\n", - "| people | 8 bytes | Int64 |\n", - "| snps | 8 bytes | Int64 |\n", - "| startupfile | 57 bytes | String |\n", - "| Φgrm | 339.423 MiB | 6670×6670 Array{Float64,2} |\n" - ], - "text/plain": [ - "name size summary \n", - "––––––––––––––––– –––––––––––– ––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––\n", - "Base Module \n", - "Core Module \n", - "Main Module \n", - "Plots 22.448 MiB Module \n", - "Y 677.461 KiB 6670×13 Array{Float64,2} \n", - "cg10k 1022.983 MiB 6670×630860 SnpArray \n", - "cg10k_trait 1.386 MiB 6670×15 DataFrame \n", - "cg10kdata 679.508 MiB VarianceComponentVariate{Float64,2,Array{Float64,2},Array{Float64,2},Array{Float64,2}}\n", - "cg10kdata_rotated 340.136 MiB TwoVarCompVariateRotate{Float64,Array{Float64,2},Array{Float64,2}} \n", - "maf_cg10k 4.813 MiB 630860-element Array{Float64,1} \n", - "missings_by_snp 4.813 MiB 1×630860 Array{Int64,2} \n", - "mp 28.748 MiB 6670×630860 SparseMatrixCSC{Bool,Int32} \n", - "people 8 bytes Int64 \n", - "snps 8 bytes Int64 \n", - "startupfile 57 bytes String \n", - "Φgrm 339.423 MiB 6670×6670 Array{Float64,2} " - ] - }, - "execution_count": 83, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "#pkg\"add FileIO JLD2\"\n", - "\n", - "using JLD2, FileIO\n", - "save(\"cg10k.jld2\")\n", - "varinfo()" - ] - }, - { - "cell_type": "code", - "execution_count": 84, - "metadata": {}, - "outputs": [ - { - "data": { - "text/latex": [ - "\\begin{tabular}\n", - "{l | r | l}\n", - "name & size & summary \\\\\n", - "\\hline\n", - "Base & & Module \\\\\n", - "Core & & Module \\\\\n", - "Main & & Module \\\\\n", - "Plots & 22.448 MiB & Module \\\\\n", - "Y & 677.461 KiB & 6670×13 Array\\{Float64,2\\} \\\\\n", - "cg10k & 1022.983 MiB & 6670×630860 SnpArray \\\\\n", - "cg10k\\_trait & 1.449 MiB & 6670×15 DataFrame \\\\\n", - "cg10kdata & 679.508 MiB & VarianceComponentVariate\\{Float64,2,Array\\{Float64,2\\},Array\\{Float64,2\\},Array\\{Float64,2\\}\\} \\\\\n", - "cg10kdata\\_rotated & 340.136 MiB & TwoVarCompVariateRotate\\{Float64,Array\\{Float64,2\\},Array\\{Float64,2\\}\\} \\\\\n", - "maf\\_cg10k & 4.813 MiB & 630860-element Array\\{Float64,1\\} \\\\\n", - "missings\\_by\\_snp & 4.813 MiB & 1×630860 Array\\{Int64,2\\} \\\\\n", - "mp & 28.748 MiB & 6670×630860 SparseMatrixCSC\\{Bool,Int32\\} \\\\\n", - "people & 8 bytes & Int64 \\\\\n", - "snps & 8 bytes & Int64 \\\\\n", - "startupfile & 57 bytes & String \\\\\n", - "Φgrm & 339.423 MiB & 6670×6670 Array\\{Float64,2\\} \\\\\n", - "\\end{tabular}\n" - ], - "text/markdown": [ - "| name | size | summary |\n", - "|:----------------- | ------------:|:-------------------------------------------------------------------------------------- |\n", - "| Base | | Module |\n", - "| Core | | Module |\n", - "| Main | | Module |\n", - "| Plots | 22.448 MiB | Module |\n", - "| Y | 677.461 KiB | 6670×13 Array{Float64,2} |\n", - "| cg10k | 1022.983 MiB | 6670×630860 SnpArray |\n", - "| cg10k_trait | 1.449 MiB | 6670×15 DataFrame |\n", - "| cg10kdata | 679.508 MiB | VarianceComponentVariate{Float64,2,Array{Float64,2},Array{Float64,2},Array{Float64,2}} |\n", - "| cg10kdata_rotated | 340.136 MiB | TwoVarCompVariateRotate{Float64,Array{Float64,2},Array{Float64,2}} |\n", - "| maf_cg10k | 4.813 MiB | 630860-element Array{Float64,1} |\n", - "| missings_by_snp | 4.813 MiB | 1×630860 Array{Int64,2} |\n", - "| mp | 28.748 MiB | 6670×630860 SparseMatrixCSC{Bool,Int32} |\n", - "| people | 8 bytes | Int64 |\n", - "| snps | 8 bytes | Int64 |\n", - "| startupfile | 57 bytes | String |\n", - "| Φgrm | 339.423 MiB | 6670×6670 Array{Float64,2} |\n" - ], - "text/plain": [ - "name size summary \n", - "––––––––––––––––– –––––––––––– ––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––\n", - "Base Module \n", - "Core Module \n", - "Main Module \n", - "Plots 22.448 MiB Module \n", - "Y 677.461 KiB 6670×13 Array{Float64,2} \n", - "cg10k 1022.983 MiB 6670×630860 SnpArray \n", - "cg10k_trait 1.449 MiB 6670×15 DataFrame \n", - "cg10kdata 679.508 MiB VarianceComponentVariate{Float64,2,Array{Float64,2},Array{Float64,2},Array{Float64,2}}\n", - "cg10kdata_rotated 340.136 MiB TwoVarCompVariateRotate{Float64,Array{Float64,2},Array{Float64,2}} \n", - "maf_cg10k 4.813 MiB 630860-element Array{Float64,1} \n", - "missings_by_snp 4.813 MiB 1×630860 Array{Int64,2} \n", - "mp 28.748 MiB 6670×630860 SparseMatrixCSC{Bool,Int32} \n", - "people 8 bytes Int64 \n", - "snps 8 bytes Int64 \n", - "startupfile 57 bytes String \n", - "Φgrm 339.423 MiB 6670×6670 Array{Float64,2} " - ] - }, - "execution_count": 84, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "#using SnpArrays, JLD2, DataFrames, VarianceComponentModels, Plots\n", - "#pyplot()\n", - "load(\"cg10k.jld\")\n", - "varinfo()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Heritability of single traits" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We use Fisher scoring algorithm to fit variance component model for each single trait." - ] - }, - { - "cell_type": "code", - "execution_count": 85, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Trait1\n", - "\n", - "******************************************************************************\n", - "This program contains Ipopt, a library for large-scale nonlinear optimization.\n", - " Ipopt is released as open source code under the Eclipse Public License (EPL).\n", - " For more information visit http://projects.coin-or.org/Ipopt\n", - "******************************************************************************\n", - "\n", - "(σ2a[trait], σ2e[trait]) = (0.25947604776534905, 0.7375938878381869)\n", - "Trait2\n", - "(σ2a[trait], σ2e[trait]) = (0.18588150448759697, 0.8137880281704691)\n", - "Trait3\n", - "(σ2a[trait], σ2e[trait]) = (0.3196093756535387, 0.6795300238388639)\n", - "Trait4\n", - "(σ2a[trait], σ2e[trait]) = (0.2657824469749975, 0.7304994606945179)\n", - "Trait5\n", - "(σ2a[trait], σ2e[trait]) = (0.28143198005766734, 0.7169747061765026)\n", - "Trait6\n", - "(σ2a[trait], σ2e[trait]) = (0.28300551312417344, 0.7168800753377119)\n", - "Trait7\n", - "(σ2a[trait], σ2e[trait]) = (0.21565421413116753, 0.7816910320891622)\n", - "Trait8\n", - "(σ2a[trait], σ2e[trait]) = (0.19408878271207622, 0.8058201577783586)\n", - "Trait9\n", - "(σ2a[trait], σ2e[trait]) = (0.24746236011764494, 0.7512222977091667)\n", - "Trait10\n", - "(σ2a[trait], σ2e[trait]) = (0.0992417256213372, 0.9007769787053679)\n", - "Trait11\n", - "(σ2a[trait], σ2e[trait]) = (0.16457266481403124, 0.8343110221526249)\n", - "Trait12\n", - "(σ2a[trait], σ2e[trait]) = (0.08224956591863997, 0.9166483378364418)\n", - "Trait13\n", - "(σ2a[trait], σ2e[trait]) = (0.05687679106196685, 0.9424058676223442)\n", - " 6.559467 seconds (12.64 M allocations: 611.362 MiB, 11.95% gc time)\n" - ] - } - ], - "source": [ - "# heritability from single trait analysis\n", - "hST = zeros(13)\n", - "# standard errors of estimated heritability\n", - "hST_se = zeros(13)\n", - "# additive genetic effects\n", - "σ2a = zeros(13)\n", - "# enviromental effects\n", - "σ2e = zeros(13)\n", - "\n", - "@time for trait in 1:13\n", - " println(names(cg10k_trait)[trait + 2])\n", - " # form data set for trait j\n", - " traitj_data = TwoVarCompVariateRotate(cg10kdata_rotated.Yrot[:, trait], cg10kdata_rotated.Xrot, \n", - " cg10kdata_rotated.eigval, cg10kdata_rotated.eigvec, cg10kdata_rotated.logdetV2)\n", - " # initialize model parameters\n", - " traitj_model = VarianceComponentModel(traitj_data)\n", - " # estimate variance components\n", - " _, _, _, Σcov, _, _ = mle_fs!(traitj_model, traitj_data; solver=:Ipopt, verbose=false)\n", - " σ2a[trait] = traitj_model.Σ[1][1]\n", - " σ2e[trait] = traitj_model.Σ[2][1]\n", - " @show σ2a[trait], σ2e[trait]\n", - " h, hse = heritability(traitj_model.Σ, Σcov)\n", - " hST[trait] = h[1]\n", - " hST_se[trait] = hse[1]\n", - "end" - ] - }, - { - "cell_type": "code", - "execution_count": 86, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "2×13 Array{Float64,2}:\n", - " 0.260239 0.185943 0.319885 … 0.164757 0.0823403 0.0569176\n", - " 0.0799434 0.08689 0.0739664 0.0887138 0.0944375 0.0953072" - ] - }, - "execution_count": 86, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# heritability and standard errors\n", - "[hST'; hST_se']" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Pairwise traits\n", - "\n", - "Joint analysis of multiple traits is subject to intensive research recently. Following code snippet does joint analysis of all pairs of traits, a total of 78 bivariate variane component models." - ] - }, - { - "cell_type": "code", - "execution_count": 87, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Trait1Trait2\n", - "(Σa[i, j], Σe[i, j]) = ([0.258529 0.173943; 0.173943 0.184525], [0.738519 0.58639; 0.58639 0.815122])\n", - "Trait1Trait3\n", - "(Σa[i, j], Σe[i, j]) = ([0.259934 -0.0139395; -0.0139395 0.320025], [0.737149 -0.120342; -0.120342 0.67913])\n", - "Trait1Trait4\n", - "(Σa[i, j], Σe[i, j]) = ([0.259307 0.222267; 0.222267 0.265791], [0.737759 0.600077; 0.600077 0.730492])\n", - "Trait1Trait5\n", - "(Σa[i, j], Σe[i, j]) = ([0.259264 -0.147125; -0.147125 0.282102], [0.737798 -0.254669; -0.254669 0.716327])\n", - "Trait1Trait6\n", - "(Σa[i, j], Σe[i, j]) = ([0.259171 -0.129519; -0.129519 0.283267], [0.73789 -0.231373; -0.231373 0.716628])\n", - "Trait1Trait7\n", - "(Σa[i, j], Σe[i, j]) = ([0.258784 -0.140357; -0.140357 0.21535], [0.738269 -0.197894; -0.197894 0.782002])\n", - "Trait1Trait8\n", - "(Σa[i, j], Σe[i, j]) = ([0.259467 -0.0336174; -0.0336174 0.194107], [0.737603 -0.12623; -0.12623 0.805802])\n", - "Trait1Trait9\n", - "(Σa[i, j], Σe[i, j]) = ([0.261524 -0.204467; -0.204467 0.246378], [0.735632 -0.308118; -0.308118 0.752288])\n", - "Trait1Trait10\n", - "(Σa[i, j], Σe[i, j]) = ([0.259326 -0.0987008; -0.0987008 0.0962217], [0.73774 -0.304854; -0.304854 0.90378])\n", - "Trait1Trait11\n", - "(Σa[i, j], Σe[i, j]) = ([0.259194 -0.137578; -0.137578 0.162813], [0.737868 -0.36076; -0.36076 0.836058])\n", - "Trait1Trait12\n", - "(Σa[i, j], Σe[i, j]) = ([0.26168 -0.145305; -0.145305 0.0806851], [0.735514 -0.0421192; -0.0421192 0.918294])\n", - "Trait1Trait13\n", - "(Σa[i, j], Σe[i, j]) = ([0.260776 -0.108277; -0.108277 0.0513909], [0.736359 -0.114756; -0.114756 0.947913])\n", - "Trait2Trait3\n", - "(Σa[i, j], Σe[i, j]) = ([0.185992 0.14505; 0.14505 0.321413], [0.813678 0.0987898; 0.0987898 0.677805])\n", - "Trait2Trait4\n", - "(Σa[i, j], Σe[i, j]) = ([0.18554 0.074367; 0.074367 0.26577], [0.814123 0.22144; 0.22144 0.730512])\n", - "Trait2Trait5\n", - "(Σa[i, j], Σe[i, j]) = ([0.185854 -0.0115021; -0.0115021 0.281443], [0.813815 -0.0368393; -0.0368393 0.716964])\n", - "Trait2Trait6\n", - "(Σa[i, j], Σe[i, j]) = ([0.185903 -0.00350261; -0.00350261 0.283064], [0.813767 -0.0207933; -0.0207933 0.716823])\n", - "Trait2Trait7\n", - "(Σa[i, j], Σe[i, j]) = ([0.185498 -0.0301226; -0.0301226 0.215413], [0.814164 -0.00124563; -0.00124563 0.781929])\n", - "Trait2Trait8\n", - "(Σa[i, j], Σe[i, j]) = ([0.186397 0.032688; 0.032688 0.194625], [0.813285 -0.0321045; -0.0321045 0.805301])\n", - "Trait2Trait9\n", - "(Σa[i, j], Σe[i, j]) = ([0.184426 -0.084907; -0.084907 0.246288], [0.815228 -0.0814978; -0.0814978 0.752373])\n", - "Trait2Trait10\n", - "(Σa[i, j], Σe[i, j]) = ([0.18604 -0.123261; -0.123261 0.0992567], [0.813659 -0.273285; -0.273285 0.90084])\n", - "Trait2Trait11\n", - "(Σa[i, j], Σe[i, j]) = ([0.184894 -0.116422; -0.116422 0.165892], [0.814757 -0.297755; -0.297755 0.833033])\n", - "Trait2Trait12\n", - "(Σa[i, j], Σe[i, j]) = ([0.185297 -0.0908872; -0.0908872 0.0814927], [0.814404 0.0456529; 0.0456529 0.917465])\n", - "Trait2Trait13\n", - "(Σa[i, j], Σe[i, j]) = ([0.185348 -0.07104; -0.07104 0.0546791], [0.814367 0.0740412; 0.0740412 0.944638])\n", - "Trait3Trait4\n", - "(Σa[i, j], Σe[i, j]) = ([0.319575 -0.15468; -0.15468 0.265006], [0.679563 -0.303309; -0.303309 0.731254])\n", - "Trait3Trait5\n", - "(Σa[i, j], Σe[i, j]) = ([0.320016 0.184736; 0.184736 0.282719], [0.679136 0.336276; 0.336276 0.715725])\n", - "Trait3Trait6\n", - "(Σa[i, j], Σe[i, j]) = ([0.320608 0.166929; 0.166929 0.285124], [0.678565 0.297633; 0.297633 0.714823])\n", - "Trait3Trait7\n", - "(Σa[i, j], Σe[i, j]) = ([0.319614 0.167317; 0.167317 0.215458], [0.679526 0.346897; 0.346897 0.781883])\n", - "Trait3Trait8\n", - "(Σa[i, j], Σe[i, j]) = ([0.321532 0.0573397; 0.0573397 0.197221], [0.677674 0.0445311; 0.0445311 0.802757])\n", - "Trait3Trait9\n", - "(Σa[i, j], Σe[i, j]) = ([0.319777 0.138268; 0.138268 0.246547], [0.679369 0.266317; 0.266317 0.752112])\n", - "Trait3Trait10\n", - "(Σa[i, j], Σe[i, j]) = ([0.319905 -0.0779601; -0.0779601 0.100192], [0.679245 -0.141562; -0.141562 0.899839])\n", - "Trait3Trait11\n", - "(Σa[i, j], Σe[i, j]) = ([0.318882 -0.0177352; -0.0177352 0.164446], [0.680233 -0.114434; -0.114434 0.834435])\n", - "Trait3Trait12\n", - "(Σa[i, j], Σe[i, j]) = ([0.321958 0.0844644; 0.0844644 0.0863388], [0.67727 0.034186; 0.034186 0.912603])\n", - "Trait3Trait13\n", - "(Σa[i, j], Σe[i, j]) = ([0.32405 0.109731; 0.109731 0.0613165], [0.675287 -0.00619989; -0.00619989 0.938011])\n", - "Trait4Trait5\n", - "(Σa[i, j], Σe[i, j]) = ([0.26593 -0.216116; -0.216116 0.283173], [0.730347 -0.376697; -0.376697 0.715289])\n", - "Trait4Trait6\n", - "(Σa[i, j], Σe[i, j]) = ([0.266395 -0.200793; -0.200793 0.284547], [0.729899 -0.346915; -0.346915 0.715387])\n", - "Trait4Trait7\n", - "(Σa[i, j], Σe[i, j]) = ([0.264731 -0.182912; -0.182912 0.21441], [0.731526 -0.326257; -0.326257 0.782928])\n", - "Trait4Trait8\n", - "(Σa[i, j], Σe[i, j]) = ([0.266914 -0.0976281; -0.0976281 0.196119], [0.729401 -0.150498; -0.150498 0.803836])\n", - "Trait4Trait9\n", - "(Σa[i, j], Σe[i, j]) = ([0.270267 -0.227463; -0.227463 0.247665], [0.726155 -0.415849; -0.415849 0.751008])\n", - "Trait4Trait10\n", - "(Σa[i, j], Σe[i, j]) = ([0.265756 -0.0339156; -0.0339156 0.0987618], [0.730536 -0.227666; -0.227666 0.901251])\n", - "Trait4Trait11\n", - "(Σa[i, j], Σe[i, j]) = ([0.265858 -0.0963336; -0.0963336 0.163013], [0.730427 -0.273145; -0.273145 0.835846])\n", - "Trait4Trait12\n", - "(Σa[i, j], Σe[i, j]) = ([0.268394 -0.141452; -0.141452 0.0796851], [0.72801 -0.0831954; -0.0831954 0.919263])\n", - "Trait4Trait13\n", - "(Σa[i, j], Σe[i, j]) = ([0.266334 -0.0971798; -0.0971798 0.0541478], [0.729966 -0.226082; -0.226082 0.945138])\n", - "Trait5Trait6\n", - "(Σa[i, j], Σe[i, j]) = ([0.281794 0.281033; 0.281033 0.282364], [0.716628 0.660607; 0.660607 0.717505])\n", - "Trait5Trait7\n", - "(Σa[i, j], Σe[i, j]) = ([0.281023 0.232223; 0.232223 0.211943], [0.717383 0.67449; 0.67449 0.785343])\n", - "Trait5Trait8\n", - "(Σa[i, j], Σe[i, j]) = ([0.281551 0.164011; 0.164011 0.192737], [0.716865 0.221187; 0.221187 0.807144])\n", - "Trait5Trait9\n", - "(Σa[i, j], Σe[i, j]) = ([0.284085 0.244479; 0.244479 0.240901], [0.714415 0.508795; 0.508795 0.757606])\n", - "Trait5Trait10\n", - "(Σa[i, j], Σe[i, j]) = ([0.281993 -0.0460624; -0.0460624 0.100628], [0.716433 -0.0574194; -0.0574194 0.899407])\n", - "Trait5Trait11\n", - "(Σa[i, j], Σe[i, j]) = ([0.280652 0.0199425; 0.0199425 0.163738], [0.717732 -0.0349079; -0.0349079 0.83513])\n", - "Trait5Trait12\n", - "(Σa[i, j], Σe[i, j]) = ([0.281652 0.0612882; 0.0612882 0.0820476], [0.716765 0.0533362; 0.0533362 0.916851])\n", - "Trait5Trait13\n", - "(Σa[i, j], Σe[i, j]) = ([0.282494 0.0696137; 0.0696137 0.0570685], [0.715955 0.0537392; 0.0537392 0.942222])\n", - "Trait6Trait7\n", - "(Σa[i, j], Σe[i, j]) = ([0.283018 0.220794; 0.220794 0.214128], [0.716868 0.58124; 0.58124 0.783191])\n", - "Trait6Trait8\n", - "(Σa[i, j], Σe[i, j]) = ([0.283018 0.183995; 0.183995 0.192405], [0.716869 0.436926; 0.436926 0.807474])\n", - "Trait6Trait9\n", - "(Σa[i, j], Σe[i, j]) = ([0.285057 0.234311; 0.234311 0.24283], [0.714902 0.477263; 0.477263 0.755728])\n", - "Trait6Trait10\n", - "(Σa[i, j], Σe[i, j]) = ([0.283699 -0.0433873; -0.0433873 0.101183], [0.716209 -0.0593852; -0.0593852 0.89886])\n", - "Trait6Trait11\n", - "(Σa[i, j], Σe[i, j]) = ([0.281626 0.027639; 0.027639 0.163178], [0.718219 -0.0520113; -0.0520113 0.83568])\n", - "Trait6Trait12\n", - "(Σa[i, j], Σe[i, j]) = ([0.283196 0.0569533; 0.0569533 0.0819944], [0.716699 0.0481832; 0.0481832 0.916905])\n", - "Trait6Trait13\n", - "(Σa[i, j], Σe[i, j]) = ([0.283862 0.0600111; 0.0600111 0.0571748], [0.716057 0.0544601; 0.0544601 0.942116])\n", - "Trait7Trait8\n", - "(Σa[i, j], Σe[i, j]) = ([0.214129 0.0883281; 0.0883281 0.192399], [0.78319 -0.0565873; -0.0565873 0.80748])\n", - "Trait7Trait9\n", - "(Σa[i, j], Σe[i, j]) = ([0.219059 0.217228; 0.217228 0.243722], [0.778422 0.463004; 0.463004 0.754868])\n", - "Trait7Trait10\n", - "(Σa[i, j], Σe[i, j]) = ([0.216477 -0.042081; -0.042081 0.101258], [0.78089 -0.0859952; -0.0859952 0.898786])\n", - "Trait7Trait11\n", - "(Σa[i, j], Σe[i, j]) = ([0.214308 0.0205244; 0.0205244 0.163203], [0.783006 -0.0479474; -0.0479474 0.835656])\n", - "Trait7Trait12\n", - "(Σa[i, j], Σe[i, j]) = ([0.215192 0.0752708; 0.0752708 0.0802075], [0.782157 0.0353123; 0.0353123 0.918686])\n", - "Trait7Trait13\n", - "(Σa[i, j], Σe[i, j]) = ([0.216198 0.0740251; 0.0740251 0.0547527], [0.781186 0.0399163; 0.0399163 0.944534])\n", - "Trait8Trait9\n", - "(Σa[i, j], Σe[i, j]) = ([0.194551 0.112282; 0.112282 0.246832], [0.805384 0.185461; 0.185461 0.751837])\n", - "Trait8Trait10\n", - "(Σa[i, j], Σe[i, j]) = ([0.194399 -0.0154035; -0.0154035 0.0995883], [0.805519 0.0117323; 0.0117323 0.900435])\n", - "Trait8Trait11\n", - "(Σa[i, j], Σe[i, j]) = ([0.193837 0.022072; 0.022072 0.164393], [0.806067 -0.0267843; -0.0267843 0.834489])\n", - "Trait8Trait12\n", - "(Σa[i, j], Σe[i, j]) = ([0.193914 -0.00259164; -0.00259164 0.0821237], [0.805992 0.0333257; 0.0333257 0.916773])\n", - "Trait8Trait13\n", - "(Σa[i, j], Σe[i, j]) = ([0.193913 0.00330925; 0.00330925 0.0569126], [0.805993 0.0386552; 0.0386552 0.94237])\n", - "Trait9Trait10\n", - "(Σa[i, j], Σe[i, j]) = ([0.246837 -0.00323396; -0.00323396 0.0989732], [0.751834 0.074992; 0.074992 0.901043])\n", - "Trait9Trait11\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(Σa[i, j], Σe[i, j]) = ([0.247395 0.0305136; 0.0305136 0.164575], [0.751288 0.153633; 0.153633 0.834308])\n", - "Trait9Trait12\n", - "(Σa[i, j], Σe[i, j]) = ([0.249928 0.0844216; 0.0844216 0.0880992], [0.748829 0.108018; 0.108018 0.910884])\n", - "Trait9Trait13\n", - "(Σa[i, j], Σe[i, j]) = ([0.248998 0.0925603; 0.0925603 0.0580327], [0.749747 0.0992651; 0.0992651 0.94131])\n", - "Trait10Trait11\n", - "(Σa[i, j], Σe[i, j]) = ([0.0923059 0.0990858; 0.0990858 0.164638], [0.907657 0.475503; 0.475503 0.834248])\n", - "Trait10Trait12\n", - "(Σa[i, j], Σe[i, j]) = ([0.0957712 0.0574496; 0.0574496 0.0785646], [0.904228 0.0843534; 0.0843534 0.920326])\n", - "Trait10Trait13\n", - "(Σa[i, j], Σe[i, j]) = ([0.100109 -0.0266483; -0.0266483 0.0578511], [0.899941 0.164674; 0.164674 0.941451])\n", - "Trait11Trait12\n", - "(Σa[i, j], Σe[i, j]) = ([0.163544 0.0571549; 0.0571549 0.0784378], [0.835325 0.145549; 0.145549 0.920432])\n", - "Trait11Trait13\n", - "(Σa[i, j], Σe[i, j]) = ([0.164571 -0.00169939; -0.00169939 0.0575331], [0.834326 0.200725; 0.200725 0.941755])\n", - "Trait12Trait13\n", - "(Σa[i, j], Σe[i, j]) = ([0.08398 0.0685417; 0.0685417 0.0559416], [0.91494 0.573206; 0.573206 0.943343])\n", - " 4.258555 seconds (4.55 M allocations: 306.219 MiB, 3.55% gc time)\n" - ] - } - ], - "source": [ - "# additive genetic effects (2x2 psd matrices) from bavariate trait analysis;\n", - "Σa = Array{Matrix{Float64}}(undef, 13, 13)\n", - "# environmental effects (2x2 psd matrices) from bavariate trait analysis;\n", - "Σe = Array{Matrix{Float64}}(undef, 13, 13)\n", - "\n", - "\n", - "@time for i in 1:13\n", - " for j in (i+1):13\n", - " println(names(cg10k_trait)[i + 2], names(cg10k_trait)[j + 2])\n", - " # form data set for (trait1, trait2)\n", - " traitij_data = TwoVarCompVariateRotate(cg10kdata_rotated.Yrot[:, [i;j]], cg10kdata_rotated.Xrot, \n", - " cg10kdata_rotated.eigval, cg10kdata_rotated.eigvec, cg10kdata_rotated.logdetV2)\n", - " # initialize model parameters\n", - " traitij_model = VarianceComponentModel(traitij_data)\n", - " # estimate variance components\n", - " mle_fs!(traitij_model, traitij_data; solver=:Ipopt, verbose=false)\n", - " Σa[i, j] = traitij_model.Σ[1]\n", - " Σe[i, j] = traitij_model.Σ[2]\n", - " @show Σa[i, j], Σe[i, j]\n", - " end\n", - "end" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 3-trait analysis\n", - "\n", - "Researchers want to jointly analyze traits 5-7. Our strategy is to try both Fisher scoring and MM algorithm with different starting point, and choose the best local optimum. We first form the data set and run Fisher scoring, which yields a final objective value -1.4700991+04." - ] - }, - { - "cell_type": "code", - "execution_count": 88, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "This is Ipopt version 3.12.10, running with linear solver mumps.\n", - "NOTE: Other linear solvers might be more efficient (see Ipopt documentation).\n", - "\n", - "Number of nonzeros in equality constraint Jacobian...: 0\n", - "Number of nonzeros in inequality constraint Jacobian.: 0\n", - "Number of nonzeros in Lagrangian Hessian.............: 78\n", - "\n", - "Total number of variables............................: 12\n", - " variables with only lower bounds: 0\n", - " variables with lower and upper bounds: 0\n", - " variables with only upper bounds: 0\n", - "Total number of equality constraints.................: 0\n", - "Total number of inequality constraints...............: 0\n", - " inequality constraints with only lower bounds: 0\n", - " inequality constraints with lower and upper bounds: 0\n", - " inequality constraints with only upper bounds: 0\n", - "\n", - "iter objective inf_pr inf_du lg(mu) ||d|| lg(rg) alpha_du alpha_pr ls\n", - " 0 3.0244169e+04 0.00e+00 1.00e+02 0.0 0.00e+00 - 0.00e+00 0.00e+00 0 \n", - " 5 1.6834042e+04 0.00e+00 4.07e+02 -11.0 3.66e-01 - 1.00e+00 1.00e+00f 1 MaxS\n", - " 10 1.4744248e+04 0.00e+00 1.12e+02 -11.0 2.36e-01 - 1.00e+00 1.00e+00f 1 MaxS\n", - " 15 1.4701415e+04 0.00e+00 1.25e+01 -11.0 1.08e-01 -4.5 1.00e+00 1.00e+00f 1 MaxS\n", - " 20 1.4700955e+04 0.00e+00 6.34e-01 -11.0 1.66e-04 -6.9 1.00e+00 1.00e+00f 1 MaxS\n", - " 25 1.4700954e+04 0.00e+00 2.63e-02 -11.0 7.01e-06 -9.2 1.00e+00 1.00e+00f 1 MaxS\n", - " 30 1.4700954e+04 0.00e+00 1.09e-03 -11.0 2.90e-07 -11.6 1.00e+00 1.00e+00f 1 MaxS\n", - " 35 1.4700954e+04 0.00e+00 4.49e-05 -11.0 1.20e-08 -14.0 1.00e+00 1.00e+00f 1 MaxS\n", - " 40 1.4700954e+04 0.00e+00 1.86e-06 -11.0 4.96e-10 -16.4 1.00e+00 1.00e+00h 1 MaxSA\n", - " 45 1.4700954e+04 0.00e+00 7.67e-08 -11.0 2.05e-11 -18.8 1.00e+00 1.00e+00h 1 MaxSA\n", - "iter objective inf_pr inf_du lg(mu) ||d|| lg(rg) alpha_du alpha_pr ls\n", - "\n", - "Number of Iterations....: 49\n", - "\n", - " (scaled) (unscaled)\n", - "Objective...............: 4.4662368766169078e+02 1.4700954216526392e+04\n", - "Dual infeasibility......: 5.8683114125458765e-09 1.9315987885869595e-07\n", - "Constraint violation....: 0.0000000000000000e+00 0.0000000000000000e+00\n", - "Complementarity.........: 0.0000000000000000e+00 0.0000000000000000e+00\n", - "Overall NLP error.......: 5.8683114125458765e-09 1.9315987885869595e-07\n", - "\n", - "\n", - "Number of objective function evaluations = 50\n", - "Number of objective gradient evaluations = 50\n", - "Number of equality constraint evaluations = 0\n", - "Number of inequality constraint evaluations = 0\n", - "Number of equality constraint Jacobian evaluations = 0\n", - "Number of inequality constraint Jacobian evaluations = 0\n", - "Number of Lagrangian Hessian evaluations = 49\n", - "Total CPU secs in IPOPT (w/o function evaluations) = 0.022\n", - "Total CPU secs in NLP function evaluations = 0.059\n", - "\n", - "EXIT: Optimal Solution Found.\n", - " 0.098447 seconds (46.49 k allocations: 5.053 MiB)\n" - ] - }, - { - "data": { - "text/plain": [ - "VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}(Array{Float64}(0,3), ([0.28137 0.280153 0.232512; 0.280153 0.284916 0.220363; 0.232512 0.220363 0.212921], [0.717042 0.661484 0.674207; 0.661484 0.714964 0.581648; 0.674207 0.581648 0.784373]), Array{Float64}(0,0), Char[], Float64[], -Inf, Inf)" - ] - }, - "execution_count": 88, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "traitidx = 5:7\n", - "# form data set\n", - "trait57_data = TwoVarCompVariateRotate(cg10kdata_rotated.Yrot[:, traitidx], cg10kdata_rotated.Xrot, \n", - " cg10kdata_rotated.eigval, cg10kdata_rotated.eigvec, cg10kdata_rotated.logdetV2)\n", - "# initialize model parameters\n", - "trait57_model = VarianceComponentModel(trait57_data)\n", - "# estimate variance components\n", - "@time mle_fs!(trait57_model, trait57_data; solver=:Ipopt, verbose=true)\n", - "trait57_model" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We then run the MM algorithm, starting from the Fisher scoring answer. MM finds an improved solution with objective value 8.955397e+03." - ] - }, - { - "cell_type": "code", - "execution_count": 89, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - " MM Algorithm\n", - " Iter Objective \n", - "-------- -------------\n", - " 0 -1.470095e+04\n", - " 1 -1.470095e+04\n", - "\n", - " 0.548238 seconds (1.04 M allocations: 51.897 MiB, 5.51% gc time)\n" - ] - }, - { - "data": { - "text/plain": [ - "VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}(Array{Float64}(0,3), ([0.28137 0.280153 0.232512; 0.280153 0.284916 0.220363; 0.232512 0.220363 0.212921], [0.717042 0.661484 0.674207; 0.661484 0.714964 0.581648; 0.674207 0.581648 0.784373]), Array{Float64}(0,0), Char[], Float64[], -Inf, Inf)" - ] - }, - "execution_count": 89, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# trait59_model contains the fitted model by Fisher scoring now\n", - "@time mle_mm!(trait57_model, trait57_data; verbose=true)\n", - "trait57_model" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Do another run of MM algorithm from default starting point. It leads to a slightly better local optimum -1.470104e+04, slighly worse than the Fisher scoring result. Follow up anlaysis should use the Fisher scoring result." - ] - }, - { - "cell_type": "code", - "execution_count": 90, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - " MM Algorithm\n", - " Iter Objective \n", - "-------- -------------\n", - " 0 -3.024417e+04\n", - " 1 -2.040172e+04\n", - " 2 -1.656053e+04\n", - " 3 -1.528554e+04\n", - " 4 -1.491023e+04\n", - " 5 -1.480677e+04\n", - " 6 -1.477849e+04\n", - " 7 -1.477005e+04\n", - " 8 -1.476675e+04\n", - " 9 -1.476478e+04\n", - " 10 -1.476318e+04\n", - " 20 -1.475020e+04\n", - " 30 -1.474023e+04\n", - " 40 -1.473255e+04\n", - " 50 -1.472662e+04\n", - " 60 -1.472201e+04\n", - " 70 -1.471839e+04\n", - " 80 -1.471554e+04\n", - " 90 -1.471326e+04\n", - " 100 -1.471142e+04\n", - " 110 -1.470993e+04\n", - " 120 -1.470871e+04\n", - " 130 -1.470770e+04\n", - " 140 -1.470686e+04\n", - " 150 -1.470616e+04\n", - " 160 -1.470557e+04\n", - " 170 -1.470506e+04\n", - " 180 -1.470463e+04\n", - " 190 -1.470426e+04\n", - " 200 -1.470394e+04\n", - " 210 -1.470366e+04\n", - " 220 -1.470342e+04\n", - " 230 -1.470321e+04\n", - " 240 -1.470302e+04\n", - " 250 -1.470285e+04\n", - " 260 -1.470270e+04\n", - " 270 -1.470257e+04\n", - " 280 -1.470245e+04\n", - " 290 -1.470234e+04\n", - " 300 -1.470225e+04\n", - " 310 -1.470216e+04\n", - " 320 -1.470208e+04\n", - " 330 -1.470201e+04\n", - " 340 -1.470195e+04\n", - " 350 -1.470189e+04\n", - " 360 -1.470183e+04\n", - " 370 -1.470178e+04\n", - " 380 -1.470173e+04\n", - " 390 -1.470169e+04\n", - " 400 -1.470165e+04\n", - " 410 -1.470162e+04\n", - " 420 -1.470158e+04\n", - " 430 -1.470155e+04\n", - " 440 -1.470152e+04\n", - " 450 -1.470149e+04\n", - " 460 -1.470147e+04\n", - " 470 -1.470144e+04\n", - " 480 -1.470142e+04\n", - " 490 -1.470140e+04\n", - " 500 -1.470138e+04\n", - " 510 -1.470136e+04\n", - " 520 -1.470134e+04\n", - " 530 -1.470132e+04\n", - " 540 -1.470131e+04\n", - " 550 -1.470129e+04\n", - " 560 -1.470128e+04\n", - " 570 -1.470127e+04\n", - " 580 -1.470125e+04\n", - " 590 -1.470124e+04\n", - " 600 -1.470123e+04\n", - " 610 -1.470122e+04\n", - " 620 -1.470121e+04\n", - " 630 -1.470120e+04\n", - " 640 -1.470119e+04\n", - " 650 -1.470118e+04\n", - " 660 -1.470117e+04\n", - " 670 -1.470116e+04\n", - " 680 -1.470116e+04\n", - " 690 -1.470115e+04\n", - " 700 -1.470114e+04\n", - " 710 -1.470113e+04\n", - " 720 -1.470113e+04\n", - " 730 -1.470112e+04\n", - " 740 -1.470112e+04\n", - " 750 -1.470111e+04\n", - " 760 -1.470110e+04\n", - " 770 -1.470110e+04\n", - " 780 -1.470109e+04\n", - " 790 -1.470109e+04\n", - " 800 -1.470108e+04\n", - " 810 -1.470108e+04\n", - " 820 -1.470108e+04\n", - " 830 -1.470107e+04\n", - " 840 -1.470107e+04\n", - " 850 -1.470106e+04\n", - " 860 -1.470106e+04\n", - " 870 -1.470106e+04\n", - " 880 -1.470105e+04\n", - " 890 -1.470105e+04\n", - " 900 -1.470105e+04\n", - " 910 -1.470104e+04\n", - " 920 -1.470104e+04\n", - " 930 -1.470104e+04\n", - " 940 -1.470103e+04\n", - " 950 -1.470103e+04\n", - " 960 -1.470103e+04\n", - " 970 -1.470103e+04\n", - " 980 -1.470102e+04\n", - " 990 -1.470102e+04\n", - " 1000 -1.470102e+04\n", - " 1010 -1.470102e+04\n", - " 1020 -1.470102e+04\n", - " 1030 -1.470101e+04\n", - " 1040 -1.470101e+04\n", - " 1050 -1.470101e+04\n", - " 1060 -1.470101e+04\n", - " 1070 -1.470101e+04\n", - " 1080 -1.470101e+04\n", - " 1090 -1.470100e+04\n", - " 1100 -1.470100e+04\n", - " 1110 -1.470100e+04\n", - "\n", - " 0.794444 seconds (135.15 k allocations: 13.553 MiB, 1.62% gc time)\n" - ] - }, - { - "data": { - "text/plain": [ - "VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}(Array{Float64}(0,3), ([0.281394 0.280169 0.232564; 0.280169 0.285001 0.220511; 0.232564 0.220511 0.213147], [0.717018 0.661467 0.674155; 0.661467 0.714877 0.581502; 0.674155 0.581502 0.784149]), Array{Float64}(0,0), Char[], Float64[], -Inf, Inf)" - ] - }, - "execution_count": 90, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# default starting point\n", - "trait57_model = VarianceComponentModel(trait57_data)\n", - "@time _, _, _, Σcov, = mle_mm!(trait57_model, trait57_data; verbose=true)\n", - "trait57_model" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Heritability from 3-variate estimate and their standard errors." - ] - }, - { - "cell_type": "code", - "execution_count": 91, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "2×3 Array{Float64,2}:\n", - " 0.281842 0.285036 0.213725 \n", - " 0.0777056 0.0772501 0.0840114" - ] - }, - "execution_count": 91, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "h, hse = heritability(trait57_model.Σ, Σcov)\n", - "[h'; hse']" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 13-trait joint analysis\n", - "\n", - "In some situations, such as studying the genetic covariance, we need to jointly analyze 13 traits. We first try the **Fisher scoring algorithm**." - ] - }, - { - "cell_type": "code", - "execution_count": 92, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "This is Ipopt version 3.12.10, running with linear solver mumps.\n", - "NOTE: Other linear solvers might be more efficient (see Ipopt documentation).\n", - "\n", - "Number of nonzeros in equality constraint Jacobian...: 0\n", - "Number of nonzeros in inequality constraint Jacobian.: 0\n", - "Number of nonzeros in Lagrangian Hessian.............: 16653\n", - "\n", - "Total number of variables............................: 182\n", - " variables with only lower bounds: 0\n", - " variables with lower and upper bounds: 0\n", - " variables with only upper bounds: 0\n", - "Total number of equality constraints.................: 0\n", - "Total number of inequality constraints...............: 0\n", - " inequality constraints with only lower bounds: 0\n", - " inequality constraints with lower and upper bounds: 0\n", - " inequality constraints with only upper bounds: 0\n", - "\n", - "iter objective inf_pr inf_du lg(mu) ||d|| lg(rg) alpha_du alpha_pr ls\n", - " 0 1.3111983e+05 0.00e+00 1.00e+02 0.0 0.00e+00 - 0.00e+00 0.00e+00 0 \n", - " 5 8.2228529e+04 0.00e+00 6.03e+02 -11.0 2.42e+00 - 1.00e+00 1.00e+00f 1 MaxS\n", - " 10 1.2570490e+05 0.00e+00 9.38e+02 -11.0 6.72e+01 -5.4 1.00e+00 1.00e+00h 1 MaxS\n" - ] - }, - { - "ename": "PosDefException", - "evalue": "PosDefException: matrix is not positive definite; Cholesky factorization failed.", - "output_type": "error", - "traceback": [ - "PosDefException: matrix is not positive definite; Cholesky factorization failed.", - "", - "Stacktrace:", - " [1] chkposdef at /Users/osx/buildbot/slave/package_osx64/build/usr/share/julia/stdlib/v0.7/LinearAlgebra/src/lapack.jl:50 [inlined]", - " [2] sygvd!(::Int64, ::Char, ::Char, ::Array{Float64,2}, ::Array{Float64,2}) at /Users/osx/buildbot/slave/package_osx64/build/usr/share/julia/stdlib/v0.7/LinearAlgebra/src/lapack.jl:5075", - " [3] eigen! at /Users/osx/buildbot/slave/package_osx64/build/usr/share/julia/stdlib/v0.7/LinearAlgebra/src/symmetric.jl:638 [inlined]", - " [4] eigen(::Symmetric{Float64,Array{Float64,2}}, ::Symmetric{Float64,Array{Float64,2}}) at /Users/osx/buildbot/slave/package_osx64/build/usr/share/julia/stdlib/v0.7/LinearAlgebra/src/eigen.jl:383", - " [5] TwoVarCompModelRotate(::VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}) at /Users/juhyun-kim/.julia/packages/VarianceComponentModels/Mh6LK/src/VarianceComponentModels.jl:119", - " [6] logpdf at /Users/juhyun-kim/.julia/packages/VarianceComponentModels/Mh6LK/src/two_variance_component.jl:62 [inlined]", - " [7] eval_f(::VarianceComponentModels.TwoVarCompOptProb{VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}},TwoVarCompVariateRotate{Float64,Array{Float64,2},Array{Float64,2}},Array{Float64,2},Array{Float64,1},VarianceComponentAuxData{Array{Float64,2},Array{Float64,1}}}, ::Array{Float64,1}) at /Users/juhyun-kim/.julia/packages/VarianceComponentModels/Mh6LK/src/two_variance_component.jl:731", - " [8] (::getfield(Ipopt, Symbol(\"#eval_f_cb#6\")){VarianceComponentModels.TwoVarCompOptProb{VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}},TwoVarCompVariateRotate{Float64,Array{Float64,2},Array{Float64,2}},Array{Float64,2},Array{Float64,1},VarianceComponentAuxData{Array{Float64,2},Array{Float64,1}}}})(::Array{Float64,1}) at /Users/juhyun-kim/.julia/packages/Ipopt/vosfb/src/MPB_wrapper.jl:64", - " [9] eval_f_wrapper(::Int32, ::Ptr{Float64}, ::Int32, ::Ptr{Float64}, ::Ptr{Nothing}) at /Users/juhyun-kim/.julia/packages/Ipopt/vosfb/src/Ipopt.jl:128", - " [10] solveProblem(::Ipopt.IpoptProblem) at /Users/juhyun-kim/.julia/packages/Ipopt/vosfb/src/Ipopt.jl:346", - " [11] optimize!(::Ipopt.IpoptMathProgModel) at /Users/juhyun-kim/.julia/packages/Ipopt/vosfb/src/MPB_wrapper.jl:141", - " [12] macro expansion at ./simdloop.jl:67 [inlined]", - " [13] #mle_fs!#27(::Int64, ::Symbol, ::Symbol, ::Bool, ::Function, ::VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}, ::TwoVarCompVariateRotate{Float64,Array{Float64,2},Array{Float64,2}}) at /Users/juhyun-kim/.julia/packages/VarianceComponentModels/Mh6LK/src/two_variance_component.jl:942", - " [14] (::getfield(VarianceComponentModels, Symbol(\"#kw##mle_fs!\")))(::NamedTuple{(:solver, :verbose),Tuple{Symbol,Bool}}, ::typeof(mle_fs!), ::VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}, ::TwoVarCompVariateRotate{Float64,Array{Float64,2},Array{Float64,2}}) at ./none:0", - " [15] top-level scope at util.jl:156", - " [16] top-level scope at In[92]:3" - ] - } - ], - "source": [ - "# initialize model parameters\n", - "traitall_model = VarianceComponentModel(cg10kdata_rotated)\n", - "# estimate variance components using Fisher scoring algorithm\n", - "@time mle_fs!(traitall_model, cg10kdata_rotated; solver=:Ipopt, verbose=true)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "From the output we can see the Fisher scoring algorithm ran into some numerical issues. Let's try the **MM algorithm**." - ] - }, - { - "cell_type": "code", - "execution_count": 93, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - " MM Algorithm\n", - " Iter Objective \n", - "-------- -------------\n", - " 0 -1.311198e+05\n", - " 1 -8.001595e+04\n", - " 2 -5.806836e+04\n", - " 3 -4.926167e+04\n", - " 4 -4.611163e+04\n", - " 5 -4.511722e+04\n", - " 6 -4.482795e+04\n", - " 7 -4.474405e+04\n", - " 8 -4.471603e+04\n", - " 9 -4.470277e+04\n", - " 10 -4.469345e+04\n", - " 20 -4.462308e+04\n", - " 30 -4.456930e+04\n", - " 40 -4.452802e+04\n", - " 50 -4.449620e+04\n", - " 60 -4.447148e+04\n", - " 70 -4.445209e+04\n", - " 80 -4.443672e+04\n", - " 90 -4.442442e+04\n", - " 100 -4.441448e+04\n", - " 110 -4.440635e+04\n", - " 120 -4.439966e+04\n", - " 130 -4.439409e+04\n", - " 140 -4.438943e+04\n", - " 150 -4.438549e+04\n", - " 160 -4.438215e+04\n", - " 170 -4.437928e+04\n", - " 180 -4.437682e+04\n", - " 190 -4.437469e+04\n", - " 200 -4.437283e+04\n", - " 210 -4.437121e+04\n", - " 220 -4.436978e+04\n", - " 230 -4.436853e+04\n", - " 240 -4.436742e+04\n", - " 250 -4.436643e+04\n", - " 260 -4.436555e+04\n", - " 270 -4.436476e+04\n", - " 280 -4.436406e+04\n", - " 290 -4.436342e+04\n", - " 300 -4.436285e+04\n", - " 310 -4.436234e+04\n", - " 320 -4.436187e+04\n", - " 330 -4.436144e+04\n", - " 340 -4.436106e+04\n", - " 350 -4.436071e+04\n", - " 360 -4.436039e+04\n", - " 370 -4.436009e+04\n", - " 380 -4.435982e+04\n", - " 390 -4.435957e+04\n", - " 400 -4.435935e+04\n", - " 410 -4.435914e+04\n", - " 420 -4.435895e+04\n", - " 430 -4.435877e+04\n", - " 440 -4.435860e+04\n", - " 450 -4.435845e+04\n", - " 460 -4.435831e+04\n", - " 470 -4.435818e+04\n", - " 480 -4.435806e+04\n", - " 490 -4.435794e+04\n", - " 500 -4.435784e+04\n", - " 510 -4.435774e+04\n", - " 520 -4.435765e+04\n", - " 530 -4.435757e+04\n", - " 540 -4.435749e+04\n", - " 550 -4.435741e+04\n", - " 560 -4.435734e+04\n", - " 570 -4.435728e+04\n", - " 580 -4.435722e+04\n", - " 590 -4.435716e+04\n", - " 600 -4.435711e+04\n", - " 610 -4.435706e+04\n", - " 620 -4.435701e+04\n", - " 630 -4.435696e+04\n", - " 640 -4.435692e+04\n", - " 650 -4.435688e+04\n", - " 660 -4.435685e+04\n", - " 670 -4.435681e+04\n", - " 680 -4.435678e+04\n", - " 690 -4.435675e+04\n", - " 700 -4.435672e+04\n", - " 710 -4.435669e+04\n", - " 720 -4.435666e+04\n", - " 730 -4.435664e+04\n", - " 740 -4.435662e+04\n", - " 750 -4.435659e+04\n", - " 760 -4.435657e+04\n", - " 770 -4.435655e+04\n", - " 780 -4.435653e+04\n", - " 790 -4.435652e+04\n", - " 800 -4.435650e+04\n", - " 810 -4.435648e+04\n", - " 820 -4.435647e+04\n", - " 830 -4.435645e+04\n", - " 840 -4.435644e+04\n", - " 850 -4.435643e+04\n", - " 860 -4.435641e+04\n", - " 870 -4.435640e+04\n", - " 880 -4.435639e+04\n", - " 890 -4.435638e+04\n", - " 900 -4.435637e+04\n", - " 910 -4.435636e+04\n", - " 920 -4.435635e+04\n", - " 930 -4.435634e+04\n", - " 940 -4.435633e+04\n", - " 950 -4.435633e+04\n", - " 960 -4.435632e+04\n", - " 970 -4.435631e+04\n", - " 980 -4.435630e+04\n", - " 990 -4.435630e+04\n", - " 1000 -4.435629e+04\n", - " 1010 -4.435628e+04\n", - " 1020 -4.435628e+04\n", - " 1030 -4.435627e+04\n", - " 1040 -4.435627e+04\n", - " 1050 -4.435626e+04\n", - " 1060 -4.435626e+04\n", - " 1070 -4.435625e+04\n", - " 1080 -4.435625e+04\n", - "\n", - " 3.423491 seconds (131.35 k allocations: 65.559 MiB, 0.86% gc time)\n" - ] - }, - { - "data": { - "text/plain": [ - "(-44356.24416253091, VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}(Array{Float64}(0,13), ([0.272112 0.190023 … -0.128464 -0.0976418; 0.190023 0.216814 … -0.0687833 -0.04341; … ; -0.128464 -0.0687833 … 0.116994 0.0900933; -0.0976418 -0.04341 … 0.0900933 0.105876], [0.725183 0.570497 … -0.0589748 -0.125486; 0.570497 0.783023 … 0.0235685 0.0464638; … ; -0.0589748 0.0235685 … 0.882056 0.551829; -0.125486 0.0464638 … 0.551829 0.893642]), Array{Float64}(0,0), Char[], Float64[], -Inf, Inf), ([0.0111615 0.0131072 … 0.0128913 0.0127584; 0.0131071 0.0151656 … 0.017153 0.0171365; … ; 0.0128911 0.017153 … 0.0174174 0.018212; 0.0127593 0.0171355 … 0.0182119 0.0188], [0.0112229 0.0133031 … 0.013004 0.0127777; 0.0133033 0.015806 … 0.0178516 0.0177826; … ; 0.0130042 0.0178517 … 0.0179556 0.0187634; 0.0127776 0.0177822 … 0.0187633 0.0193475]), [0.00012458 7.242e-5 … -3.70019e-7 -1.40553e-5; 7.23929e-5 0.000171795 … -2.04581e-5 -3.19854e-6; … ; -3.8356e-7 -2.04665e-5 … 0.000352066 -1.46068e-5; -1.40476e-5 -3.19013e-6 … -1.46022e-5 0.000374325], Array{Float64}(0,13), Array{Float64}(0,0))" - ] - }, - "execution_count": 93, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# reset model parameters\n", - "traitall_model = VarianceComponentModel(cg10kdata_rotated)\n", - "# estimate variance components using Fisher scoring algorithm\n", - "@time mle_mm!(traitall_model, cg10kdata_rotated; verbose=true)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "It converges after ~1000 iterations." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Save analysis results" - ] - }, - { - "cell_type": "code", - "execution_count": 95, - "metadata": {}, - "outputs": [ - { - "data": { - "text/latex": [ - "\\begin{tabular}\n", - "{l | r | l}\n", - "name & size & summary \\\\\n", - "\\hline\n", - "Base & & Module \\\\\n", - "Core & & Module \\\\\n", - "Main & & Module \\\\\n", - "Plots & 22.448 MiB & Module \\\\\n", - "Y & 677.461 KiB & 6670×13 Array\\{Float64,2\\} \\\\\n", - "\\_ & 240 bytes & Tuple\\{Array\\{Float64,2\\},Array\\{Float64,2\\}\\} \\\\\n", - "cg10k & 1022.983 MiB & 6670×630860 SnpArray \\\\\n", - "cg10k\\_trait & 1.443 MiB & 6670×15 DataFrame \\\\\n", - "cg10kdata & 679.508 MiB & VarianceComponentVariate\\{Float64,2,Array\\{Float64,2\\},Array\\{Float64,2\\},Array\\{Float64,2\\}\\} \\\\\n", - "cg10kdata\\_rotated & 340.136 MiB & TwoVarCompVariateRotate\\{Float64,Array\\{Float64,2\\},Array\\{Float64,2\\}\\} \\\\\n", - "h & 64 bytes & 3-element Array\\{Float64,1\\} \\\\\n", - "hST & 144 bytes & 13-element Array\\{Float64,1\\} \\\\\n", - "hST\\_se & 144 bytes & 13-element Array\\{Float64,1\\} \\\\\n", - "hse & 64 bytes & 3-element Array\\{Float64,1\\} \\\\\n", - "maf\\_cg10k & 4.813 MiB & 630860-element Array\\{Float64,1\\} \\\\\n", - "missings\\_by\\_snp & 4.813 MiB & 1×630860 Array\\{Int64,2\\} \\\\\n", - "mp & 28.748 MiB & 6670×630860 SparseMatrixCSC\\{Bool,Int32\\} \\\\\n", - "people & 8 bytes & Int64 \\\\\n", - "snps & 8 bytes & Int64 \\\\\n", - "startupfile & 57 bytes & String \\\\\n", - "trait57\\_data & 339.627 MiB & TwoVarCompVariateRotate\\{Float64,Array\\{Float64,2\\},Array\\{Float64,2\\}\\} \\\\\n", - "trait57\\_model & 472 bytes & VarianceComponentModel\\{Float64,2,Array\\{Float64,2\\},Array\\{Float64,2\\}\\} \\\\\n", - "traitall\\_model & 2.961 KiB & VarianceComponentModel\\{Float64,2,Array\\{Float64,2\\},Array\\{Float64,2\\}\\} \\\\\n", - "traitidx & 16 bytes & 3-element UnitRange\\{Int64\\} \\\\\n", - "Σa & 6.844 KiB & 13×13 Array\\{Array\\{Float64,2\\},2\\} \\\\\n", - "Σcov & 2.570 KiB & 18×18 Array\\{Float64,2\\} \\\\\n", - "Σe & 6.844 KiB & 13×13 Array\\{Array\\{Float64,2\\},2\\} \\\\\n", - "Φgrm & 339.423 MiB & 6670×6670 Array\\{Float64,2\\} \\\\\n", - "σ2a & 144 bytes & 13-element Array\\{Float64,1\\} \\\\\n", - "σ2e & 144 bytes & 13-element Array\\{Float64,1\\} \\\\\n", - "\\end{tabular}\n" - ], - "text/markdown": [ - "| name | size | summary |\n", - "|:----------------- | ------------:|:-------------------------------------------------------------------------------------- |\n", - "| Base | | Module |\n", - "| Core | | Module |\n", - "| Main | | Module |\n", - "| Plots | 22.448 MiB | Module |\n", - "| Y | 677.461 KiB | 6670×13 Array{Float64,2} |\n", - "| _ | 240 bytes | Tuple{Array{Float64,2},Array{Float64,2}} |\n", - "| cg10k | 1022.983 MiB | 6670×630860 SnpArray |\n", - "| cg10k_trait | 1.443 MiB | 6670×15 DataFrame |\n", - "| cg10kdata | 679.508 MiB | VarianceComponentVariate{Float64,2,Array{Float64,2},Array{Float64,2},Array{Float64,2}} |\n", - "| cg10kdata_rotated | 340.136 MiB | TwoVarCompVariateRotate{Float64,Array{Float64,2},Array{Float64,2}} |\n", - "| h | 64 bytes | 3-element Array{Float64,1} |\n", - "| hST | 144 bytes | 13-element Array{Float64,1} |\n", - "| hST_se | 144 bytes | 13-element Array{Float64,1} |\n", - "| hse | 64 bytes | 3-element Array{Float64,1} |\n", - "| maf_cg10k | 4.813 MiB | 630860-element Array{Float64,1} |\n", - "| missings_by_snp | 4.813 MiB | 1×630860 Array{Int64,2} |\n", - "| mp | 28.748 MiB | 6670×630860 SparseMatrixCSC{Bool,Int32} |\n", - "| people | 8 bytes | Int64 |\n", - "| snps | 8 bytes | Int64 |\n", - "| startupfile | 57 bytes | String |\n", - "| trait57_data | 339.627 MiB | TwoVarCompVariateRotate{Float64,Array{Float64,2},Array{Float64,2}} |\n", - "| trait57_model | 472 bytes | VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}} |\n", - "| traitall_model | 2.961 KiB | VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}} |\n", - "| traitidx | 16 bytes | 3-element UnitRange{Int64} |\n", - "| Σa | 6.844 KiB | 13×13 Array{Array{Float64,2},2} |\n", - "| Σcov | 2.570 KiB | 18×18 Array{Float64,2} |\n", - "| Σe | 6.844 KiB | 13×13 Array{Array{Float64,2},2} |\n", - "| Φgrm | 339.423 MiB | 6670×6670 Array{Float64,2} |\n", - "| σ2a | 144 bytes | 13-element Array{Float64,1} |\n", - "| σ2e | 144 bytes | 13-element Array{Float64,1} |\n" - ], - "text/plain": [ - "name size summary \n", - "––––––––––––––––– –––––––––––– ––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––\n", - "Base Module \n", - "Core Module \n", - "Main Module \n", - "Plots 22.448 MiB Module \n", - "Y 677.461 KiB 6670×13 Array{Float64,2} \n", - "_ 240 bytes Tuple{Array{Float64,2},Array{Float64,2}} \n", - "cg10k 1022.983 MiB 6670×630860 SnpArray \n", - "cg10k_trait 1.443 MiB 6670×15 DataFrame \n", - "cg10kdata 679.508 MiB VarianceComponentVariate{Float64,2,Array{Float64,2},Array{Float64,2},Array{Float64,2}}\n", - "cg10kdata_rotated 340.136 MiB TwoVarCompVariateRotate{Float64,Array{Float64,2},Array{Float64,2}} \n", - "h 64 bytes 3-element Array{Float64,1} \n", - "hST 144 bytes 13-element Array{Float64,1} \n", - "hST_se 144 bytes 13-element Array{Float64,1} \n", - "hse 64 bytes 3-element Array{Float64,1} \n", - "maf_cg10k 4.813 MiB 630860-element Array{Float64,1} \n", - "missings_by_snp 4.813 MiB 1×630860 Array{Int64,2} \n", - "mp 28.748 MiB 6670×630860 SparseMatrixCSC{Bool,Int32} \n", - "people 8 bytes Int64 \n", - "snps 8 bytes Int64 \n", - "startupfile 57 bytes String \n", - "trait57_data 339.627 MiB TwoVarCompVariateRotate{Float64,Array{Float64,2},Array{Float64,2}} \n", - "trait57_model 472 bytes VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}} \n", - "traitall_model 2.961 KiB VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}} \n", - "traitidx 16 bytes 3-element UnitRange{Int64} \n", - "Σa 6.844 KiB 13×13 Array{Array{Float64,2},2} \n", - "Σcov 2.570 KiB 18×18 Array{Float64,2} \n", - "Σe 6.844 KiB 13×13 Array{Array{Float64,2},2} \n", - "Φgrm 339.423 MiB 6670×6670 Array{Float64,2} \n", - "σ2a 144 bytes 13-element Array{Float64,1} \n", - "σ2e 144 bytes 13-element Array{Float64,1} " - ] - }, - "execution_count": 95, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "#using JLD2, FileIO\n", - "#save(\"copd.jld2\")\n", - "#varinfo()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Julia 0.7.0", - "language": "julia", - "name": "julia-0.7" - }, - "language_info": { - "file_extension": ".jl", - "mimetype": "application/julia", - "name": "julia", - "version": "0.7.0" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/docs/mle_reml.ipynb b/docs/mle_reml.ipynb deleted file mode 100644 index 32b500f..0000000 --- a/docs/mle_reml.ipynb +++ /dev/null @@ -1,1312 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# MLE and REML" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Machine information" - ] - }, - { - "cell_type": "code", - "execution_count": 54, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Julia Version 0.7.0\n", - "Commit a4cb80f3ed (2018-08-08 06:46 UTC)\n", - "Platform Info:\n", - " OS: macOS (x86_64-apple-darwin14.5.0)\n", - " CPU: Intel(R) Core(TM) i5-6267U CPU @ 2.90GHz\n", - " WORD_SIZE: 64\n", - " LIBM: libopenlibm\n", - " LLVM: libLLVM-6.0.0 (ORCJIT, skylake)\n" - ] - } - ], - "source": [ - "versioninfo()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Demo data" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "For demonstration, we generate a random data set." - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "# generate data from a d-variate response variane component model\n", - "using Random, LinearAlgebra\n", - "Random.seed!(123)\n", - "n = 1000 # no. observations\n", - "d = 2 # dimension of responses\n", - "m = 2 # no. variance components\n", - "p = 2 # no. covariates\n", - "# n-by-p design matrix\n", - "X = randn(n, p)\n", - "# p-by-d mean component regression coefficient\n", - "B = ones(p, d) \n", - "# a tuple of m covariance matrices\n", - "V = ntuple(x -> zeros(n, n), m) \n", - "for i = 1:m-1\n", - " Vi = randn(n, 50)\n", - " copyto!(V[i], Vi * Vi')\n", - "end\n", - "copyto!(V[m], Matrix(I, n, n)) # last covarianec matrix is idendity\n", - "# a tuple of m d-by-d variance component parameters\n", - "Σ = ntuple(x -> zeros(d, d), m) \n", - "for i in 1:m\n", - " Σi = randn(d, d)\n", - " copyto!(Σ[i], Σi' * Σi)\n", - "end\n", - "# form overall nd-by-nd covariance matrix Ω\n", - "Ω = zeros(n * d, n * d)\n", - "for i = 1:m\n", - " Ω += kron(Σ[i], V[i])\n", - "end\n", - "Ωchol = cholesky(Ω)\n", - "# n-by-d responses\n", - "Y = X * B + reshape(Ωchol.L * randn(n*d), n, d);" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Maximum likelihood estimation (MLE)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "To find the MLE of parameters $(B,\\Sigma_1,\\ldots,\\Sigma_m)$, we take 3 steps: \n", - "\n", - "**Step 1 (Construct data)**. Construct an instance of `VarianceComponentVariate`, which consists fields \n", - "\n", - "* `Y`: $n$-by-$d$ responses \n", - "* `X`: $n$-by-$p$ covariate matrix \n", - "* `V=(V[1],...,V[m])`: a tuple of $n$-by-$n$ covariance matrices. The last covariance matrix must be positive definite and usually is the identity matrix. " - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(:Y, :X, :V)" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "using VarianceComponentModels\n", - "vcdata = VarianceComponentVariate(Y, X, V)\n", - "fieldnames(typeof(vcdata))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "In the absence of covariates $X$, we can simply initialize by `vcdata = VarianceComponentVariate(Y, V)`." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "**Step 2 (Construct a model)**. Construct an instance of `VarianceComponentModel`, which consists of fields \n", - "\n", - "* `B`: $n$-by-$p$ mean regression coefficients \n", - "* `Σ=(Σ[1],...,Σ[m])`: variane component parameters respectively. \n", - "\n", - "When constructed from a `VarianceComponentVariate` instance, the mean parameters $B$ are initialized to be zero and the tuple of variance component parameters $\\Sigma$ to be `(eye(d),...,eye(d))`." - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(:B, :Σ, :A, :sense, :b, :lb, :ub)" - ] - }, - "execution_count": 16, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "vcmodel = VarianceComponentModel(vcdata)\n", - "fieldnames(typeof(vcmodel))" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}([0.0 0.0; 0.0 0.0], ([1.0 0.0; 0.0 1.0], [1.0 0.0; 0.0 1.0]), Array{Float64}(0,4), Char[], Float64[], -Inf, Inf)" - ] - }, - "execution_count": 17, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "vcmodel" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The remaining fields `A`, `sense`, `b`, `lb`, `ub` specify (optional) constraints on the mean parameters `B`:\n", - "\n", - "$A * \\text{vec}(B) \\,\\, =(\\text{or } \\ge \\text{or } \\le) \\,\\, b$\n", - "\n", - "$lb \\le \\text{vec}(B) \\le ub$\n", - "\n", - "`A` is an constraint matrix with $pd$ columns, `sense` is a vector of charaters taking values `'<'`, `'='` or `'>'`, and `lb` and `ub` are the lower and upper bounds for `vec(B)`. By default, `A`, `sense`, `b` are empty, `lb` is `-Inf`, and `ub` is `Inf`. If any constraits are non-trivial, final estimates of `B` are enforced to satisfy them." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "When a better initial guess is available, we can initialize by calling `vcmodel=VarianceComponentModel(B0, Σ0)` directly." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "**Step 3 (Fit model)**. Call optmization routine `fit_mle!`. The keywork `algo` dictates the optimization algorithm: `:MM` (minorization-maximization algorithm) or `:FS` (Fisher scoring algorithm)." - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - " MM Algorithm\n", - " Iter Objective \n", - "-------- -------------\n", - " 0 -6.253551e+03\n", - "\n", - "******************************************************************************\n", - "This program contains Ipopt, a library for large-scale nonlinear optimization.\n", - " Ipopt is released as open source code under the Eclipse Public License (EPL).\n", - " For more information visit http://projects.coin-or.org/Ipopt\n", - "******************************************************************************\n", - "\n", - " 1 -3.881454e+03\n", - " 2 -3.853179e+03\n", - " 3 -3.846525e+03\n", - " 4 -3.844906e+03\n", - " 5 -3.844506e+03\n", - " 6 -3.844406e+03\n", - " 7 -3.844381e+03\n", - " 8 -3.844375e+03\n", - " 9 -3.844374e+03\n", - " 10 -3.844373e+03\n", - "\n", - " 4.546981 seconds (11.23 M allocations: 566.109 MiB, 5.23% gc time)\n" - ] - } - ], - "source": [ - "vcmodel_mle = deepcopy(vcmodel)\n", - "@time logl, vcmodel_mle, Σse, Σcov, Bse, Bcov = fit_mle!(vcmodel_mle, vcdata; algo = :MM);" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The output of `fit_mle!` contains \n", - "\n", - "* final log-likelihood " - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "-3844.3731814180883" - ] - }, - "execution_count": 19, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "logl" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "* fitted model" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(:B, :Σ, :A, :sense, :b, :lb, :ub)" - ] - }, - "execution_count": 21, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "fieldnames(typeof(vcmodel_mle))" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}([1.092 1.04727; 0.955346 1.01632], ([0.380637 -0.305465; -0.305465 4.51938], [1.84009 0.265569; 0.265569 2.17275]), Array{Float64}(0,4), Char[], Float64[], -Inf, Inf)" - ] - }, - "execution_count": 22, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "vcmodel_mle" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "* standard errors of the estimated varianec component parameters" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "([0.0765136 0.263047; 0.263047 0.904332], [0.0844292 0.0917441; 0.0917441 0.0996927])" - ] - }, - "execution_count": 23, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "Σse" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "* covariance matrix of the variance component parameters estimates" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "8×8 Array{Float64,2}:\n", - " 0.00585433 -0.00467019 -0.00467019 … -1.07903e-6 -1.557e-7 \n", - " -0.00467019 0.0691937 0.00372555 -1.557e-7 -1.27444e-6 \n", - " -0.00467019 0.00372555 0.0691937 -8.83212e-6 -1.27444e-6 \n", - " 0.00372555 -0.055198 -0.055198 -1.27444e-6 -1.04316e-5 \n", - " -7.4779e-6 -1.07903e-6 -1.07903e-6 0.00102878 0.000148477\n", - " -1.07903e-6 -8.83212e-6 -1.557e-7 … 0.000148477 0.00121477 \n", - " -1.07903e-6 -1.557e-7 -8.83212e-6 0.00841698 0.00121477 \n", - " -1.557e-7 -1.27444e-6 -1.27444e-6 0.00121477 0.00993864 " - ] - }, - "execution_count": 24, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "Σcov" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "* standard errors of the estimated mean parameters" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "2×2 Array{Float64,2}:\n", - " 0.0425562 0.0483834\n", - " 0.0430596 0.0492809" - ] - }, - "execution_count": 25, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "Bse" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "* covariance matrix of the mean parameter estimates" - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "4×4 Array{Float64,2}:\n", - " 0.00181103 -1.96485e-5 0.000243441 -4.38252e-6 \n", - " -1.96485e-5 0.00185413 -4.38252e-6 0.000246407\n", - " 0.000243441 -4.38252e-6 0.00234096 -5.73331e-6 \n", - " -4.38252e-6 0.000246407 -5.73331e-6 0.00242861 " - ] - }, - "execution_count": 26, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "Bcov" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Restricted maximum likelihood estimation (REML)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "[REML (restricted maximum likelihood estimation)](https://en.wikipedia.org/wiki/Restricted_maximum_likelihood) is a popular alternative to the MLE. To find the REML of a variane component model, we replace the above step 3 by " - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "**Step 3**. Call optmization routine `fit_reml!`." - ] - }, - { - "cell_type": "code", - "execution_count": 28, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - " MM Algorithm\n", - " Iter Objective \n", - "-------- -------------\n", - " 0 -4.215053e+03\n", - " 1 -3.925799e+03\n", - " 2 -3.865114e+03\n", - " 3 -3.851105e+03\n", - " 4 -3.847732e+03\n", - " 5 -3.846903e+03\n", - " 6 -3.846698e+03\n", - " 7 -3.846647e+03\n", - " 8 -3.846634e+03\n", - " 9 -3.846631e+03\n", - " 10 -3.846630e+03\n", - "\n", - " 0.443964 seconds (8.09 k allocations: 62.532 MiB, 2.39% gc time)\n" - ] - } - ], - "source": [ - "vcmodel_reml = deepcopy(vcmodel)\n", - "@time logl, vcmodel_reml, Σse, Σcov, Bse, Bcov = fit_reml!(vcmodel_reml, vcdata; algo = :MM);" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The output of `fit_reml!` contains" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "* the final log-likelihood at REML estimate" - ] - }, - { - "cell_type": "code", - "execution_count": 29, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "-3844.3777179025046" - ] - }, - "execution_count": 29, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "logl" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "* REML estimates" - ] - }, - { - "cell_type": "code", - "execution_count": 31, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(:B, :Σ, :A, :sense, :b, :lb, :ub)" - ] - }, - "execution_count": 31, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "fieldnames(typeof(vcmodel_reml))" - ] - }, - { - "cell_type": "code", - "execution_count": 32, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}([1.092 1.04727; 0.955345 1.01632], ([0.380594 -0.305485; -0.305485 4.51994], [1.84285 0.261963; 0.261963 2.17842]), Array{Float64}(0,4), Char[], Float64[], -Inf, Inf)" - ] - }, - "execution_count": 32, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "vcmodel_reml" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "* standard errors of the estimated varianec component parameters" - ] - }, - { - "cell_type": "code", - "execution_count": 33, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "([0.0765055 0.26305; 0.26305 0.904446], [0.0845559 0.0919325; 0.0919325 0.0999526])" - ] - }, - "execution_count": 33, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "Σse" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "* covariance matrix of the variance component parameters estimates" - ] - }, - { - "cell_type": "code", - "execution_count": 34, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "8×8 Array{Float64,2}:\n", - " 0.0058531 -0.00467005 -0.00467005 … -1.06597e-6 -1.51499e-7 \n", - " -0.00467005 0.0691951 0.00372613 -1.51499e-7 -1.26041e-6 \n", - " -0.00467005 0.00372613 0.0691951 -8.86843e-6 -1.26041e-6 \n", - " 0.00372613 -0.0552092 -0.0552092 -1.26041e-6 -1.0486e-5 \n", - " -7.50035e-6 -1.06597e-6 -1.06597e-6 0.00101633 0.000144472\n", - " -1.06597e-6 -8.86843e-6 -1.51499e-7 … 0.000144472 0.0012014 \n", - " -1.06597e-6 -1.51499e-7 -8.86843e-6 0.00845158 0.0012014 \n", - " -1.51499e-7 -1.26041e-6 -1.26041e-6 0.0012014 0.00999052 " - ] - }, - "execution_count": 34, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "Σcov" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "* standard errors of the estimated mean parameters" - ] - }, - { - "cell_type": "code", - "execution_count": 35, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "2×2 Array{Float64,2}:\n", - " 0.0425881 0.0484485\n", - " 0.0430919 0.0493475" - ] - }, - "execution_count": 35, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "Bse" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "* covariance matrix of the mean parameter estimates" - ] - }, - { - "cell_type": "code", - "execution_count": 36, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "4×4 Array{Float64,2}:\n", - " 0.00181375 -1.96783e-5 0.000239868 -4.34611e-6 \n", - " -1.96783e-5 0.00185691 -4.34611e-6 0.000242745\n", - " 0.000239868 -4.34611e-6 0.00234726 -5.73082e-6 \n", - " -4.34611e-6 0.000242745 -5.73082e-6 0.00243518 " - ] - }, - "execution_count": 36, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "Bcov" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Optimization algorithms" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Finding the MLE or REML of variance component models is a non-trivial nonlinear optimization problem. The main complications are the non-convexity of objective function and the positive semi-definiteness constraint of variane component parameters $\\Sigma_1,\\ldots,\\Sigma_m$. In specific applications, users should try different algorithms with different starting points in order to find a better solution. Here are some tips for efficient computation. " - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "In general the optimization algorithm needs to invert the $nd$ by $nd$ overall covariance matrix $\\Omega = \\Sigma_1 \\otimes V_1 + \\cdots + \\Sigma_m \\otimes V_m$ in each iteration. Inverting a matrix is an expensive operation with $O(n^3 d^3)$ floating operations. When there are only **two** varianec components ($m=2$), this tedious task can be avoided by taking one (generalized) eigendecomposion of $(V_1, V_2)$ and rotating data $(Y, X)$ by the eigen-vectors. " - ] - }, - { - "cell_type": "code", - "execution_count": 38, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(:Yrot, :Xrot, :eigval, :eigvec, :logdetV2)" - ] - }, - "execution_count": 38, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "vcdatarot = TwoVarCompVariateRotate(vcdata)\n", - "fieldnames(typeof(vcdatarot))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Two optimization algorithms are implemented: [Fisher scoring](https://books.google.com/books?id=QYqeYTftPNwC&lpg=PP1&pg=PA142#v=onepage&q&f=false) (`mle_fs!`) and the [minorization-maximization (MM) algorithm](http://arxiv.org/abs/1509.07426) (`mle_mm!`). Both take the rotated data as input. These two functions give finer control of the optimization algorithms. Generally speaking, MM algorithm is more stable while Fisher scoring (if it converges) yields more accurate answer." - ] - }, - { - "cell_type": "code", - "execution_count": 39, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - " MM Algorithm\n", - " Iter Objective \n", - "-------- -------------\n", - " 0 -6.253551e+03\n", - " 1 -3.881454e+03\n", - " 2 -3.853179e+03\n", - " 3 -3.846525e+03\n", - " 4 -3.844906e+03\n", - " 5 -3.844506e+03\n", - " 6 -3.844406e+03\n", - " 7 -3.844381e+03\n", - " 8 -3.844375e+03\n", - " 9 -3.844374e+03\n", - " 10 -3.844373e+03\n", - "\n", - " 0.042187 seconds (21.56 k allocations: 1.366 MiB)\n" - ] - } - ], - "source": [ - "vcmodel_mm = deepcopy(vcmodel)\n", - "@time mle_mm!(vcmodel_mm, vcdatarot; maxiter=10000, funtol=1e-8, verbose = true);" - ] - }, - { - "cell_type": "code", - "execution_count": 40, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "2×2 Array{Float64,2}:\n", - " 1.092 1.04727\n", - " 0.955346 1.01632" - ] - }, - "execution_count": 40, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# MM estimates\n", - "vcmodel_mm.B" - ] - }, - { - "cell_type": "code", - "execution_count": 41, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "([0.380637 -0.305465; -0.305465 4.51938], [1.84009 0.265569; 0.265569 2.17275])" - ] - }, - "execution_count": 41, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# MM estimates\n", - "vcmodel_mm.Σ" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Fisher scoring (`mle_fs!`) uses either [Ipopt.jl](https://github.com/JuliaOpt/Ipopt.jl) (keyword `solver=:Ipopt`) or [KNITRO.jl](https://github.com/JuliaOpt/KNITRO.jl) (keyword `solver=:Knitro`) as the backend solver. Ipopt is open source and installation of [Ipopt.jl](https://github.com/JuliaOpt/Ipopt.jl) package alone is sufficient." - ] - }, - { - "cell_type": "code", - "execution_count": 42, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "This is Ipopt version 3.12.10, running with linear solver mumps.\n", - "NOTE: Other linear solvers might be more efficient (see Ipopt documentation).\n", - "\n", - "Number of nonzeros in equality constraint Jacobian...: 0\n", - "Number of nonzeros in inequality constraint Jacobian.: 0\n", - "Number of nonzeros in Lagrangian Hessian.............: 21\n", - "\n", - "Total number of variables............................: 6\n", - " variables with only lower bounds: 0\n", - " variables with lower and upper bounds: 0\n", - " variables with only upper bounds: 0\n", - "Total number of equality constraints.................: 0\n", - "Total number of inequality constraints...............: 0\n", - " inequality constraints with only lower bounds: 0\n", - " inequality constraints with lower and upper bounds: 0\n", - " inequality constraints with only upper bounds: 0\n", - "\n", - "iter objective inf_pr inf_du lg(mu) ||d|| lg(rg) alpha_du alpha_pr ls\n", - " 0 4.2109423e+03 0.00e+00 1.00e+02 0.0 0.00e+00 - 0.00e+00 0.00e+00 0 \n", - " 5 3.8445586e+03 0.00e+00 7.87e-01 -11.0 4.94e-02 - 1.00e+00 1.00e+00f 1 MaxS\n", - " 10 3.8443870e+03 0.00e+00 2.25e-01 -11.0 1.38e-02 - 1.00e+00 1.00e+00f 1 MaxS\n", - " 15 3.8443742e+03 0.00e+00 6.23e-02 -11.0 3.78e-03 - 1.00e+00 1.00e+00f 1 MaxS\n", - " 20 3.8443733e+03 0.00e+00 1.70e-02 -11.0 1.03e-03 - 1.00e+00 1.00e+00f 1 MaxS\n", - " 25 3.8443732e+03 0.00e+00 4.61e-03 -11.0 2.79e-04 - 1.00e+00 1.00e+00f 1 MaxS\n", - " 30 3.8443732e+03 0.00e+00 1.25e-03 -11.0 7.56e-05 - 1.00e+00 1.00e+00f 1 MaxS\n", - " 35 3.8443732e+03 0.00e+00 3.39e-04 -11.0 2.05e-05 - 1.00e+00 1.00e+00f 1 MaxS\n", - " 40 3.8443732e+03 0.00e+00 9.19e-05 -11.0 5.55e-06 - 1.00e+00 1.00e+00f 1 MaxS\n", - " 45 3.8443732e+03 0.00e+00 2.49e-05 -11.0 1.51e-06 - 1.00e+00 1.00e+00f 1 MaxS\n", - "iter objective inf_pr inf_du lg(mu) ||d|| lg(rg) alpha_du alpha_pr ls\n", - " 50 3.8443732e+03 0.00e+00 6.76e-06 -11.0 4.08e-07 - 1.00e+00 1.00e+00f 1 MaxSA\n", - " 55 3.8443732e+03 0.00e+00 1.83e-06 -11.0 1.11e-07 - 1.00e+00 1.00e+00f 1 MaxSA\n", - " 60 3.8443732e+03 0.00e+00 4.97e-07 -11.0 3.00e-08 - 1.00e+00 1.00e+00f 1 MaxSA\n", - "\n", - "Number of Iterations....: 63\n", - "\n", - " (scaled) (unscaled)\n", - "Objective...............: 3.4496886481728779e+02 3.8443731733053728e+03\n", - "Dual infeasibility......: 2.2693631701157965e-07 2.5290047251948971e-06\n", - "Constraint violation....: 0.0000000000000000e+00 0.0000000000000000e+00\n", - "Complementarity.........: 0.0000000000000000e+00 0.0000000000000000e+00\n", - "Overall NLP error.......: 2.2693631701157965e-07 2.5290047251948971e-06\n", - "\n", - "\n", - "Number of objective function evaluations = 64\n", - "Number of objective gradient evaluations = 64\n", - "Number of equality constraint evaluations = 0\n", - "Number of inequality constraint evaluations = 0\n", - "Number of equality constraint Jacobian evaluations = 0\n", - "Number of inequality constraint Jacobian evaluations = 0\n", - "Number of Lagrangian Hessian evaluations = 63\n", - "Total CPU secs in IPOPT (w/o function evaluations) = 1.488\n", - "Total CPU secs in NLP function evaluations = 0.375\n", - "\n", - "EXIT: Solved To Acceptable Level.\n", - " 2.486697 seconds (4.13 M allocations: 201.947 MiB, 3.03% gc time)\n" - ] - } - ], - "source": [ - "# Fisher scoring using Ipopt\n", - "vcmodel_ipopt = deepcopy(vcmodel)\n", - "@time mle_fs!(vcmodel_ipopt, vcdatarot; solver=:Ipopt, maxiter=1000, verbose=true);" - ] - }, - { - "cell_type": "code", - "execution_count": 43, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "2×2 Array{Float64,2}:\n", - " 1.092 1.04727\n", - " 0.955346 1.01632" - ] - }, - "execution_count": 43, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# Ipopt estimates\n", - "vcmodel_ipopt.B" - ] - }, - { - "cell_type": "code", - "execution_count": 44, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "([0.380552 -0.305594; -0.305594 4.52106], [1.84008 0.265385; 0.265385 2.17287])" - ] - }, - "execution_count": 44, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# Ipopt estimates\n", - "vcmodel_ipopt.Σ" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Knitro is a commercial software and users need to follow instructions at [KNITRO.jl](https://github.com/JuliaOpt/KNITRO.jl) for proper functioning. Following code invokes Knitro as the backend optimization solver.\n", - "```julia\n", - "using KNITRO\n", - "\n", - "# Fisher scoring using Knitro\n", - "vcmodel_knitro = deepcopy(vcmodel)\n", - "@time mle_fs!(vcmodel_knitro, vcdatarot; solver=:Knitro, maxiter=1000, verbose=true);\n", - "\n", - "# Knitro estimates\n", - "vcmodel_knitro.B\n", - "\n", - "# Knitro estimates\n", - "vcmodel_knitro.Σ\n", - "```" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Starting point" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Here are a few strategies for successful optimization. " - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "* For $d>1$ (multivariate response), initialize $B, \\Sigma$ from univariate estimates. \n", - "* Use REML estimate as starting point for MLE. \n", - "* When there are only $m=2$ variance components, pre-compute `TwoVarCompVariateRotate` and use it for optimization." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Constrained estimation of `B`\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Many applications invoke constraints on the mean parameters `B`. For demonstration, we enforce `B[1,1]=B[1,2]` and all entries of `B` are within [0, 2]." - ] - }, - { - "cell_type": "code", - "execution_count": 45, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "VarianceComponentModel{Float64,2,Array{Float64,2},Array{Float64,2}}([0.0 0.0; 0.0 0.0], ([1.0 0.0; 0.0 1.0], [1.0 0.0; 0.0 1.0]), [1.0 0.0 -1.0 0.0], '=', 0.0, 0.0, 2.0)" - ] - }, - "execution_count": 45, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# set up constraints on B\n", - "vcmodel_constr = deepcopy(vcmodel)\n", - "vcmodel_constr.A = [1.0 0.0 -1.0 0.0]\n", - "vcmodel_constr.sense = '='\n", - "vcmodel_constr.b = 0.0\n", - "vcmodel_constr.lb = 0.0\n", - "vcmodel_constr.ub = 2.0\n", - "vcmodel_constr" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We first try the MM algorithm." - ] - }, - { - "cell_type": "code", - "execution_count": 46, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - " MM Algorithm\n", - " Iter Objective \n", - "-------- -------------\n", - " 0 -6.253551e+03\n", - " 1 -3.881820e+03\n", - " 2 -3.853477e+03\n", - " 3 -3.846807e+03\n", - " 4 -3.845184e+03\n", - " 5 -3.844783e+03\n", - " 6 -3.844683e+03\n", - " 7 -3.844658e+03\n", - " 8 -3.844652e+03\n", - " 9 -3.844650e+03\n", - " 10 -3.844650e+03\n", - "\n", - " 0.170236 seconds (170.93 k allocations: 8.918 MiB)\n" - ] - } - ], - "source": [ - "# MM algorithm for constrained estimation of B\n", - "@time mle_mm!(vcmodel_constr, vcdatarot; maxiter=10000, funtol=1e-8, verbose = true);" - ] - }, - { - "cell_type": "code", - "execution_count": 48, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(:B, :Σ, :A, :sense, :b, :lb, :ub)" - ] - }, - "execution_count": 48, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "fieldnames(typeof(vcmodel_constr))" - ] - }, - { - "cell_type": "code", - "execution_count": 49, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "2×2 Array{Float64,2}:\n", - " 1.07177 1.07177\n", - " 0.955683 1.01591" - ] - }, - "execution_count": 49, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "vcmodel_constr.B" - ] - }, - { - "cell_type": "code", - "execution_count": 50, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "([0.380624 -0.305498; -0.305498 4.51948], [1.84051 0.265065; 0.265065 2.17336])" - ] - }, - "execution_count": 50, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "vcmodel_constr.Σ" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now let's try Fisher scoring." - ] - }, - { - "cell_type": "code", - "execution_count": 51, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "This is Ipopt version 3.12.10, running with linear solver mumps.\n", - "NOTE: Other linear solvers might be more efficient (see Ipopt documentation).\n", - "\n", - "Number of nonzeros in equality constraint Jacobian...: 0\n", - "Number of nonzeros in inequality constraint Jacobian.: 0\n", - "Number of nonzeros in Lagrangian Hessian.............: 21\n", - "\n", - "Total number of variables............................: 6\n", - " variables with only lower bounds: 0\n", - " variables with lower and upper bounds: 0\n", - " variables with only upper bounds: 0\n", - "Total number of equality constraints.................: 0\n", - "Total number of inequality constraints...............: 0\n", - " inequality constraints with only lower bounds: 0\n", - " inequality constraints with lower and upper bounds: 0\n", - " inequality constraints with only upper bounds: 0\n", - "\n", - "iter objective inf_pr inf_du lg(mu) ||d|| lg(rg) alpha_du alpha_pr ls\n", - " 0 4.2114270e+03 0.00e+00 1.00e+02 0.0 0.00e+00 - 0.00e+00 0.00e+00 0 \n", - " 5 3.8448353e+03 0.00e+00 7.87e-01 -11.0 4.94e-02 - 1.00e+00 1.00e+00f 1 MaxS\n", - " 10 3.8446636e+03 0.00e+00 2.25e-01 -11.0 1.38e-02 - 1.00e+00 1.00e+00f 1 MaxS\n", - " 15 3.8446509e+03 0.00e+00 6.23e-02 -11.0 3.78e-03 - 1.00e+00 1.00e+00f 1 MaxS\n", - " 20 3.8446499e+03 0.00e+00 1.70e-02 -11.0 1.03e-03 - 1.00e+00 1.00e+00f 1 MaxS\n", - " 25 3.8446498e+03 0.00e+00 4.61e-03 -11.0 2.79e-04 - 1.00e+00 1.00e+00f 1 MaxS\n", - " 30 3.8446498e+03 0.00e+00 1.25e-03 -11.0 7.56e-05 - 1.00e+00 1.00e+00f 1 MaxS\n", - " 35 3.8446498e+03 0.00e+00 3.39e-04 -11.0 2.05e-05 - 1.00e+00 1.00e+00f 1 MaxS\n", - " 40 3.8446498e+03 0.00e+00 9.19e-05 -11.0 5.56e-06 - 1.00e+00 1.00e+00f 1 MaxS\n", - " 45 3.8446498e+03 0.00e+00 2.49e-05 -11.0 1.51e-06 - 1.00e+00 1.00e+00f 1 MaxS\n", - "iter objective inf_pr inf_du lg(mu) ||d|| lg(rg) alpha_du alpha_pr ls\n", - " 50 3.8446498e+03 0.00e+00 6.76e-06 -11.0 4.08e-07 - 1.00e+00 1.00e+00f 1 MaxSA\n", - " 55 3.8446498e+03 0.00e+00 1.83e-06 -11.0 1.11e-07 - 1.00e+00 1.00e+00h 1 MaxSA\n", - " 60 3.8446498e+03 0.00e+00 4.97e-07 -11.0 3.00e-08 - 1.00e+00 1.00e+00f 1 MaxSA\n", - "\n", - "Number of Iterations....: 63\n", - "\n", - " (scaled) (unscaled)\n", - "Objective...............: 3.4484507551949679e+02 3.8446498170293380e+03\n", - "Dual infeasibility......: 2.2694405212011240e-07 2.5301808562731130e-06\n", - "Constraint violation....: 0.0000000000000000e+00 0.0000000000000000e+00\n", - "Complementarity.........: 0.0000000000000000e+00 0.0000000000000000e+00\n", - "Overall NLP error.......: 2.2694405212011240e-07 2.5301808562731130e-06\n", - "\n", - "\n", - "Number of objective function evaluations = 64\n", - "Number of objective gradient evaluations = 64\n", - "Number of equality constraint evaluations = 0\n", - "Number of inequality constraint evaluations = 0\n", - "Number of equality constraint Jacobian evaluations = 0\n", - "Number of inequality constraint Jacobian evaluations = 0\n", - "Number of Lagrangian Hessian evaluations = 63\n", - "Total CPU secs in IPOPT (w/o function evaluations) = 0.024\n", - "Total CPU secs in NLP function evaluations = 0.625\n", - "\n", - "EXIT: Solved To Acceptable Level.\n", - " 0.729483 seconds (102.83 k allocations: 8.150 MiB, 0.88% gc time)\n" - ] - } - ], - "source": [ - "# Fisher scoring using Ipopt for constrained estimation of B\n", - "vcmodel_constr = deepcopy(vcmodel)\n", - "vcmodel_constr.A = [1.0 0.0 -1.0 0.0]\n", - "vcmodel_constr.sense = '='\n", - "vcmodel_constr.b = 0.0\n", - "vcmodel_constr.lb = 0.0\n", - "vcmodel_constr.ub = 2.0\n", - "vcmodel_constr\n", - "@time mle_fs!(vcmodel_constr, vcdatarot; solver=:Ipopt, maxiter=1000, verbose=true);" - ] - }, - { - "cell_type": "code", - "execution_count": 52, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "2×2 Array{Float64,2}:\n", - " 1.07177 1.07177\n", - " 0.955683 1.01591" - ] - }, - "execution_count": 52, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "vcmodel_constr.B" - ] - }, - { - "cell_type": "code", - "execution_count": 53, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "([0.380539 -0.305626; -0.305626 4.52116], [1.8405 0.264881; 0.264881 2.17348])" - ] - }, - "execution_count": 53, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "vcmodel_constr.Σ" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Julia 0.7.0", - "language": "julia", - "name": "julia-0.7" - }, - "language_info": { - "file_extension": ".jl", - "mimetype": "application/julia", - "name": "julia", - "version": "0.7.0" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} From d1f24303c7b8fdd5ee357252223f0b264c27a02b Mon Sep 17 00:00:00 2001 From: Juhyun Kim Date: Wed, 30 Jan 2019 16:25:09 -0800 Subject: [PATCH 56/56] rename files --- docs/{heritability1.1.ipynb => heritability.ipynb} | 0 docs/{mle_reml1.1.ipynb => mle_reml.ipynb} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename docs/{heritability1.1.ipynb => heritability.ipynb} (100%) rename docs/{mle_reml1.1.ipynb => mle_reml.ipynb} (100%) diff --git a/docs/heritability1.1.ipynb b/docs/heritability.ipynb similarity index 100% rename from docs/heritability1.1.ipynb rename to docs/heritability.ipynb diff --git a/docs/mle_reml1.1.ipynb b/docs/mle_reml.ipynb similarity index 100% rename from docs/mle_reml1.1.ipynb rename to docs/mle_reml.ipynb