diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..7f5db2f --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,27 @@ +name: Documentation + +on: + push: + branches: + - master + tags: '*' + pull_request: + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: julia-actions/setup-julia@latest + with: + version: '1.6' + - name: Install LuaLatex + run: sudo apt-get install texlive-full && sudo apt-get install texlive-latex-extra && sudo mktexlsr && sudo updmap-sys + - name: Install dependencies + run: julia --project=docs/ -e 'using Pkg; Pkg.develop(PackageSpec(path=pwd())); Pkg.instantiate()' + - name: Build and deploy + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # For authentication with GitHub Actions token + DOCUMENTER_KEY: ${{ secrets.DOCUMENTER_KEY }} # For authentication with SSH deploy key + run: julia --project=docs/ docs/make.jl + diff --git a/.gitignore b/.gitignore index 46e1686..16308b6 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,7 @@ .DS_Store +doc/build/ +doc/site/ +/docs/build/ +/.idea/ diff --git a/.travis.yml b/.travis.yml index cbf11e8..5eb1272 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,14 +1,34 @@ +# Documentation: http://docs.travis-ci.com/user/languages/julia language: julia -coveralls: true -os: - - linux - - osx - - windows +notifications: + email: false julia: - 1.0 - - 1 -matrix: + - 1.5 + - nightly +os: + - linux + +cache: + directories: + - ~/.julia/artifacts +jobs: + fast_finish: true allow_failures: - - julia: 1.0 -notifications: - email: false + - julia: nightly + include: + - stage: "Documentation" + julia: 1.5 + os: linux + script: + - julia --project=docs/ -e 'using Pkg; Pkg.develop(PackageSpec(path=pwd())); + Pkg.instantiate()' + - julia --project=docs/ docs/make.jl + after_success: skip +after_success: + - | + julia -e ' + using Pkg + Pkg.add("Coverage") + using Coverage + Coveralls.submit(process_folder())' \ No newline at end of file diff --git a/Project.toml b/Project.toml index 90500f2..1d846c4 100644 --- a/Project.toml +++ b/Project.toml @@ -22,6 +22,8 @@ SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" SpecialFunctions = "276daf66-3868-5448-9aa4-cd146d93841b" Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" +Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" + [compat] DataFrames = "0.21,0.22" @@ -40,6 +42,7 @@ Requires = "1.0.1" SpecialFunctions = "0.8,0.10,1.0,1.1,1.2" StatsBase = "0.25,0.26,0.27,0.28,0.29,0.30,0.31,0.32,0.33" julia = "1" +Documenter = "0.26" [extras] LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" diff --git a/README.md b/README.md index c21f74e..8534e23 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,6 @@ This library supports representation, inference, and learning in Bayesian networks. -Please read the [documentation](http://nbviewer.ipython.org/github/sisl/BayesNets.jl/blob/master/doc/BayesNets.ipynb). +Please read the [documentation](https://sisl.github.io/BayesNets.jl/dev/index.html). -[![Build Status](https://travis-ci.org/sisl/BayesNets.jl.svg?branch=master)](https://travis-ci.org/sisl/BayesNets.jl) [![Coverage Status](https://coveralls.io/repos/sisl/BayesNets.jl/badge.svg?branch=master&service=github)](https://coveralls.io/github/sisl/BayesNets.jl?branch=master) +[![Build Status](https://github.com/sisl/BayesNets.jl/actions/workflows/main.yml/badge.svg)](https://travis-ci.org/sisl/BayesNets.jl) [![Coverage Status](https://coveralls.io/repos/sisl/BayesNets.jl/badge.svg?branch=master&service=github)](https://coveralls.io/github/sisl/BayesNets.jl?branch=master) diff --git a/doc/.gitignore b/docs/.gitignore similarity index 100% rename from doc/.gitignore rename to docs/.gitignore diff --git a/doc/BayesNets.ipynb b/docs/BayesNets.ipynb similarity index 100% rename from doc/BayesNets.ipynb rename to docs/BayesNets.ipynb diff --git a/docs/Project.toml b/docs/Project.toml new file mode 100644 index 0000000..15aa336 --- /dev/null +++ b/docs/Project.toml @@ -0,0 +1,8 @@ +[deps] +Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" + +[compat] +Documenter = "0.26" + +[extras] +TikzGraphs = "b4f28e30-c73f-5eaf-a395-8a9db949a742" \ No newline at end of file diff --git a/docs/make.jl b/docs/make.jl new file mode 100644 index 0000000..59a3c10 --- /dev/null +++ b/docs/make.jl @@ -0,0 +1,29 @@ +push!(LOAD_PATH, "../src") +import Pkg +Pkg.add("BayesNets") +Pkg.add("TikzPictures") +Pkg.add("TikzGraphs") +Pkg.add("Documenter") +Pkg.add("Discretizers") +Pkg.add("RDatasets") +using Documenter, BayesNets, TikzGraphs, TikzPictures, Discretizers, RDatasets + +makedocs( + modules = [BayesNets, TikzPictures, TikzGraphs, Discretizers, RDatasets], + format = Documenter.HTML( + mathengine = Documenter.MathJax2() + ), + sitename = "BayesNets.jl", + pages = [ + "Table of Contents" => [ + "index.md", + "install.md", + "usage.md", + "concepts.md" + ] + ] +) + +deploydocs( + repo = "github.com/sisl/BayesNets.jl.git", +) \ No newline at end of file diff --git a/docs/src/concepts.md b/docs/src/concepts.md new file mode 100644 index 0000000..5a25a5a --- /dev/null +++ b/docs/src/concepts.md @@ -0,0 +1,10 @@ +# Concepts + + +## Bayesian Networks + +A Bayesian Network (BN) represents a probability distribution over a set of variables, ``P(x_1, x_2, \ldots, x_n)``. +Bayesian networks leverage variable relations in order to efficiently decompose the joint distribution into smaller conditional probability distributions. + +A BN is defined by a directed acyclic graph and a set of conditional probability distributions. Each node in the graph corresponds to a variable ``x_i`` and is associated with a conditional probability distribution ``P(x_i \mid \text{parents}(x_i))``. + diff --git a/docs/src/index.md b/docs/src/index.md new file mode 100644 index 0000000..b3c3070 --- /dev/null +++ b/docs/src/index.md @@ -0,0 +1,9 @@ +# [BayesNets](https://github.com/sisl/BayesNets.jl) + +*A Julia package for Bayesian Networks* + +### Table of Contents + +```@contents +Pages = ["install.md", "usage.md", "concepts.md] +``` \ No newline at end of file diff --git a/docs/src/install.md b/docs/src/install.md new file mode 100644 index 0000000..8d0f814 --- /dev/null +++ b/docs/src/install.md @@ -0,0 +1,6 @@ +# Installation + +```julia +Pkg.add("BayesNets"); +``` +Default visualization of the network structure is provided by the GraphPlot package. However, we recommend using tex-formatted graphs provided by the TikzGraphs package. Installation requirements for TikzGraphs (e.g., PGF/Tikz and pdf2svg) are provided [here](http://nbviewer.ipython.org/github/sisl/TikzGraphs.jl/blob/master/doc/TikzGraphs.ipynb). Simply run using `TikzGraphs` in your Julia session to automatically switch to tex-formatted graphs (thanks to the Requires.jl package). \ No newline at end of file diff --git a/docs/src/usage.md b/docs/src/usage.md new file mode 100644 index 0000000..578f11c --- /dev/null +++ b/docs/src/usage.md @@ -0,0 +1,374 @@ +# Usage + +```@setup bayesnet +using BayesNets, TikzGraphs, TikzPictures +``` + +```julia +using Random +Random.seed!(0) # seed the random number generator to 0, for a reproducible demonstration +using BayesNets +using TikzGraphs # required to plot tex-formatted graphs (recommended), otherwise GraphPlot.jl is used +using TikzPictures +``` + +## Representation + +Bayesian Networks are represented with the `BayesNet` type. This type contains the directed acyclic graph (a LightTables.DiGraph) and a list of conditional probability distributions (a list of CPDs). +Here we construct the BayesNet $a \rightarrow b$, with Gaussians $a$ and $b$: + +```math +a = \mathcal{N}(0,1) \qquad b = \mathcal{N}(2a +3,1) +``` + +```@example bayesnet +bn = BayesNet() +push!(bn, StaticCPD(:a, Normal(1.0))) +push!(bn, LinearGaussianCPD(:b, [:a], [2.0], 3.0, 1.0)) +plot = BayesNets.plot(bn) +TikzPictures.save(SVG("plot1"), plot) +``` + +![](plot1.svg) + +## Conditional Probability Distributions + +Conditional Probablity Distributions, $P(x_i \mid \text{parents}(x_i))$, are defined in BayesNets.CPDs. Each CPD knows its own name, the names of its parents, and is associated with a distribution from Distributions.jl. + +| `CPDForm` | Description | +| ------------------------------ | ----------- | +| `StaticCPD` | Any `Distributions.distribution`; indepedent of any parents | +| `FunctionalCPD` | Allows for a CPD defined with a custom eval function | +| `ParentFunctionalCPD` | Modification to `FunctionalCPD` allowing the parent values to be passed in | +| `CategoricalCPD` | Categorical distribution, assumes integer parents in $1:N$ | +| `LinearGaussianCPD` | Linear Gaussian, assumes target and parents are numeric | +| `ConditionalLinearGaussianCPD` | A linear Gaussian for each discrete parent instantiation| + +Each CPD can be learned from data using `fit`. +Here we learn the same network as above. + +```@example bayesnet +a = randn(100) +b = randn(100) .+ 2*a .+ 3 + +data = DataFrame(a=a, b=b) +cpdA = fit(StaticCPD{Normal}, data, :a) +cpdB = fit(LinearGaussianCPD, data, :b, [:a]) + +bn2 = BayesNet([cpdA, cpdB]) +plot = BayesNets.plot(bn2) # hide +TikzPictures.save(SVG("plot2"), plot) # hide +``` + +![](plot2.svg) + +Each `CPD` implements four functions: + +* `name(cpd)` - obtain the name of the variable target variable +* `parents(cpd)` - obtain the list of parents +* `nparams(cpd` - obtain the number of free parameters in the CPD +* `cpd(assignment)` - allows calling `cpd()` to obtain the conditional distribution +* `Distributions.fit(Type{CPD}, data, target, parents)` + +```@example bayesnet +cpdB(:a=>0.5) +``` + +Several functions conveniently condition and then produce their return values: + +```julia +rand(cpdB, :a=>0.5) # condition and then sample +pdf(cpdB, :a=>1.0, :b=>3.0) # condition and then compute pdf(distribution, 3) +logpdf(cpdB, :a=>1.0, :b=>3.0) # condition and then compute logpdf(distribution, 3); +``` + +The NamedCategorical distribution allows for String or Symbol return values. The FunctionalCPD allows for crafting quick and simple CPDs: + +```@example bayesnet +bn2 = BayesNet() +push!(bn2, StaticCPD(:sighted, NamedCategorical([:bird, :plane, :superman], [0.40, 0.55, 0.05]))) +push!(bn2, FunctionalCPD{Bernoulli}(:happy, [:sighted], a->Bernoulli(a == :superman ? 0.95 : 0.2))) +plot = BayesNets.plot(bn2) # hide +TikzPictures.save(SVG("plot3"), plot) # hide +``` + +![](plot3.svg) + +Variables can be removed by name using `delete!`. A warning will be issued when removing a CPD with children. + +```@example bayesnet +delete!(bn2, :happy) +plot = BayesNets.plot(bn2) # hide +TikzPictures.save(SVG("plot4"), plot) # hide +``` + +![](plot4.svg) + +## Likelihood + +A Bayesian Network represents a joint probability distribution, $P(x_1, x_2, \ldots, x_n)$. +Assignments are represented as dictionaries mapping variable names (Symbols) to variable values. +We can evaluate probabilities as we would with Distributions.jl, only we use exclamation points as we modify the internal state when we condition: + + +```@example bayesnet +pdf(bn, :a=>0.5, :b=>2.0) # evaluate the probability density +``` + +We can also evaluate the likelihood of a dataset: + +```julia +data = DataFrame(a=[0.5,1.0,2.0], b=[4.0,5.0,7.0]) +pdf(bn, data) # 0.00215 +logpdf(bn, data) # -6.1386; +``` + +Or the likelihood for a particular cpd: + +```@example bayesnet +pdf(cpdB, data) # 0.006 +logpdf(cpdB, data) # -5.201 +``` + +## Sampling + +Assignments can be sampled from a `BayesNet`. + +```@example bayesnet +rand(bn) +``` + +```@example bayesnet +rand(bn, 5) +``` + +In general, sampling can be done according to `rand(BayesNet, BayesNetSampler, nsamples)` to produce a table of samples, `rand(BayesNet, BayesNetSampler)` to produce a single Assignment, or `rand!(Assignment, BayesNet, BayesNetSampler)` to modify an assignment in-place. +New samplers need only implement `rand!`. +The functions above default to the `DirectSampler`, which samples the variables in topographical order. + +Rejection sampling can be used to draw samples that are consistent with a provided assignment: + +```@example bayesnet +bn = BayesNet() +push!(bn, StaticCPD(:a, Categorical([0.3,0.7]))) +push!(bn, StaticCPD(:b, Categorical([0.6,0.4]))) +push!(bn, CategoricalCPD{Bernoulli}(:c, [:a, :b], [2,2], [Bernoulli(0.1), Bernoulli(0.2), Bernoulli(1.0), Bernoulli(0.4)])) +plot = BayesNets.plot(bn) # hide +TikzPictures.save(SVG("plot5"), plot) # hide +``` + +![](plot5.svg) + +```julia +rand(bn, RejectionSampler(:c=>1), 5) +``` + +One can also use weighted sampling: + +```julia +rand(bn, LikelihoodWeightedSampler(:c=>1), 5) +``` + +One can also use Gibbs sampling. More options are available than are shown in the example below. + +```julia +bn_gibbs = BayesNet() +push!(bn_gibbs, StaticCPD(:a, Categorical([0.999,0.001]))) +push!(bn_gibbs, StaticCPD(:b, Normal(1.0))) +push!(bn_gibbs, LinearGaussianCPD(:c, [:a, :b], [3.0, 1.0], 0.0, 1.0)) + +evidence = Assignment(:c => 10.0) +initial_sample = Assignment(:a => 1, :b => 1, :c => 10.0) +gsampler = GibbsSampler(evidence, burn_in=500, thinning=1, initial_sample=initial_sample) +rand(bn_gibbs, gsampler, 5) +``` + +## Parameter Learning + +BayesNets.jl supports parameter learning for an entire graph. + +```julia +fit(BayesNet, data, (:a=>:b), [StaticCPD{Normal}, LinearGaussianCPD]) +``` + +```julia +fit(BayesNet, data, (:a=>:b), LinearGaussianCPD) +``` +Fitting can be done for specific BayesNets types as well: + +```@example bayesnet +data = DataFrame(c=[1,1,1,1,2,2,2,2,3,3,3,3], +b=[1,1,1,2,2,2,2,1,1,2,1,1], +a=[1,1,1,2,1,1,2,1,1,2,1,1]) + +bn5 = fit(DiscreteBayesNet, data, (:a=>:b, :a=>:c, :b=>:c)) +plot = BayesNets.plot(bn5) # hide +TikzPictures.save(SVG("plot6"), plot) # hide +``` + +![](plot6.svg) + +Fitting a ```DiscreteCPD```, which is a ```CategoricalCPD{Categorical}```, can be done with a specified number of categories. This prevents cases where your test data does not provide an example for every category. + +```julia +cpd = fit(DiscreteCPD, DataFrame(a=[1,2,1,2,2]), :a, ncategories=3); +cpd = fit(DiscreteCPD, data, :b, [:a], parental_ncategories=[3], target_ncategories=3); +``` + +## Inference + +Inference methods for discrete Bayesian networks can be used via the `infer` method: + +```@example bayesnet +bn = DiscreteBayesNet() +push!(bn, DiscreteCPD(:a, [0.3,0.7])) +push!(bn, DiscreteCPD(:b, [0.2,0.8])) +push!(bn, DiscreteCPD(:c, [:a, :b], [2,2], + [Categorical([0.1,0.9]), + Categorical([0.2,0.8]), + Categorical([1.0,0.0]), + Categorical([0.4,0.6]), + ])) + +plot = BayesNets.plot(bn) # hide +TikzPictures.save(SVG("plot7"), plot) # hide +``` + +![](plot7.svg) + +```@example bayesnet +ϕ = infer(bn, :c, evidence=Assignment(:b=>1)) +``` + +Several inference methods are available. Exact inference is the default. + +| `Inference Method` | Description | +| ------------------ | ----------- | +| `ExactInference` | Performs exact inference using discrete factors and variable elimination| +| `LikelihoodWeightingInference` | Approximates p(query \ evidence) with N weighted samples using likelihood weighted sampling | +| `LoopyBelief` | The loopy belief propagation algorithm | +| `GibbsSamplingNodewise` | Gibbs sampling where each iteration changes one node | +| `GibbsSamplingFull` | Gibbs sampling where each iteration changes all nodes | + +```julia +ϕ = infer(GibbsSamplingNodewise(), bn, [:a, :b], evidence=Assignment(:c=>2)) +``` + +Inference produces a `Factor` type. It can be converted to a DataFrame. + +```julia +convert(DataFrame, ϕ) +``` + +## Structure Learning + +Structure learning can be done as well. + +```@example bayesnet +using Discretizers +using RDatasets +iris = dataset("datasets", "iris") +names(iris) +data = DataFrame( + SepalLength = iris[!,:SepalLength], + SepalWidth = iris[!,:SepalWidth], + PetalLength = iris[!,:PetalLength], + PetalWidth = iris[!,:PetalWidth], + Species = encode(CategoricalDiscretizer(iris[!,:Species]), iris[!,:Species]), +) + +data[1:3,:] # only display a subset... +``` + +Here we use the K2 structure learning algorithm which runs in polynomial time but requires that we specify a topological node ordering. + +```@example bayesnet +parameters = K2GraphSearch([:Species, :SepalLength, :SepalWidth, :PetalLength, :PetalWidth], + ConditionalLinearGaussianCPD, + max_n_parents=2) +bn = fit(BayesNet, data, parameters) + +plot = BayesNets.plot(bn) # hide +TikzPictures.save(SVG("plot8"), plot) # hide +``` + +![](plot8.svg) + +CPD types can also be specified per-node. Note that complete CPD definitions are required - simply using `StaticCPD` is insufficient as you need the target distribution type as well, as in `StaticCPD{Categorical}`. + +Changing the ordering will change the structure. + +```julia +CLG = ConditionalLinearGaussianCPD +parameters = K2GraphSearch([:Species, :PetalLength, :PetalWidth, :SepalLength, :SepalWidth], + [StaticCPD{Categorical}, CLG, CLG, CLG, CLG], + max_n_parents=2) +fit(BayesNet, data, parameters) +``` + +A `ScoringFunction` allows for extracting a scoring metric for a CPD given data. The negative BIC score is implemented in `NegativeBayesianInformationCriterion`. + +A `GraphSearchStrategy` defines a structure learning algorithm. The K2 algorithm is defined through `K2GraphSearch` and `GreedyHillClimbing` is implemented for discrete Bayesian networks and the Bayesian score: + +```@example bayesnet +data = DataFrame(c=[1,1,1,1,2,2,2,2,3,3,3,3], + b=[1,1,1,2,2,2,2,1,1,2,1,1], + a=[1,1,1,2,1,1,2,1,1,2,1,1]) +parameters = GreedyHillClimbing(ScoreComponentCache(data), max_n_parents=3, prior=UniformPrior()) +bn = fit(DiscreteBayesNet, data, parameters) + +plot = BayesNets.plot(bn) # hide +TikzPictures.save(SVG("plot9"), plot) # hide +``` + +![](plot9.svg) + +We can specify the number of categories for each variable in case it cannot be correctly inferred: + +```julia +bn = fit(DiscreteBayesNet, data, parameters, ncategories=[3,3,2]) +``` + +A whole suite of features are supported for DiscreteBayesNets. Here, we illustrate the following: + +1. Obtain a list of counts for a node +2. Obtain sufficient statistics from a discrete dataset +3. Obtain the factor table for a node +4. Obtain a factor table matching a particular assignment + +We also detail obtaining a bayesian score for a network structure in the next section. + +```julia +count(bn, :a, data) # 1 +statistics(bn.dag, data) # 2 +table(bn, :b) # 3 +table(bn, :c, :a=>1) # 4 +``` + +## Reading from XDSL + +Discrete Bayesian Networks can be read from the .XDSL file format. + +```@example bayesnet +bn = readxdsl(joinpath(dirname(pathof(BayesNets)), "..", "test", "sample_bn.xdsl")) + +plot = BayesNets.plot(bn) # hide +TikzPictures.save(SVG("plot10"), plot) # hide +``` + +![](plot10.svg) + +## Bayesian Score for a Network Structure + +The bayesian score for a discrete-valued BayesNet can can be calculated based only on the structure and data (the CPDs do not need to be defined beforehand). This is implemented with a method of ```bayesian_score``` that takes in a directed graph, the names of the nodes and data. + +```@example bayesnet +data = DataFrame(c=[1,1,1,1,2,2,2,2,3,3,3,3], + b=[1,1,1,2,2,2,2,1,1,2,1,1], + a=[1,1,1,2,1,1,2,1,1,2,1,1]) +g = DAG(3) +add_edge!(g,1,2); add_edge!(g,2,3); add_edge!(g,1,3) +bayesian_score(g, [:a,:b,:c], data) +``` + + diff --git a/test/test_docs.jl b/test/test_docs.jl index df2374c..368bb80 100644 --- a/test/test_docs.jl +++ b/test/test_docs.jl @@ -1,5 +1,5 @@ using NBInclude let - @nbinclude(joinpath(dirname(@__DIR__), "doc", "BayesNets.ipynb")) + @nbinclude(joinpath(dirname(@__DIR__), "docs", "BayesNets.ipynb")) end